package urpm::media;
# $Id: media.pm 270394 2010-07-30 00:48:07Z misc $
use strict;
use urpm qw(file_from_local_medium is_local_medium);
use urpm::msg;
use urpm::util qw(any append_to_file basename cat_ difference2 dirname intersection member output_safe begins_with copy_and_own file_size offset_pathname reduce_pathname);
use urpm::removable;
use urpm::lock;
use urpm::md5sum;
use MDV::Distribconf;
our @PER_MEDIA_OPT = qw(
disable-certificate-check
downloader
ignore
key-ids
list
media_info_dir
mirrorlist
name
no-media-info
noreconfigure
priority-upgrade
removable
static
synthesis
update
url
verify-rpm
virtual
with-dir
with_hdlist
with_synthesis
xml-info
);
my @xml_media_info = ('info', 'files', 'changelog');
my @media_info_prefix_suffix = (
[ 'synthesis.hdlist', '.cz' ],
[ 'hdlist', '.cz' ],
[ 'descriptions', '' ],
[ 'names', '' ],
[ 'MD5SUM', '' ],
(map { [ $_, '.xml.lzma' ] } @xml_media_info),
);
=head1 NAME
urpm::media - Media routines for urpmi
=head1 SYNOPSIS
=head1 DESCRIPTION
=over
=cut
sub get_medium_option {
my ($urpm, $medium, $option_name) = @_;
defined $medium->{$option_name} ? $medium->{$option_name} : $urpm->{options}{$option_name};
}
sub _is_iso {
my ($removable_dev) = @_;
$removable_dev && $removable_dev =~ /\.iso$/i;
}
sub only_media_opts {
my ($m) = @_;
my %m = map { $_ => $m->{$_} } grep { defined $m->{$_} } @PER_MEDIA_OPT;
\%m;
}
sub _only_media_opts_read {
my ($m) = @_;
my $c = only_media_opts($m);
$c->{media_info_dir} ||= 'media_info';
$c->{iso} = delete $c->{removable} if $c->{removable} && _is_iso($c->{removable});
$c;
}
sub _only_media_opts_write {
my ($m) = @_;
my $c = only_media_opts($m);
delete $c->{media_info_dir} if $c->{media_info_dir} eq 'media_info';
delete $c->{url} if $c->{mirrorlist};
$c->{removable} = delete $c->{iso} if $c->{iso};
$c;
}
sub read_private_netrc {
my ($urpm) = @_;
my @words = split(/\s+/, scalar cat_($urpm->{private_netrc}));
my @l;
my $e;
while (@words) {
my $keyword = shift @words;
if ($keyword eq 'machine') {
push @l, $e = { machine => shift(@words) };
} elsif ($keyword eq 'default') {
push @l, $e = { default => '' };
} elsif ($keyword eq 'login' || $keyword eq 'password' || $keyword eq 'account') {
$e->{$keyword} = shift(@words);
} else {
$urpm->{error}("unknown netrc command $keyword");
}
}
@l;
}
sub read_config_add_passwords {
my ($urpm, $config) = @_;
my @netrc = read_private_netrc($urpm) or return;
foreach (grep { $_->{url} } @{$config->{media}}) {
my $u = urpm::download::parse_url_with_login($_->{url}) or next;
if (my ($e) = grep { ($_->{default} || $_->{machine} eq $u->{machine}) && $_->{login} eq $u->{login} } @netrc) {
$_->{url} = sprintf('%s://%s:%s@%s%s', $u->{proto}, $u->{login}, $e->{password}, $u->{machine}, $u->{dir});
} else {
$urpm->{log}(sprintf('no password found for %s@%s', $u->{login}, $u->{machine}));
}
}
}
sub remove_passwords_and_write_private_netrc {
my ($urpm, $config) = @_;
my @l;
foreach (grep { $_->{url} } @{$config->{media}}) {
my $u = urpm::download::parse_url_with_login($_->{url}) or next;
#- check whether a password is visible
$u->{password} or next;
push @l, $u;
$_->{url} = sprintf('%s://%s@%s%s', $u->{proto}, $u->{login}, $u->{machine}, $u->{dir});
}
{
my $fh = urpm::sys::open_safe($urpm, '>', $urpm->{private_netrc}) or return;
foreach my $u (@l) {
printf $fh "machine %s login %s password %s\n", $u->{machine}, $u->{login}, $u->{password};
}
}
chmod 0600, $urpm->{private_netrc};
}
#- handle deprecated way of saving passwords
sub recover_url_from_list {
my ($urpm, $medium) = @_;
my $list = delete $medium->{list} or return;
my $statedir_list = "$urpm->{statedir}/$list";
#- /./ is end of url marker in list file (typically generated by a
#- find . -name "*.rpm" > list
#- for exportable list file.
if (my @probe = map { m!^(.*)/\./! || m!^(.*)/[^/]*$! } cat_($statedir_list)) {
$urpm->{log}("recovering url from $statedir_list");
($medium->{url}) = sort { length($a) <=> length($b) } @probe;
$urpm->{modified} = 1; #- ensure urpmi.cfg is handled using only partially hidden url + netrc, since file list won't be generated anymore
unlink $statedir_list;
}
}
sub _read_config__read_media_info {
my ($urpm) = @_;
require File::Glob;
# we can't use perl's glob() because there could be spaces in
# $urpm->{mediacfgdir}
my %url2mediamap;
my %mirrorlist2mediamap;
foreach my $media_dir (File::Glob::bsd_glob("$urpm->{mediacfgdir}/*")) {
next if !-d $media_dir;
$urpm->{debug} and $urpm->{debug}("parsing: $media_dir");
my $media_cfg = $media_dir . '/media.cfg';
my $distribconf = MDV::Distribconf->new($media_cfg, undef) or next;
$distribconf->settree('mageia');
$distribconf->parse_mediacfg($media_cfg) or next;
foreach (cat_($media_dir . '/url')) {
chomp($_);
foreach my $medium ($distribconf->listmedia) {
my $medium_path = reduce_pathname($_ . '/' . $distribconf->getpath($medium, 'path'));
$url2mediamap{$medium_path} = [$distribconf, $medium];
}
}
foreach my $mirrorlist (cat_($media_dir . '/mirrorlist')) {
chomp($mirrorlist);
foreach my $medium ($distribconf->listmedia) {
my $medium_path = $distribconf->getpath($medium, 'path');
$mirrorlist2mediamap{$mirrorlist}{$medium_path} = [ $distribconf, $medium ];
}
}
}
(\%url2mediamap, \%mirrorlist2mediamap);
}
sub _associate_media_with_mediacfg {
my ($urpm, $media) = @_;
my ($url2mediamap, $mirrorlist2mediamap) = _read_config__read_media_info($urpm);
foreach my $medium (@$media) {
if ($medium->{mirrorlist}) {
$medium->{mediacfg} = $mirrorlist2mediamap->{$medium->{mirrorlist}}{$medium->{'with-dir'}};
} elsif ($medium->{url}) {
$medium->{mediacfg} = $url2mediamap->{$medium->{url}};
}
}
}
=item read_config($urpm, $nocheck)
Loads /etc/urpmi/urpmi.cfg and performs basic checks.
It does not handle old format: <name> <url> [with <path_hdlist>]
=cut
sub read_config {
my ($urpm, $nocheck) = @_;
return if $urpm->{media}; #- media already loaded
$urpm->{media} = [];
my $config = urpm::cfg::load_config($urpm->{config})
or $urpm->{fatal}(6, $urpm::cfg::err);
#- per-media options
read_config_add_passwords($urpm, $config);
my @media;
foreach my $m (@{$config->{media}}) {
my $medium = _only_media_opts_read($m);
if (!$medium->{url} && !$medium->{mirrorlist}) {
#- recover the url the old deprecated way...
#- only useful for migration, new urpmi.cfg will use netrc
recover_url_from_list($urpm, $medium);
$medium->{url} or $urpm->{error}("unable to find url in list file $medium->{name}, medium ignored");
}
push @media, $medium;
}
# associate medias read from the config file with their description in a
# media.cfg file
# @media content will be modified and then add_existing medium will take
# care of copying the media to $urpm
_associate_media_with_mediacfg($urpm, \@media);
add_existing_medium($urpm, $_, $nocheck) foreach @media;
eval { require urpm::ldap; urpm::ldap::load_ldap_media($urpm) };
}
#- if invalid, set {ignore}
sub check_existing_medium {
my ($urpm, $medium) = @_;
my $err;
if (!$medium->{url} && !$medium->{mirrorlist}) {
$err = $medium->{virtual} ?
N("virtual medium \"%s\" should have a clear url, medium ignored",
$medium->{name}) :
N("unable to access list file of \"%s\", medium ignored", $medium->{name});
} elsif (!$medium->{ignore}
&& !-r any_synthesis($urpm, $medium)) {
$err = N("unable to access synthesis file of \"%s\", medium ignored", $medium->{name});
}
if ($err) {
$medium->{ignore} = 1;
$urpm->{error}($err);
}
}
sub _migrate__with_synthesis {
my ($medium, $with_synthesis) = @_;
#- try to migrate to media_info_dir
my $b = basename($with_synthesis);
if ($b eq 'synthesis.hdlist.cz' || $b eq 'hdlist.cz') {
$medium->{media_info_dir} = dirname($with_synthesis);
} else {
$with_synthesis =~ s/(synthesis\.)?(hdlist.*\.cz)$/synthesis.$2/;
$medium->{with_synthesis} = $with_synthesis;
}
}
#- probe medium to be used, take old medium into account too.
sub add_existing_medium {
my ($urpm, $medium, $nocheck) = @_;
if (name2medium($urpm, $medium->{name})) {
$urpm->{error}(N("trying to override existing medium \"%s\", skipping", $medium->{name}));
return;
}
if ($medium->{with_hdlist}) {
_migrate__with_synthesis($medium, delete $medium->{with_hdlist});
$urpm->{modified} = 1;
}
check_existing_medium($urpm, $medium) if !$nocheck;
_migrate_removable_device($urpm, $medium);
#- clear URLs for trailing /es.
$medium->{url} and $medium->{url} =~ s|(.*?)/*$|$1|;
push @{$urpm->{media}}, $medium;
}
sub file_from_file_url {
my ($url) = @_;
$url =~ m!^(?:file:/)?(/.*)! && $1;
}
sub _local_file {
my ($medium) = @_;
$medium->{url} && file_from_file_url($medium->{url});
}
sub _is_local_virtual {
my ($medium) = @_;
$medium->{virtual} && _local_file($medium);
}
sub _is_remote_virtual {
my ($medium) = @_;
$medium->{virtual} && !_local_file($medium);
}
sub _url_with_synthesis_basename {
my ($medium) = @_;
$medium->{with_synthesis}
? basename($medium->{with_synthesis})
: 'synthesis.hdlist.cz';
}
sub _synthesis_dir_rel {
my ($medium) = @_;
$medium->{'no-media-info'} || $medium->{unknown_media_info} and return;
$medium->{with_synthesis}
? "$medium->{with_synthesis}/.."
: $medium->{media_info_dir};
}
sub _synthesis_dir {
my ($medium) = @_;
my $rel = _synthesis_dir_rel($medium) or return;
my $base = file_from_local_medium($medium) || $medium->{url};
reduce_pathname("$base/$rel");
}
# the difference between _valid_synthesis_dir and _synthesis_dir
# is only to handle upgrades from older urpmi.cfg where no {media_info_dir}
# meant no-media-info
sub _valid_synthesis_dir {
my ($medium) = @_;
my $dir = _synthesis_dir($medium);
$dir && -d $dir;
}
sub _url_with_synthesis_rel {
my ($medium) = @_;
$medium->{with_synthesis} ||
"$medium->{media_info_dir}/synthesis.hdlist.cz";
}
sub _url_with_synthesis {
my ($medium) = @_;
my $rel = _url_with_synthesis_rel($medium) or return;
my $base = file_from_local_medium($medium) || $medium->{url};
reduce_pathname("$base/$rel");
}
sub synthesis {
my ($medium) = @_;
statedir_media_info_basename($medium, 'synthesis.hdlist', '.cz');
}
sub statedir_media_info_basename {
my ($medium, $prefix, $suffix) = @_;
$medium->{name} && "$prefix.$medium->{name}$suffix";
}
sub statedir_media_info_dir {
my ($urpm, $medium) = @_;
$medium->{name} && "$urpm->{statedir}/$medium->{name}";
}
sub old_statedir_media_info_file {
my ($urpm, $medium, $prefix, $suffix) = @_;
$medium->{name} && "$urpm->{statedir}/" . statedir_media_info_basename($medium, $prefix, $suffix);
}
sub new_statedir_media_info_file {
my ($urpm, $medium, $prefix, $suffix) = @_;
my $dir = statedir_media_info_dir($urpm, $medium);
$dir && "$dir/$prefix$suffix";
}
sub statedir_media_info_file {
my ($urpm, $medium, $prefix, $suffix) = @_;
my $dir = statedir_media_info_dir($urpm, $medium);
-d $dir ? "$dir/$prefix$suffix" : old_statedir_media_info_file($urpm, $medium, $prefix, $suffix);
}
sub statedir_synthesis {
my ($urpm, $medium) = @_;
statedir_media_info_file($urpm, $medium, 'synthesis.hdlist', '.cz');
}
sub statedir_descriptions {
my ($urpm, $medium) = @_;
statedir_media_info_file($urpm, $medium, 'descriptions', '');
}
sub statedir_names {
my ($urpm, $medium) = @_;
old_statedir_media_info_file($urpm, $medium, 'names', '');
}
sub statedir_MD5SUM {
my ($urpm, $medium) = @_;
statedir_media_info_file($urpm, $medium, 'MD5SUM', '');
}
sub statedir_hdlist {
my ($urpm, $medium) = @_;
statedir_media_info_file($urpm, $medium, 'hdlist', '.cz');
}
sub statedir_xml_info {
my ($urpm, $medium, $xml_info) = @_;
statedir_media_info_file($urpm, $medium, $xml_info, '.xml.lzma');
}
sub cachedir_with_synthesis {
my ($urpm, $medium) = @_;
_url_with_synthesis($medium) && "$urpm->{cachedir}/partial/synthesis.hdlist.cz";
}
sub any_synthesis {
my ($urpm, $medium) = @_;
my $f = _is_local_virtual($medium) ? _url_with_synthesis($medium)
: statedir_synthesis($urpm, $medium);
-e $f && $f;
}
sub any_media_info_file {
my ($urpm, $medium, $prefix, $suffix, $quiet, $o_callback) = @_;
if (my $base = _local_file($medium)) {
my $f = $medium->{with_synthesis}
? reduce_pathname("$base/$prefix." . _synthesis_suffix($medium) . $suffix)
: _synthesis_dir($medium) . "/$prefix$suffix";
if (! -e $f) {
# in some weird cases (iso on disk), the hdlist is not available where it should be,
# but we can use the statedir copy
$f = statedir_media_info_file($urpm, $medium, $prefix, $suffix);
}
-e $f && $f;
} else {
_any_media_info__or_download($urpm, $medium, $prefix, $suffix, $quiet, $o_callback);
}
}
sub any_hdlist {
my ($urpm, $medium, $quiet) = @_;
any_media_info_file($urpm, $medium, 'hdlist', '.cz', $quiet, \&urpm::download::sync_logger);
}
sub any_xml_info {
my ($urpm, $medium, $xml_info, $quiet, $o_callback) = @_;
any_media_info_file($urpm, $medium, $xml_info, '.xml.lzma', $quiet, $o_callback || \&urpm::download::sync_logger);
}
sub name2medium {
my ($urpm, $name) = @_;
my ($medium) = grep { $_->{name} eq $name } @{$urpm->{media}};
$medium;
}
sub userdirs {
my ($urpm) = @_;
my $prefix = urpm::userdir_prefix($urpm);
grep { m!^\Q$prefix\E\d+$! && -d $_ && ! -l $_ } glob("$prefix*");
}
sub remove_user_media_info_files {
my ($urpm, $medium) = @_;
foreach my $dir (userdirs($urpm)) {
require File::Glob;
# we can't use perl's glob() because $medium->{name} can contain spaces
my @files = map { File::Glob::bsd_glob("$dir/*.$medium->{name}.$_") } 'cz', 'xml.lzma' or next;
$urpm->{log}("cleaning $dir");
foreach (@files) {
unlink $_ or $urpm->{error}("removing $_ failed");
}
}
}
#- probe device associated with a removable device.
sub _migrate_removable_device {
my ($urpm, $medium) = @_;
$medium->{url} or return;
# always drop {removable}, it is obsolete
# (nb: for iso files, {removable} has already been renamed into {iso} internally)
delete $medium->{removable};
if (my $url = _migrate_removable_url($medium->{url})) {
$medium->{url} = $url;
} else {
$urpm->{error}(N("failed to migrate removable device, ignoring media"));
$medium->{ignore} = 1;
}
}
sub _migrate_removable_url {
my ($url) = @_;
if ($url =~ /^removable/) {
$url =~ s!^removable(.*?)://!/!;
if ($url =~ s!/(mnt|media)/cd\w+/?!cdrom://!i) {
# success!
} else {
return;
}
}
$url;
}
=item write_urpmi_cfg($urpm)
Writes the urpmi.cfg file.
=cut
sub write_urpmi_cfg {
my ($urpm) = @_;
#- avoid trashing exiting configuration if it wasn't loaded
$urpm->{media} or return;
my $config = {
#- global config options found in the config file, without the ones
#- set from the command-line
global => $urpm->{global_config},
media => [ map { _only_media_opts_write($_) } grep { !$_->{external} } @{$urpm->{media}} ],
};
remove_passwords_and_write_private_netrc($urpm, $config);
urpm::cfg::dump_config($urpm->{config}, $config)
or $urpm->{fatal}(6, N("unable to write config file [%s]", $urpm->{config}));
$urpm->{log}(N("wrote config file [%s]", $urpm->{config}));
#- everything should be synced now.
delete $urpm->{modified};
}
sub write_config {
my ($urpm) = @_;
write_urpmi_cfg($urpm);
}
sub _tempignore {
my ($medium, $ignore) = @_;
$medium->{ignore} = $ignore;
}
=item configure($urpm, %options)
Read urpmi.cfg file as well as necessary synthesis files.
Options :
=over
=item *
root (deprecated, set directly $urpm->{root})
=item *
cmdline_skiplist
=item *
download_callback (used by _auto_update_media)
=item *
callback (urpmf)
=item *
nodepslist (for urpmq, urpmf: when we don't need the synthesis)
=item *
no_skiplist (urpmf)
=item *
synthesis (use this synthesis file, and only this synthesis file)
=item *
parallel
=item *
usedistrib (otherwise uses urpmi.cfg)
=item *
media
=item *
excludemedia
=item *
sortmedia
=item *
update
=item *
searchmedia
=back
=cut
sub configure {
my ($urpm, %options) = @_;
clean($urpm);
$options{parallel} && $options{usedistrib} and $urpm->{fatal}(1, N("Can't use parallel mode with use-distrib mode"));
if ($options{parallel}) {
require urpm::parallel;
urpm::parallel::configure($urpm, $options{parallel});
if (!$options{media} && $urpm->{parallel_handler}{media}) {
$options{media} = $urpm->{parallel_handler}{media};
$urpm->{log}->(N("using associated media for parallel mode: %s", $options{media}));
}
} else {
#- nb: can't have both parallel and root
$urpm->{root} = $options{root} if $options{root};
}
if ($urpm->{root} && ! -c "$urpm->{root}/dev/null") {
mkdir "$urpm->{root}/dev";
system("/bin/cp", "-a", '/dev/null', "$urpm->{root}/dev");
}
if ($options{synthesis}) {
if ($options{synthesis} ne 'none') {
#- synthesis take precedence over media, update options.
$options{media} || $options{excludemedia} || $options{sortmedia} || $options{update} || $options{usedistrib} || $options{parallel} and
$urpm->{fatal}(1, N("--synthesis cannot be used with --media, --excludemedia, --sortmedia, --update, --use-distrib or --parallel"));
my $synthesis = $options{synthesis};
if ($synthesis !~ m!^/!) {
require Cwd;
$synthesis = Cwd::getcwd() . '/' . $synthesis;
}
my ($url, $with) = $synthesis =~ m!(.*)/+(media_info/+synthesis\.hdlist\.cz)$! ? ($1, $2) :
(dirname($synthesis), basename($synthesis));
$urpm->{media} = [];
add_medium($urpm, 'Virtual', $url, $with, %options, virtual => 1, on_the_fly => 1);
}
} elsif ($options{usedistrib}) {
$urpm->{media} = [];
add_distrib_media($urpm, "Virtual", $options{usedistrib}, %options, virtual => 1, on_the_fly => 1);
} else {
read_config($urpm, '');
if (!$options{media} && $urpm->{options}{'default-media'}) {
$options{media} = $urpm->{options}{'default-media'};
}
}
if ($options{media}) {
delete $_->{modified} foreach @{$urpm->{media} || []};
select_media($urpm, split /,/, $options{media});
foreach (@{$urpm->{media} || []}) {
_tempignore($_, !$_->{modified});
}
}
if ($options{searchmedia}) {
foreach (select_media_by_name($urpm, [ split /,/, $options{searchmedia} ])) {
#- Ensure this media is selected
$_->{modified} = 1;
_tempignore($_, 0);
$_->{searchmedia} = 1;
}
}
if ($options{update}) {
foreach (grep { !$_->{ignore} && $_->{update} } @{$urpm->{media} || []}) {
#- Ensure update media are selected
$_->{modified} = 1;
_tempignore($_, 0);
$_->{searchmedia} = 1;
}
}
if ($options{excludemedia}) {
delete $_->{modified} foreach @{$urpm->{media} || []};
foreach (select_media_by_name($urpm, [ split /,/, $options{excludemedia} ])) {
$_->{modified} = 1;
#- this is only a local ignore that will not be saved.
_tempignore($_, 1);
}
}
if ($options{sortmedia}) {
my @sorted_media = map { select_media_by_name($urpm, [$_]) } split(/,/, $options{sortmedia});
my @remaining = difference2($urpm->{media}, \@sorted_media);
$urpm->{media} = [ @sorted_media, @remaining ];
}
_auto_update_media($urpm, %options);
_pick_mirror_if_needed($urpm, $_, '') foreach non_ignored_media($urpm);
if (!$options{nodepslist}) {
parse_media($urpm, \%options);
#- determine package to withdraw (from skip.list file) only if something should be withdrawn.
_compute_flags_for_skiplist($urpm, $options{cmdline_skiplist}) if !$options{no_skiplist};
_compute_flags_for_instlist($urpm);
}
}
#- for remote "virtual" media
#- options: download_callback, nomd5sum, quiet, nopubkey
sub _auto_update_media {
my ($urpm, %options) = @_;
$options{callback} = delete $options{download_callback};
foreach (grep { _is_remote_virtual($_) || $urpm->{options}{'auto-update'} }
non_ignored_media($urpm)) {
_update_medium($urpm, $_, %options);
}
}
=item needed_extra_media($urpm)
Return 2 booleans telling whether nonfree & tainted packages are installed respectively.
=cut
sub needed_extra_media {
my ($urpm) = @_;
my $db = urpm::db_open_or_die_($urpm);
my ($nonfree, $tainted);
$db->traverse(sub {
my ($pkg) = @_;
return if $nonfree && $tainted;
my $rel = $pkg->release;
$nonfree ||= $rel =~ /nonfree$/;
$tainted ||= $rel =~ /tainted$/;
});
($nonfree, $tainted);
}
sub is_media_to_add_by_default {
my ($urpm, $distribconf, $medium, $product_id, $nonfree, $tainted) = @_;
my $add_by_default = !$distribconf->getvalue($medium, 'noauto');
my @media_types = split(':', $distribconf->getvalue($medium, 'media_type'));
return $add_by_default if !@media_types;
if ($product_id->{product} eq 'Free') {
if (member('non-free', @media_types)) {
$urpm->{log}(N("ignoring non-free medium `%s'", $medium));
$add_by_default = 0;
}
} else {
my $non_regular_medium = intersection(\@media_types, [ qw(backports debug source testing) ]);
if (!$add_by_default && !$non_regular_medium) {
my $medium_name = $distribconf->getvalue($medium, 'name') || '';
if ($medium_name =~ /Nonfree/ && $nonfree) {
$add_by_default = 1;
$urpm->{log}(N("un-ignoring non-free medium `%s' b/c nonfree packages are installed", $medium_name));
}
if ($medium_name =~ /Tainted/ && $tainted) {
$add_by_default = 1;
$urpm->{log}(N("un-ignoring tainted medium `%s' b/c tainted packages are installed", $medium_name));
}
}
}
$add_by_default;
}
sub non_ignored_media {
my ($urpm, $b_only_marked_update) = @_;
grep { !$_->{ignore} && (!$b_only_marked_update || $_->{update}) } @{$urpm->{media} || []};
}
sub all_media_to_update {
my ($urpm, $b_only_marked_update) = @_;
grep { !$_->{static} && !urpm::is_cdrom_url($_->{url}) && !$_->{iso} } non_ignored_media($urpm, $b_only_marked_update);
}
sub parse_media {
my ($urpm, $options) = @_;
foreach (non_ignored_media($urpm)) {
delete @$_{qw(start end)};
_parse_synthesis_or_ignore($urpm, $_, $options->{callback});
if ($_->{searchmedia}) {
$urpm->{searchmedia} = 1;
$urpm->{debug} and $urpm->{debug}(N("Search start: %s end: %s", $_->{start}, $_->{end}));
}
$< == 0 and _generate_medium_names($urpm, $_);
}
}
sub _compute_flags_for_skiplist {
my ($urpm, $cmdline_skiplist) = @_;
my %uniq;
$urpm->compute_flags(
urpm::sys::get_packages_list($urpm->{skiplist}, $cmdline_skiplist),
skip => 1,
callback => sub {
my ($urpm, $pkg) = @_;
$pkg->is_arch_compat && ! exists $uniq{$pkg->fullname} or return;
$uniq{$pkg->fullname} = undef;
$urpm->{debug} and $urpm->{debug}(N("skipping package %s", scalar($pkg->fullname)));
},
);
}
sub _compute_flags_for_instlist {
my ($urpm) = @_;
my %uniq;
$urpm->compute_flags(
urpm::sys::get_packages_list($urpm->{instlist}),
disable_obsolete => 1,
callback => sub {
my ($urpm, $pkg) = @_;
$pkg->is_arch_compat && ! exists $uniq{$pkg->fullname} or return;
$uniq{$pkg->fullname} = undef;
$urpm->{log}(N("would install instead of upgrade package %s", scalar($pkg->fullname)));
},
);
}
sub maybe_find_zeroconf {
my ($urpm, $url, $options) = @_;
if (delete $options->{zeroconf}) {
$url and die "unexpected url $url together with zeroconf\n";
$url = find_zeroconf_repository($urpm);
if ($url) {
$url = urpm::mirrors::_add__with_dir($url, delete $options->{"with-dir"});
delete $options->{mirrorlist};
}
}
return $url;
}
sub find_zeroconf_repository {
my ($urpm) = @_;
my $zeroconf_timeout = 10;
my $res;
eval {
local $SIG{ALRM} = sub { die "timeout" };
alarm($zeroconf_timeout);
$urpm->{debug} and $urpm->{debug}("trying to find a zeroconf repository");
require Net::Bonjour;
$res = Net::Bonjour->new('mdv_urpmi');
alarm(0);
};
if ($@) {
$urpm->{error}("zeroconf error: $@"), return;
}
require urpm::mirrors;
my $product_id = urpm::mirrors::parse_LDAP_namespace_structure(cat_('/etc/product.id'));
my $path_suffix = join('/', lc($product_id->{branch}), $product_id->{version}, $product_id->{arch});
foreach my $entry ($res->entries) {
my $base_url = $entry->attribute('protocol') . '://' . $entry->address . ':' . $entry->port . $entry->attribute('path');
my $url = $base_url . '/' . $path_suffix;
my $distribconf = _new_distribconf_and_download($urpm, { url => $url });
if ($distribconf) {
$urpm->{log}(sprintf("found zeroconf repository: %s", $url));
return $url;
}
}
$urpm->{debug} and $urpm->{debug}("unable to find zeroconf repository");
return;
}
=item add_medium($urpm, $name, $url, $with_synthesis, %options)
Add a new medium and sync the config file accordingly.
It returns the new medium's name (might be different from the requested
name if index_name was specified).
Options: ignore, index_name, nolock, update, virtual, media_info_dir, mirrorlist, zeroconf, with-dir, xml-info, on_the_fly
=cut
sub add_medium {
my ($urpm, $name, $url, $with_synthesis, %options) = @_;
#- make sure configuration has been read.
$urpm->{media} or die "caller should have used ->read_config or ->configure first";
#- if a medium with that name has already been found, we have to exit now
if (defined $options{index_name}) {
my ($i, $basename) = ($options{index_name}, $name);
while (1) {
++$i;
$name = $basename . ($i == 1 ? '' : $i);
last if !name2medium($urpm, $name);
}
} else {
name2medium($urpm, $name) and $urpm->{fatal}(5, N("medium \"%s\" already exists", $name));
}
$url = maybe_find_zeroconf($urpm, $url, \%options);
$url =~ s,/*$,,; #- clear URLs for trailing /es.
#- creating the medium info.
my $medium = { name => $name,
url => $url,
modified => !$options{ignore},
(defined $options{'verify-rpm'} ? ('verify-rpm' => $options{'verify-rpm'}) : ()),
};
foreach (qw(downloader update ignore media_info_dir mirrorlist with-dir xml-info)) {
$medium->{$_} = $options{$_} if exists $options{$_};
}
#- those files must not be there (cf mdvbz#36267)
_clean_statedir_medium_files($urpm, $medium);
if (!($options{virtual} && _local_file($medium))
&& !$urpm->{urpmi_root}) { # with --urpmi-root, we do not use statedir_media_info_file to allow compatibility with older urpmi
mkdir statedir_media_info_dir($urpm, $medium), 0755;
}
if ($options{virtual}) {
$medium->{virtual} = 1;
} else {
_migrate_removable_device($urpm, $medium);
}
if ($with_synthesis) {
_migrate__with_synthesis($medium, $with_synthesis);
} elsif (!$medium->{media_info_dir}) {
if (!is_local_medium($medium)) {
$medium->{media_info_dir} = 'media_info';
} else {
$medium->{unknown_media_info} = 1;
}
}
#- local media have priority, other are added at the end.
my $inserted;
my $ignore_text = $medium->{ignore} ? ' ' . N("(ignored by default)") : '';
if (_local_file($medium)) {
#- insert before first remote medium
@{$urpm->{media}} = map {
if (!_local_file($_) && !$inserted) {
$inserted = 1;
$urpm->{$options{on_the_fly} ? 'log' : 'info'}(N("adding medium \"%s\" before remote medium \"%s\"", $name, $_->{name}) . $ignore_text);
$medium, $_;
} else { $_ }
} @{$urpm->{media}};
}
if (!$inserted) {
$urpm->{$options{on_the_fly} ? 'log' : 'info'}(N("adding medium \"%s\"", $name) . $ignore_text);
push @{$urpm->{media}}, $medium;
}
$urpm->{modified} = 1;
$name;
}
sub _register_media_cfg {
my ($urpm, $url, $mirrorlist, $distribconf, $media_cfg) = @_;
my $arch = $distribconf->getvalue('media_info', 'arch') || '';
my $branch = $distribconf->getvalue('media_info', 'branch') || '';
my $product = $distribconf->getvalue('media_info', 'product') || '';
my $version = $distribconf->getvalue('media_info', 'version') || '';
#official mirrors define $branch but not $product, other RPM repos do the
#opposite :-/
my $media_dir = (($branch || $product) . '-' . $version . '-' . $arch);
$media_dir =~ tr!/!-!;
my $media_path = $urpm->{mediacfgdir} . '/' . $media_dir;
require File::Path;
File::Path::mkpath($media_path);
copy_and_own($media_cfg, $media_path . '/media.cfg')
or $urpm->{info}(1, N("failed to copy media.cfg to %s (%d)", $media_path, $? >> 8));
if ($url) {
my $filename = $media_path . "/url";
my @urls = split(/\n/, scalar cat_($filename));
if (!member($url, @urls)) {
append_to_file($filename, $url . "\n");
}
}
if ($mirrorlist) {
if ($mirrorlist ne '$MIRRORLIST') {
require urpm::cfg;
$mirrorlist = urpm::cfg::expand_line($mirrorlist);
}
my $filename = $media_path . "/mirrorlist";
my @mirrorlists = split(/\n/, scalar cat_($filename));
if (!member($mirrorlist, @mirrorlists)) {
append_to_file($filename, $mirrorlist . "\n");
}
}
}
=item add_distrib_media($urpm, $name, $url, %options)
Add distribution media, according to url given.
Returns the list of names of added media.
Options :
=over
=item *
initial_number : when adding several numbered media, start with this number
=item *
probe_with : force use of rpms instead of using synthesis
=item *
ask_media : callback to know whether each media should be added
=item *
only_updates : only add "update" media (used by rpmdrake)
=item *
mirrorlist
=item *
zeroconf
=back
Other options are passed to add_medium(): ignore, nolock, virtual
=cut
sub add_distrib_media {
my ($urpm, $name, $url, %options) = @_;
#- make sure configuration has been read.
$urpm->{media} or die "caller should have used ->read_config or ->configure first";
my $distribconf;
if ($url && urpm::is_local_url($url)) {
$url = _migrate_removable_url($url) or return();
my $m = { url => $url };
urpm::removable::try_mounting_medium_($urpm, $m) or $urpm->{error}(N("directory %s does not exist", $url));
$distribconf = MDV::Distribconf->new(file_from_file_url($url) || $url, undef);
$distribconf->settree('mageia');
my $dir = file_from_local_medium($m);
my $media_cfg = reduce_pathname("$dir/" . $distribconf->getpath(undef, 'infodir') . '/media.cfg');
$distribconf->parse_mediacfg($media_cfg)
or $urpm->{error}(N("this location doesn't seem to contain any distribution")), return ();
if (!$options{virtual}) {
_register_media_cfg($urpm, $dir, undef, $distribconf, $media_cfg);
}
} else {
$url = maybe_find_zeroconf($urpm, $url, \%options);
if ($options{mirrorlist}) {
$url and die "unexpected url $url together with mirrorlist $options{mirrorlist}\n";
}
my $m = { mirrorlist => $options{mirrorlist}, url => $url };
my $parse_ok;
try__maybe_mirrorlist($urpm, $m, 'probe', sub {
my $media_cfg = "$urpm->{cachedir}/partial/media.cfg";
$distribconf = _new_distribconf_and_download($urpm, $m);
$parse_ok = $distribconf && $distribconf->parse_mediacfg($media_cfg);
if ($parse_ok && !$options{virtual}) {
_register_media_cfg($urpm, urpm::cfg::expand_line($m->{url}), $options{mirrorlist}, $distribconf, $media_cfg);
}
$parse_ok;
});
$url = $m->{url};
if ($distribconf) {
$parse_ok or $urpm->{error}(N("unable to parse media.cfg")), return();
} else {
$urpm->{error}(N("...retrieving failed: %s", $@));
$urpm->{error}(N("unable to access the distribution medium (no media.cfg file found)"));
return ();
}
}
#- cosmetic update of name if it contains spaces.
$name =~ /\s/ and $name .= ' ';
my @newnames;
#- at this point, we have found a media.cfg file, so parse it
#- and create all necessary media according to it.
my $medium_index = $options{initial_number} || 1;
require urpm::mirrors;
my $product_id = urpm::mirrors::parse_LDAP_namespace_structure(cat_('/etc/product.id'));
my ($nonfree, $tainted) = needed_extra_media($urpm);
foreach my $media ($distribconf->listmedia) {
my $media_name = $distribconf->getvalue($media, 'name') || '';
if (my $media_arch = $distribconf->getvalue($media, 'arch')) {
if (!URPM::archscore($media_arch)) {
$urpm->{log}(N("skipping non compatible media `%s' (for %s)",
$media, $media_arch));
next;
}
}
my $is_update_media = $distribconf->getvalue($media, 'updates_for');
if ($options{only_updates}) {
$is_update_media or next;
}
my $add_by_default = is_media_to_add_by_default($urpm, $distribconf, $media, $product_id, $nonfree, $tainted);
my $ignore;
if ($options{ask_media}) {
$options{ask_media}->($media_name, $add_by_default) or next;
} else {
my $simple_rpms = !$distribconf->getvalue($media, 'rpms');
$add_by_default || $simple_rpms or next;
$ignore = !$add_by_default;
}
my $use_copied_synthesis = urpm::is_cdrom_url($url) || $urpm->{options}{use_copied_hdlist} || $distribconf->getvalue($media, 'use_copied_hdlist');
my $with_synthesis = $use_copied_synthesis && offset_pathname(
$url,
$distribconf->getpath($media, 'path'),
) . '/' . $distribconf->getpath($media, 'synthesis');
push @newnames, add_medium($urpm,
$name ? "$media_name ($name$medium_index)" : $media_name,
reduce_pathname($distribconf->getfullpath($media, 'path')),
$with_synthesis,
!$use_copied_synthesis ? (media_info_dir => 'media_info') : @{[]},
!$use_copied_synthesis && $options{probe_with} ? ($options{probe_with} => 1) : (),
index_name => $name ? undef : 0,
$ignore ? (ignore => 1) : @{[]},
%options,
# the following override %options
$options{mirrorlist} ? ('with-dir' => $distribconf->getpath($media, 'path')) : (),
update => $is_update_media ? 1 : undef,
);
++$medium_index;
}
# associate newly added medias with their description in a media.cfg file
# @media content will be modified and then add_existing medium will take
# care of copying the media to $urpm
_associate_media_with_mediacfg($urpm, [ map { name2medium($urpm, $_) } @newnames ]);
return @newnames;
}
sub _new_distribconf_and_download {
my ($urpm, $medium) = @_;
my $distribconf = MDV::Distribconf->new($medium->{url}, undef);
$distribconf->settree('mageia');
$urpm->{log}(N("retrieving media.cfg file..."));
my $url = $medium->{url};
$medium->{url} = urpm::cfg::expand_line($url);
urpm::download::sync_rel_one($urpm, $medium, $distribconf->getpath(undef, 'infodir') . '/media.cfg',
quiet => 1, preclean => 1) or return;
$medium->{url} = urpm::cfg::substitute_back($medium->{url}, $url);
$distribconf;
}
#- deprecated, use select_media_by_name instead
sub select_media {
my $urpm = shift;
my $options = {};
if (ref $_[0]) { $options = shift }
foreach (select_media_by_name($urpm, [ @_ ], $options->{strict_match})) {
#- select medium by setting the modified flag, do not check ignore.
$_->{modified} = 1;
}
}
sub select_media_by_name {
my ($urpm, $names, $b_strict_match) = @_;
my %wanted = map { $_ => 1 } @$names;
#- first the exact matches
my @l = grep { delete $wanted{$_->{name}} } @{$urpm->{media}};
#- check if some arguments don't correspond to the medium name.
#- in such case, try to find the unique medium (or list candidate
#- media found).
foreach (keys %wanted) {
my $q = quotemeta;
my (@found, @foundi);
my $regex = $b_strict_match ? qr/^$q$/ : qr/$q/;
my $regexi = $b_strict_match ? qr/^$q$/i : qr/$q/i;
foreach my $medium (@{$urpm->{media}}) {
$medium->{name} =~ $regex and push @found, $medium;
$medium->{name} =~ $regexi and push @foundi, $medium;
}
@found = @foundi if !@found;
if (@found == 0) {
$urpm->{error}(N("trying to select nonexistent medium \"%s\"", $_));
} else {
if (@found > 1) {
$urpm->{log}(N("selecting multiple media: %s", join(", ", map { qq("$_->{name}") } @found)));
}
#- changed behaviour to select all occurences by default.
push @l, @found;
}
}
@l;
}
#- deprecated, use remove_media instead
sub remove_selected_media {
my ($urpm) = @_;
remove_media($urpm, [ grep { $_->{modified} } @{$urpm->{media}} ]);
}
sub _remove_medium_from_mediacfg {
my ($urpm, $mediacfg_dir, $url, $is_mirrorlist) = @_;
my $filename = $mediacfg_dir;
$filename .= $is_mirrorlist ? "/mirrorlist" : "/url";
my @urls = split(/\n/, scalar cat_($filename));
$urpm->{debug} and $urpm->{debug}("removing $url from $filename");
output_safe($filename, join('\n', grep { $url ne $_ } @urls));
}
sub _cleanup_mediacfg_dir {
my ($urpm, $to_remove) = @_;
foreach my $medium (@$to_remove) {
$medium->{mediacfg} or next;
#this should never happen but dirname(undef) returns . on which we call
#clean_dir so better be safe than sorry
$medium->{mediacfg}[0]{root} or next;
my $dir = reduce_pathname(dirname($medium->{mediacfg}[0]{root}));
begins_with($medium->{mediacfg}[0]{root}, $dir) or next;
if (!any { $_->{mediacfg}[0]{root} == $medium->{mediacfg}[0]{root} } @{$urpm->{media}}) {
$urpm->{debug} and $urpm->{debug}("removing no longer used $dir");
-d $dir and urpm::sys::clean_dir($dir);
next;
}
if ($medium->{mirrorlist}) {
if (!any { $_->{mirrorlist} eq $medium->{mirrorlist} } @{$urpm->{media}}) {
_remove_medium_from_mediacfg($urpm, $dir, $medium->{mirrorlist}, 1);
}
} elsif ($medium->{url}) {
if (!any { $_->{url} eq $medium->{url} } @{$urpm->{media}}) {
_remove_medium_from_mediacfg($urpm, $dir, $medium->{url}, 0);
}
}
}
}
sub remove_media {
my ($urpm, $to_remove) = @_;
foreach my $medium (@$to_remove) {
$urpm->{info}(N("removing medium \"%s\"", $medium->{name}));
#- mark to re-write configuration.
$urpm->{modified} = 1;
_clean_statedir_medium_files($urpm, $medium);
#- remove proxy settings for this media
urpm::download::remove_proxy_media($medium->{name});
}
$urpm->{media} = [ difference2($urpm->{media}, $to_remove) ];
_cleanup_mediacfg_dir($urpm, $to_remove);
}
sub _clean_statedir_medium_files {
my ($urpm, $medium) = @_;
#- remove files associated with this medium.
unlink grep { $_ } map { old_statedir_media_info_file($urpm, $medium, $_->[0], $_->[1]) } @media_info_prefix_suffix;
my $dir = statedir_media_info_dir($urpm, $medium);
-d $dir and urpm::sys::clean_dir($dir);
remove_user_media_info_files($urpm, $medium);
}
sub _probe_with_try_list {
my ($urpm, $medium, $f) = @_;
$medium->{mirrorlist} and die "_probe_with_try_list does not handle mirrorlist\n";
my @media_info_dirs = ('media_info', '.');
my $base = file_from_local_medium($medium) || $medium->{url};
foreach my $media_info_dir (@media_info_dirs) {
my $file = "$media_info_dir/synthesis.hdlist.cz";
my $url = reduce_pathname("$base/$file");
if ($f->($url, $file)) {
$urpm->{debug} and $urpm->{debug}("found synthesis: $url");
$medium->{media_info_dir} = $media_info_dir;
delete $medium->{unknown_media_info};
return 1;
}
}
undef;
}
sub may_reconfig_urpmi {
my ($urpm, $medium) = @_;
$medium->{url} && !urpm::is_cdrom_url($medium->{url}) or return; # we should handle mirrorlist?
my $f;
if (my $dir = file_from_file_url($medium->{url})) {
$f = reduce_pathname("$dir/reconfig.urpmi");
} else {
$f = urpm::download::sync_rel_one($urpm, $medium, 'reconfig.urpmi',
quiet => 1, preclean => 1) or return;
}
my $reconfigured = -s $f && reconfig_urpmi($urpm, $f, $medium);
unlink $f if !is_local_medium($medium);
$reconfigured;
}
=item reconfig_urpmi($urpm, $rfile, $medium)
Read a reconfiguration file for urpmi, and reconfigure media accordingly.
$rfile is the reconfiguration file (local), $name is the media name
the format is similar to the RewriteRule of mod_rewrite, so:
PATTERN REPLACEMENT [FLAG]
where FLAG can be L or N
example of reconfig.urpmi:
# this is an urpmi reconfiguration file
/cauldron /cauldron/$ARCH
=cut
sub reconfig_urpmi {
my ($urpm, $rfile, $medium) = @_;
-r $rfile or return;
my ($magic, @lines) = cat_($rfile);
#- the first line of reconfig.urpmi must be magic, to be sure it's not an error file
$magic =~ /^# this is an urpmi reconfiguration file/ or return undef;
$urpm->{info}(N("reconfiguring urpmi for media \"%s\"", $medium->{name}));
my @replacements;
foreach (@lines) {
chomp;
s/^\s*//; s/#.*$//; s/\s*$//;
$_ or next;
my ($p, $r, $f) = split /\s+/, $_, 3;
push @replacements, [ quotemeta $p, $r, $f || 1 ];
}
my $reconfigured = 0;
my @reconfigurable = qw(url with_synthesis media_info_dir);
my %orig = %$medium;
URLS:
foreach my $k (@reconfigurable) {
foreach my $r (@replacements) {
if ($medium->{$k} =~ s/$r->[0]/$r->[1]/) {
$reconfigured = 1;
#- Flags stolen from mod_rewrite: L(ast), N(ext)
if ($r->[2] =~ /L/) {
last;
} elsif ($r->[2] =~ /N/) { #- dangerous option
redo URLS;
}
}
}
#- check that the new url exists before committing changes (local mirrors)
my $file = urpm::file_from_local_url($medium->{$k});
if ($file && !-e $file) {
%$medium = %orig;
$reconfigured = 0;
$urpm->{log}(N("...reconfiguration failed"));
return;
}
}
if ($reconfigured) {
$urpm->{log}(N("reconfiguration done"));
$urpm->{modified} = 1;
}
$reconfigured;
}
#- names.<media_name> is used by external progs (namely for bash-completion)
sub _generate_medium_names {
my ($urpm, $medium) = @_;
-e statedir_names($urpm, $medium) and return;
my $fh = urpm::sys::open_safe($urpm, ">", statedir_names($urpm, $medium)) or return;
foreach ($medium->{start} .. $medium->{end}) {
my $pkg = $urpm->{depslist}[$_] or
$urpm->{error}(N("Error generating names file: dependency %d not found", $_)), return;
print $fh $pkg->name . "\n";
}
}
sub _guess_synthesis_suffix {
my ($url) = @_;
$url =~ m!\bmedia/(\w+)/*\Z! && $1;
}
sub _synthesis_suffix {
my ($medium) = @_;
$medium->{with_synthesis} =~ /synthesis\.hdlist(.*?)(?:\.src)?\.cz$/ ? $1 : '';
}
sub _medium_is_up_to_date {
my ($urpm, $medium) = @_;
unlink cachedir_with_synthesis($urpm, $medium);
$urpm->{info}(N("medium \"%s\" is up-to-date", $medium->{name}));
#- the medium is now considered not modified.
$medium->{modified} = 0;
}
sub _parse_synthesis {
my ($urpm, $medium, $synthesis_file, $o_callback) = @_;
-e $synthesis_file or return;
$urpm->{log}(N("examining synthesis file [%s]", $synthesis_file));
($medium->{start}, $medium->{end}) =
$urpm->parse_synthesis($synthesis_file, $o_callback ? (callback => $o_callback) : @{[]});
}
sub _parse_synthesis_or_ignore {
my ($urpm, $medium, $o_callback) = @_;
_parse_synthesis($urpm, $medium, any_synthesis($urpm, $medium), $o_callback) or
_ignore_medium_on_parse_error($urpm, $medium);
}
sub is_valid_medium {
my ($medium) = @_;
defined $medium->{start} && defined $medium->{end};
}
sub _ignore_medium_on_parse_error {
my ($urpm, $medium) = @_;
$urpm->{error}(N("problem reading synthesis file of medium \"%s\"", $medium->{name}));
$medium->{ignore} = 1;
}
sub _copy_media_info_file {
my ($urpm, $medium, $prefix, $suffix) = @_;
my $name = "$prefix$suffix";
my $path = _synthesis_dir($medium) . "/$prefix" . _synthesis_suffix($medium) . $suffix;
-e $path or $path = file_from_local_medium($medium) . "/media_info/$name";
my $result_file = "$urpm->{cachedir}/partial/$name";
if (-e $path) {
$urpm->{log}(N("copying [%s] for medium \"%s\"...", $path, $medium->{name}));
copy_and_own($path, $result_file)
or $urpm->{error}(N("...copying failed")), return;
}
-s $result_file && $result_file;
}
sub _get_pubkey__local {
my ($urpm, $medium) = @_;
_copy_media_info_file($urpm, $medium, 'pubkey', '');
}
sub _download_pubkey {
my ($urpm, $medium) = @_;
_download_media_info_file($urpm, $medium, 'pubkey', '', { quiet => 1 });
}
# known options: quiet, callback
sub _download_media_info_file {
my ($urpm, $medium, $prefix, $suffix, $options) = @_;
my $versioned_prefix = do {
my $version = urpm::md5sum::versioned_media_info_file($urpm, $medium, "$prefix$suffix");
$version and $options->{is_versioned} = 1;
$version ? "$version-$prefix" : $prefix;
};
my $tmp = _download_media_info_file_raw($urpm, $medium,
$versioned_prefix, $suffix, $options) or return;
my $result = dirname($tmp) . "/$prefix$suffix";
$tmp eq $result or rename($tmp, $result) or return;
$result;
}
sub _download_media_info_file_raw {
my ($urpm, $medium, $prefix, $suffix, $options) = @_;
my $name = "$prefix$suffix";
my $result_file = "$urpm->{cachedir}/partial/$name";
my $found;
if (_synthesis_suffix($medium)) {
my $local_name = $prefix . _synthesis_suffix($medium) . $suffix;
if (urpm::download::sync_rel_to($urpm, $medium,
_synthesis_dir_rel($medium) . "/$local_name", $result_file,
%$options)) {
$found = 1;
}
}
if (!$found) {
urpm::download::sync_rel_one($urpm, $medium, _synthesis_dir_rel($medium) . "/$name",
%$options);
}
-s $result_file && $result_file;
}
sub get_descriptions_local {
my ($urpm, $medium) = @_;
unlink statedir_descriptions($urpm, $medium);
my $dir = file_from_local_medium($medium);
my $description_file = "$dir/media_info/descriptions"; #- new default location
-e $description_file or $description_file = "$dir/../descriptions";
-e $description_file or return;
$urpm->{log}(N("copying description file of \"%s\"...", $medium->{name}));
if (copy_and_own($description_file, statedir_descriptions($urpm, $medium))) {
$urpm->{log}(N("...copying done"));
} else {
$urpm->{error}(N("...copying failed"));
$medium->{ignore} = 1;
}
}
#- not handling different mirrors since the file is not always available
sub get_descriptions_remote {
my ($urpm, $medium) = @_;
if (-e statedir_descriptions($urpm, $medium)) {
unlink "$urpm->{cachedir}/partial/descriptions";
urpm::sys::move_or_die($urpm, statedir_descriptions($urpm, $medium), "$urpm->{cachedir}/partial/descriptions");
}
my $result = urpm::download::sync_rel_one($urpm, $medium, 'media_info/descriptions', quiet => 1, preclean => 1);
if ($result) {
urpm::sys::move_or_die($urpm, $result, statedir_descriptions($urpm, $medium));
}
}
sub get_synthesis__local {
my ($urpm, $medium, $callback) = @_;
my $f = cachedir_with_synthesis($urpm, $medium);
unlink $f;
$urpm->{log}(N("copying [%s] for medium \"%s\"...", _url_with_synthesis($medium), $medium->{name}));
$callback and $callback->('copy', $medium->{name});
if (copy_and_own(_url_with_synthesis($medium), $f)) {
$callback and $callback->('done', $medium->{name});
$urpm->{log}(N("...copying done"));
if (file_size($f) < 20) {
$urpm->{error}(N("copy of [%s] failed (file is suspiciously small)", $f));
0;
} else {
1;
}
} else {
$callback and $callback->('failed', $medium->{name});
#- force error, reported afterwards
unlink $f;
0;
}
}
sub get_synthesis__remote {
my ($urpm, $medium, $is_a_probe, $options) = @_;
my $ok = try__maybe_mirrorlist($urpm, $medium, $is_a_probe, sub {
_download_media_info_file($urpm, $medium, 'synthesis.hdlist', '.cz',
$options)
&& _check_synthesis(cachedir_with_synthesis($urpm, $medium));
});
if (!$ok) {
chomp(my $err = $@);
$urpm->{error}(N("...retrieving failed: %s", $err));
}
$ok &&= check_synthesis_md5sum($urpm, $medium) if !$options->{force} && !$options->{nomd5sum};
$ok;
}
sub _check_synthesis {
my ($synthesis_file) = @_;
file_size($synthesis_file) >= 20 or return;
# check first 2 lines do not contain typical html code
# this is useful for servers not returning a valid HTTP error (#39918)
open(my $F, '<', $synthesis_file) or return;
my $s = <$F>; $s .= <$F>;
$s !~ /<html>|<!DOCTYPE\s/i;
}
#- check copied/downloaded file has right signature.
sub check_synthesis_md5sum {
my ($urpm, $medium) = @_;
my $wanted_md5sum = urpm::md5sum::from_MD5SUM__or_warn($urpm, $medium->{parsed_md5sum}, 'synthesis.hdlist.cz');
if ($wanted_md5sum) {
$urpm->{log}(N("computing md5sum of retrieved source synthesis"));
urpm::md5sum::compute(cachedir_with_synthesis($urpm, $medium)) eq $wanted_md5sum or
$urpm->{error}(N("retrieval of [%s] failed (md5sum mismatch)", _url_with_synthesis($medium))), return;
}
1;
}
sub _call_genhdlist2 {
my ($urpm, $medium) = @_;
!$medium->{with_synthesis} or $urpm->{fatal}(1, 'with_synthesis not handled with --probe-rpms');
my $dir = file_from_local_medium($medium);
system('genhdlist2',
$urpm->{debug} ? '--verbose' : @{[]}, '--no-hdlist',
'--media_info-dir', "$urpm->{cachedir}/partial", $dir) == 0
or $urpm->{error}(N("genhdlist2 failed on %s", $dir)), return;
1;
}
sub _is_statedir_MD5SUM_uptodate {
my ($urpm, $medium, $new_MD5SUM) = @_;
my $current_MD5SUM = statedir_MD5SUM($urpm, $medium);
$urpm->{log}(N("comparing %s and %s", $new_MD5SUM, $current_MD5SUM));
cat_($new_MD5SUM) eq cat_($current_MD5SUM);
}
#- options: callback, force, nomd5sum, probe_with
sub _update_medium__parse_if_unmodified__local {
my ($urpm, $medium, $options) = @_;
if ($options->{probe_with} ne 'rpms') {
#- the directory given does not exist and may be accessible
#- by mounting some other directory. Try to figure it out and mount
#- everything that might be necessary.
urpm::removable::try_mounting_medium($urpm, $medium) or return;
}
#- check for a reconfig.urpmi file (if not already reconfigured)
if (!$medium->{noreconfigure}) {
may_reconfig_urpmi($urpm, $medium);
}
#- try to probe for possible with_synthesis parameter, unless
#- it is already defined (and valid).
if (!_valid_synthesis_dir($medium) && $options->{probe_with} ne 'rpms') {
_probe_with_try_list($urpm, $medium, sub {
my ($url) = @_;
-e $url or return;
if (file_size($url) >= 20) {
1;
} else {
$urpm->{error}(N("invalid hdlist file %s for medium \"%s\"", $url, $medium->{name}));
0;
}
});
}
if (_is_local_virtual($medium)) {
#- syncing a local virtual medium is very simple :)
1;
} elsif ($options->{probe_with} eq 'rpms' || !_valid_synthesis_dir($medium)) {
_call_genhdlist2($urpm, $medium) or return '';
if (!$medium->{'no-media-info'}) {
$medium->{'no-media-info'} = 1;
$urpm->{modified} = 1;
}
1;
} elsif (_valid_synthesis_dir($medium)) {
my $new_MD5SUM = _synthesis_dir($medium) . '/MD5SUM';
unlink "$urpm->{cachedir}/partial/MD5SUM";
if (!$options->{nomd5sum} && file_size($new_MD5SUM) > 32) {
if (!$options->{force} && _is_statedir_MD5SUM_uptodate($urpm, $medium, $new_MD5SUM)) {
_medium_is_up_to_date($urpm, $medium);
return 'unmodified';
}
$urpm->{log}(N("copying MD5SUM file of \"%s\"...", $medium->{name}));
copy_and_own($new_MD5SUM, "$urpm->{cachedir}/partial/MD5SUM");
$medium->{parsed_md5sum} = urpm::md5sum::parse($new_MD5SUM);
}
my $ok = get_synthesis__local($urpm, $medium, $options->{callback});
$ok &&= check_synthesis_md5sum($urpm, $medium) if !$options->{force} && !$options->{nomd5sum};
if ($ok) {
1;
} elsif ($urpm->{options}{'build-hdlist-on-error'}) {
#- if copying synthesis has failed, try to build it directly.
_call_genhdlist2($urpm, $medium) or return '';
1;
} else {
_ignore_medium_on_parse_error($urpm, $medium);
'';
}
}
}
sub _download_MD5SUM {
my ($urpm, $medium) = @_;
urpm::download::sync_rel_one($urpm, $medium,
_synthesis_dir_rel($medium) . '/MD5SUM',
quiet => 1, preclean => 1);
}
sub _download_MD5SUM_and_check {
my ($urpm, $medium, $is_a_probe) = @_;
my ($err, $cachedir_MD5SUM);
require urpm::mirrors;
try__maybe_mirrorlist($urpm, $medium, $is_a_probe, sub {
$cachedir_MD5SUM = _download_MD5SUM($urpm, $medium) or $err = $@;
$cachedir_MD5SUM && urpm::md5sum::check_file($cachedir_MD5SUM);
}) and return $cachedir_MD5SUM;
if ($cachedir_MD5SUM) {
$urpm->{error}(N("invalid MD5SUM file (downloaded from %s)", _synthesis_dir($medium)));
} else {
$urpm->{error}(N("...retrieving failed: %s", $err));
$is_a_probe and $urpm->{error}(N("no metadata found for medium \"%s\"", $medium->{name}));
}
undef;
}
#- options: callback, ask_retry, force, nomd5sum, probe_with, quiet
sub _update_medium__parse_if_unmodified__remote {
my ($urpm, $medium, $options) = @_;
my $updating = -e statedir_synthesis($urpm, $medium);
#- examine if a distant MD5SUM file is available.
if (!$options->{nomd5sum}) {
my $new_MD5SUM = _download_MD5SUM_and_check($urpm, $medium, !$updating);
if (!$new_MD5SUM) {
#- check for a reconfig.urpmi file (if not already reconfigured)
if (!$medium->{noreconfigure}) {
may_reconfig_urpmi($urpm, $medium)
and goto &_update_medium__parse_if_unmodified__remote;
}
return;
}
if (($options->{force} || 0) < 2 && _is_statedir_MD5SUM_uptodate($urpm, $medium, $new_MD5SUM)) {
_medium_is_up_to_date($urpm, $medium);
return 'unmodified';
}
$medium->{parsed_md5sum} = urpm::md5sum::parse($new_MD5SUM);
}
#- try to probe for possible with_synthesis parameter, unless
#- it is already defined (and valid).
$urpm->{log}(N("retrieving source synthesis of \"%s\"...", $medium->{name}));
$options->{callback} and $options->{callback}('retrieve', $medium->{name});
my $error = sub {
my ($msg) = @_;
$urpm->{error}($msg);
unlink cachedir_with_synthesis($urpm, $medium);
$options->{callback} and $options->{callback}('failed', $medium->{name});
};
if ($options->{force}) {
unlink cachedir_with_synthesis($urpm, $medium);
} else {
#- for rsync, try to sync (copy if needed) local copy after restored the previous one.
my $previous_synthesis = statedir_synthesis($urpm, $medium);
if (-e $previous_synthesis && urpm::protocol_from_url($medium->{url}) eq 'rsync') {
copy_and_own(
$previous_synthesis,
cachedir_with_synthesis($urpm, $medium),
) or $error->(N("...copying failed")), return;
}
}
my $ok = get_synthesis__remote($urpm, $medium, !$updating, $options);
$options->{callback} and $options->{callback}('done', $medium->{name});
if (!$ok) {
_ignore_medium_on_parse_error($urpm, $medium);
return;
}
1;
}
sub _get_pubkey {
my ($urpm, $medium, $b_wait_lock) = @_;
my $local = file_from_local_medium($medium);
#- examine if a pubkey file is available.
($local ? \&_get_pubkey__local : \&_download_pubkey)->($urpm, $medium);
$medium->{'key-ids'} = _read_cachedir_pubkey($urpm, $medium, $b_wait_lock);
$urpm->{modified} = 1;
}
sub _get_descriptions {
my ($urpm, $medium) = @_;
my $local = file_from_local_medium($medium);
# do not get "descriptions" on non "update" media since it's useless and potentially slow
if ($medium->{update}) {
($local ? \&get_descriptions_local : \&get_descriptions_remote)->($urpm, $medium);
}
}
# options: wait_lock, nopubkey, forcekey
sub _may_get_pubkey {
my ($urpm, $medium, %options) = @_;
_get_pubkey($urpm, $medium, $options{wait_lock}) if !$options{nopubkey} && (!$medium->{'key-ids'} || $options{forcekey});
}
sub _read_cachedir_pubkey {
my ($urpm, $medium, $b_wait_lock) = @_;
-s "$urpm->{cachedir}/partial/pubkey" or return;
$urpm->{log}(N("examining pubkey file of \"%s\"...", $medium->{name}));
my $_rpm_lock = urpm::lock::rpm_db($urpm, 'exclusive', wait => $b_wait_lock);
my $db = urpm::db_open_or_die_($urpm, 'rw');
my %key_ids;
URPM::import_needed_pubkeys_from_file($db,
"$urpm->{cachedir}/partial/pubkey",
sub {
my ($id, $imported) = @_;
if ($id) {
$key_ids{$id} = undef;
$imported and $urpm->{log}(N("...imported key %s from pubkey file of \"%s\"",
$id, $medium->{name}));
$imported or $urpm->{debug}("pubkey $id already imported") if $urpm->{debug};
} else {
$urpm->{error}(N("unable to import pubkey file of \"%s\"", $medium->{name}));
}
});
unlink "$urpm->{cachedir}/partial/pubkey";
join(',', keys %key_ids);
}
#- options: callback, ask_retry, force, nomd5sum, probe_with, quiet, forcekey, nopubkey, wait_lock
#- (from _update_medium__parse_if_unmodified__local and _update_medium__parse_if_unmodified__remote)
sub _update_medium_ {
my ($urpm, $medium, %options) = @_;
#- always delete a remaining list file or pubkey file in cache.
foreach (qw(list pubkey)) {
unlink "$urpm->{cachedir}/partial/$_";
}
_pick_mirror_if_needed($urpm, $medium, 'allow-cache-update');
{
my $rc =
is_local_medium($medium)
? _update_medium__parse_if_unmodified__local($urpm, $medium, \%options)
: _update_medium__parse_if_unmodified__remote($urpm, $medium, \%options);
if ($options{forcekey} && $rc eq 'unmodified') {
_get_pubkey($urpm, $medium, $options{wait_lock}); # we must do it now, quite hackish...
return 1;
}
if (!$rc || $rc eq 'unmodified') {
return $rc;
}
}
my $is_updating = -e statedir_synthesis($urpm, $medium);
if (!_is_local_virtual($medium)) {
if (file_size(cachedir_with_synthesis($urpm, $medium)) < 20) {
$urpm->{error}(N("no synthesis file found for medium \"%s\"", $medium->{name}));
return;
}
#- use new files
unlink statedir_synthesis($urpm, $medium);
urpm::sys::move_or_die($urpm, cachedir_with_synthesis($urpm, $medium),
statedir_synthesis($urpm, $medium));
unlink statedir_MD5SUM($urpm, $medium);
if (!$medium->{with_synthesis}) { # no MD5SUM when using with_synthesis, urpmi.update will update everytime!
urpm::sys::move_or_die($urpm, "$urpm->{cachedir}/partial/MD5SUM",
statedir_MD5SUM($urpm, $medium)) if -e "$urpm->{cachedir}/partial/MD5SUM";
}
# we never download hdlist by default. urpmf will download it via any_hdlist() if really needed
unlink statedir_hdlist($urpm, $medium);
remove_user_media_info_files($urpm, $medium);
if (!_local_file($medium)) {
_retrieve_xml_media_info_or_remove($urpm, $medium, $options{quiet}) or return;
}
}
$medium->{modified} = 0;
# generated on first _parse_media()
unlink statedir_names($urpm, $medium);
_get_descriptions($urpm, $medium);
_may_get_pubkey($urpm, $medium, %options);
$is_updating and $urpm->{info}(N("updated medium \"%s\"", $medium->{name}));
1;
}
sub _update_medium {
my ($urpm, $medium, %options) = @_;
my $rc = _update_medium_($urpm, $medium, %options);
if (!$rc && !_is_local_virtual($medium)) {
#- an error has occured for updating the medium, we have to remove temporary files.
unlink(glob("$urpm->{cachedir}/partial/*"));
}
$rc;
}
=item update_media($urpm, %options)
Update the urpmi database w.r.t. the current configuration.
Takes care of modifications, and tries some tricks to bypass
the recomputation of base files.
Recognized options :
=over
=item *
all : all medias are being rebuilt
=item *
allow_failures: whereas failing to update a medium is non fatal
=item *
ask_retry : function called when a download fails. if it returns true, the download is retried
=item *
callback : UI callback
=item *
forcekey : force retrieval of pubkey
=item *
force : try to force rebuilding base files
=item *
nomd5sum : don't verify MD5SUM of retrieved files
=item *
nopubkey : don't use rpm pubkeys
=item *
probe_with : probe synthesis or rpms
=item *
quiet : download synthesis quietly
=item *
wait_lock : block until lock can be acquired
=back
=cut
sub update_media {
my ($urpm, %options) = @_;
$urpm->{media} or return; # verify that configuration has been read
if ($options{all}) {
$_->{modified} ||= 1 foreach all_media_to_update($urpm);
}
update_those_media($urpm, [ grep { $_->{modified} } non_ignored_media($urpm) ], %options);
}
sub update_those_media {
my ($urpm, $media, %options) = @_;
$options{nopubkey} ||= $urpm->{options}{nopubkey};
#- examine each medium to see if one of them needs to be updated.
#- if this is the case and if not forced, try to use a pre-calculated
#- synthesis file, else build it from rpm files.
clean($urpm);
my %updates_result;
foreach my $medium (@$media) {
#- don't ever update static media
$medium->{static} and next;
my $unsubstituted_url = $medium->{url};
$medium->{url} = urpm::cfg::expand_line($medium->{url}) if $medium->{url};
my $rc = _update_medium($urpm, $medium, %options);
$medium->{url} = urpm::cfg::substitute_back($medium->{url}, $unsubstituted_url);
$rc or return if !$options{allow_failures};
$updates_result{$rc || 'error'}++;
}
$urpm->{debug} and $urpm->{debug}('update_medium: ' . join(' ', map { "$_=$updates_result{$_}" } keys %updates_result));
if ($updates_result{1} == 0) {
#- only errors/unmodified, leave now
#- (this ensures buggy added medium is not added to urpmi.cfg)
return $updates_result{error} == 0;
}
if ($urpm->{modified}) {
#- write config files in any case
write_config($urpm);
urpm::download::dump_proxy_config();
}
$updates_result{error} == 0;
}
sub _maybe_in_statedir_MD5SUM {
my ($urpm, $medium, $file) = @_;
my $md5sum_file = statedir_MD5SUM($urpm, $medium);
-e $md5sum_file && urpm::md5sum::parse($md5sum_file)->{$file};
}
sub _retrieve_xml_media_info_or_remove {
my ($urpm, $medium, $quiet) = @_;
my $ok = 1;
foreach my $xml_info (@xml_media_info) {
my $f = statedir_xml_info($urpm, $medium, $xml_info);
my $get_it = urpm::is_cdrom_url($medium->{url}) ||
get_medium_option($urpm, $medium, 'xml-info') eq 'always' ||
get_medium_option($urpm, $medium, 'xml-info') eq 'update-only' && -e $f;
if ($get_it && _maybe_in_statedir_MD5SUM($urpm, $medium, "$xml_info.xml.lzma")) {
$ok &&= _retrieve_media_info_file_and_check_MD5SUM($urpm, $medium, $xml_info, '.xml.lzma', $quiet);
$ok = 1 if urpm::is_cdrom_url($medium->{url});
} else {
#- "on-demand"
unlink $f;
}
}
$ok;
}
sub _retrieve_media_info_file_and_check_MD5SUM {
my ($urpm, $medium, $prefix, $suffix, $quiet) = @_;
my $name = "$prefix$suffix";
my $cachedir_file =
is_local_medium($medium) ?
_copy_media_info_file($urpm, $medium, $prefix, $suffix) :
_download_media_info_file($urpm, $medium, $prefix, $suffix, { quiet => $quiet, callback => \&urpm::download::sync_logger }) or
$urpm->{error}(N("retrieval of [%s] failed", _synthesis_dir($medium) . "/$name")), return;
my $wanted_md5sum = urpm::md5sum::from_MD5SUM__or_warn($urpm, $medium->{parsed_md5sum}, $name);
if ($wanted_md5sum) {
$urpm->{debug}("computing md5sum of retrieved $name") if $urpm->{debug};
urpm::md5sum::compute($cachedir_file) eq $wanted_md5sum or
$urpm->{error}(N("retrieval of [%s] failed (md5sum mismatch)", _synthesis_dir($medium) . "/$name")), return;
urpm::util::move($cachedir_file, statedir_media_info_file($urpm, $medium, $prefix, $suffix)) or return;
}
1;
}
sub _download_temp_md5sum_and_parse {
my ($urpm, $medium) = @_;
$urpm->{debug}("downloading MD5SUM to know updated versioned metadata filename") if $urpm->{debug};
my $md5sum_file = _download_MD5SUM($urpm, $medium);
urpm::md5sum::parse($md5sum_file);
}
sub _any_media_info__or_download {
my ($urpm, $medium, $prefix, $suffix, $quiet, $o_callback) = @_;
my $f = statedir_media_info_file($urpm, $medium, $prefix, $suffix);
-s $f and return $f;
if ($<) {
urpm::ensure_valid_cachedir($urpm);
$f = "$urpm->{cachedir}/" . statedir_media_info_basename($medium, $prefix, $suffix);
-s $f and return $f;
}
get_medium_option($urpm, $medium, 'xml-info') ne 'never' or return;
_maybe_in_statedir_MD5SUM($urpm, $medium, "$prefix$suffix") or return;
$medium->{parsed_md5sum} ||= _download_temp_md5sum_and_parse($urpm, $medium);
my $file_in_partial =
_download_media_info_file($urpm, $medium, $prefix, $suffix,
{ quiet => $quiet, callback => $o_callback }) or return;
urpm::util::move($file_in_partial, $f) or return;
$f;
}
#- side-effects:
#- + those of urpm::mirrors::pick_one ($urpm->{mirrors_cache}, $medium->{url})
sub _pick_mirror_if_needed {
my ($urpm, $medium, $allow_cache_update) = @_;
$medium->{mirrorlist} && !$medium->{url} or return;
require urpm::mirrors;
urpm::mirrors::pick_one($urpm, $medium, $allow_cache_update);
}
#- side-effects:
#- + those of urpm::mirrors::try ($urpm->{mirrors_cache}, $medium->{url})
sub try__maybe_mirrorlist {
my ($urpm, $medium, $is_a_probe, $try) = @_;
if ($medium->{mirrorlist}) {
if (urpm::download::use_metalink($urpm, $medium)) {
#- help things...
_pick_mirror_if_needed($urpm, $medium, 'allow-cache-update');
$try->();
} else {
require urpm::mirrors;
$is_a_probe
? urpm::mirrors::try_probe($urpm, $medium, $try)
: urpm::mirrors::try($urpm, $medium, $try);
}
} else {
$try->();
}
}
=item clean($urpm)
Clean params and depslist computation zone.
=cut
sub clean {
my ($urpm) = @_;
$urpm->{depslist} = [];
$urpm->{provides} = {};
foreach (@{$urpm->{media} || []}) {
delete $_->{start};
delete $_->{end};
}
}
1;
__END__
=back
=head1 COPYRIGHT
Copyright (C) 2005 MandrakeSoft SA
Copyright (C) 2005-2010 Mandriva SA
Copyright (C) 2011-2013 Mageia
=cut