use Getopt::Long;
use Test::More;
use strict;
my $tests;
BEGIN { $tests = 0 }
use lib "lib";
my %Opt;
GetOptions(
"verbose!",
) or die;
$Opt{verbose} ||= $ENV{PERL_RERSYNCRECENT_TEST_VERBOSE};
my $HAVE;
BEGIN {
# neither LibMagic nor MMagic tell them apart
for my $package (
# "File::LibMagic",
"File::MMagic",
) {
$HAVE->{$package} = eval qq{ require $package; };
}
}
use Dumpvalue;
use File::Basename qw(dirname);
use File::Copy qw(cp);
use File::Path qw(mkpath rmtree);
use File::Rsync::Mirror::Recent;
use File::Rsync::Mirror::Recentfile;
use List::MoreUtils qw(uniq);
use Storable;
use Time::HiRes qw(time sleep);
use YAML::Syck;
my $root_from = "t/ta";
my $root_to = "t/tb";
my $statusfile = "t/recent-rmirror-state.yml";
rmtree [$root_from, $root_to];
{
my @serializers;
my $test_counter;
BEGIN {
$test_counter = $tests;
@serializers = (
[".yaml","YAML::Syck"],
[".json","JSON"],
[".sto","Storable"],
[".dd","Data::Dumper"],
);
$tests += @serializers;
if ($HAVE->{"File::LibMagic"}||$HAVE->{"File::MMagic"}) {
$tests += @serializers;
}
}
printf "#test_counter[%d]\n", $test_counter;
mkpath $root_from;
my $ttt = "$root_from/ttt";
open my $fh, ">", $ttt or die "Could not open: $!";
print $fh time;
close $fh or die "Could not close: $!";
my $fm;
if ($HAVE->{"File::LibMagic"}) {
$fm = File::LibMagic->new();
} elsif ($HAVE->{"File::MMagic"}) {
$fm = File::MMagic->new();
}
for my $serializer (@serializers) {
my($s,$module) = @$serializer;
unless (eval "require $module; 1") {
ok(1, "Skipping because $module not installed");
if ($fm) {
ok(1, "Skipping the magic test for same reason");
}
next;
}
my $rf = File::Rsync::Mirror::Recentfile->new
(
filenameroot => "RECENT",
interval => q(1m),
localroot => $root_from,
serializer_suffix => $s,
);
$rf->update($ttt,"new");
if ($fm) {
my $magic = $fm->checktype_filename("$root_from/RECENT-1m$s");
ok($magic, sprintf
("Got a magic[%s] for s[%s]: [%s]",
ref $fm,
$s,
$magic,
));
}
my $content = do {open my $fh, "$root_from/RECENT-1m$s";local $/;<$fh>};
$content = Dumpvalue->new()->stringify($content);
my $want_length = 42; # or maybe 3 more
substr($content,$want_length) = "..." if length $content > 3+$want_length;
ok($content, "Got a substr for s[$s]: [$content]");
}
}
rmtree [$root_from, $root_to];
{
# very small tree, aggregate it
my @intervals;
my $test_counter;
BEGIN {
$test_counter = $tests;
@intervals = qw( 2s 4s 8s 16s 32s Z );
$tests += 2 + 2 * (10 + 14 * @intervals); # test_counter
}
printf "#test_counter[%d]\n", $test_counter;
ok(1, "starting smalltree block");
is 6, scalar @intervals, "array has 6 elements: @intervals";
printf "#test_counter[%d]\n", $test_counter+=2;
for my $pass (0,1) {
my $rf0 = File::Rsync::Mirror::Recentfile->new
(
aggregator => [@intervals[1..$#intervals]],
interval => $intervals[0],
localroot => $root_from,
rsync_options => [
compress => 0,
links => 1,
times => 1,
checksum => 0,
],
);
my $timestampfutured = 0;
for my $iv (@intervals) {
for my $i (0..3) {
my $file = sprintf
(
"%s/A%s-%02d",
$root_from,
$iv,
$i,
);
mkpath dirname $file;
open my $fh, ">", $file or die "Could not open '$file': $!";
print $fh time, ":", $file, "\n";
close $fh or die "Could not close '$file': $!";
$rf0->update($file,"new");
if ($pass==1 && !$timestampfutured) {
my $recent_events = $rf0->recent_events;
$recent_events->[0]{epoch} += 987654321;
$rf0->write_recent($recent_events);
$timestampfutured++;
}
}
}
my $recent_events = $rf0->recent_events;
# faking internals as if the contents were wide-spread in time
for my $evi (0..$#$recent_events) {
my $ev = $recent_events->[$evi];
$ev->{epoch} -= 2**($evi*.25);
}
$rf0->write_recent($recent_events);
$rf0->aggregate;
my $filesize_threshold = 1750; # XXX may be system dependent
my %size_before;
for my $iv (@intervals) {
my $rf = "$root_from/RECENT-$iv.yaml";
my $filesize = -s $rf;
$size_before{$iv} = $filesize;
# now they have $filesize_threshold+ bytes because they were merged for the
# first time ever and could not be truncated for this reason.
ok( $filesize > $filesize_threshold, "file $iv (before merging) has good size[$filesize]");
utime 0, 0, $rf; # so that the next aggregate isn't skipped
}
printf "#test_counter[%d]\n", $test_counter+=6;
open my $fh, ">", "$root_from/finissage" or die "Could not open: $!";
print $fh "fin";
close $fh or die "Could not close: $!";
$rf0->update("$root_from/finissage","new");
$rf0 = File::Rsync::Mirror::Recentfile->new_from_file("$root_from/RECENT-2s.yaml");
$rf0->aggregate;
for my $iv (@intervals) {
my $filesize = -s "$root_from/RECENT-$iv.yaml";
# now they have <$filesize_threshold bytes because the second aggregate could
# truncate them
ok($iv eq "Z" || $filesize<$size_before{$iv}, "file $iv (after merging) has good size[$filesize<$size_before{$iv}]");
}
printf "#test_counter[%d]\n", $test_counter+=6;
my $dagg1 = $rf0->_debug_aggregate;
Time::HiRes::sleep 1.2;
$rf0->aggregate;
my $dagg2 = $rf0->_debug_aggregate;
{
my $recc = File::Rsync::Mirror::Recent->new
(
local => "$root_from/RECENT-2s.yaml",
);
ok $recc->overview, "overview created";
# diag $recc->overview;
}
printf "#test_counter[%d]\n", $test_counter+=1;
for my $dirti (0,1,2) {
open my $fh2, ">", "$root_from/dirty$dirti" or die "Could not open: $!";
print $fh2 "dirty$dirti";
close $fh2 or die "Could not close: $!";
my $timestamp = $dirti <= 1 ? "999.999" : "1999.999";
my $becomes_i = $dirti <= 1 ? -1 : -3;
$rf0->update("$root_from/dirty$dirti","new",$timestamp);
$recent_events = $rf0->recent_events;
is $recent_events->[-1]{epoch}, $timestamp, "found the dirty timestamp during dirti[$dirti]";
printf "#test_counter[%d]\n", $test_counter+=1;
$rf0->aggregate(force => 1);
my $recc = File::Rsync::Mirror::Recent->new
(
localroot => $root_from,
local => "$root_from/RECENT.recent",
);
my %seen;
for my $rf (@{$recc->recentfiles}) {
my $isec = $rf->interval_secs;
my $re = $rf->recent_events;
like $re->[-1]{epoch}, qr/999\.999/, "found some dirty timestamp[$re->[-1]{epoch}] in isec[$isec]";
my $dirtymark = $rf->dirtymark;
ok $dirtymark, "dirtymark[$dirtymark]";
$seen{ $rf->dirtymark }++;
}
printf "#test_counter[%d]\n", $test_counter+=12;
is scalar keys %seen, 1, "all recentfiles have the same dirtymark";
printf "#test_counter[%d]\n", $test_counter+=1;
sleep 0.2;
$rf0->aggregate(force => 1);
my $rfs = $recc->recentfiles;
for my $i (0..$#$rfs) {
my $rf = $rfs->[$i];
my $re = $rf->recent_events;
if ($i == 0) {
unlike $re->[-1]{epoch}, qr/999\.999/, "dirty file events already moved up i[$i]";
} elsif ($i == $#$rfs) {
is $re->[$becomes_i]{epoch}, $timestamp, "found the dirty timestamp on i[$i]";
} else {
isnt $re->[-1]{epoch}, $timestamp, "dirty timestamp gone on i[$i]";
}
my $dirtymark = $rf->dirtymark;
ok $dirtymark, "dirtymark[$dirtymark]";
$seen{ $rf->dirtymark }++;
}
printf "#test_counter[%d]\n", $test_counter+=12;
is scalar keys %seen, 1, "all recentfiles have the same dirtymark";
printf "#test_counter[%d]\n", $test_counter+=1;
}
# $DB::single++;
rmtree [$root_from];
}
}
rmtree [$root_from, $root_to];
{
# replay a short history, run aggregate on it, add files, aggregate again
my $test_counter;
BEGIN {
$test_counter = $tests;
$tests += 208;
}
printf "#test_counter[%d]\n", $test_counter;
ok(1, "starting short history block");
my $rf = File::Rsync::Mirror::Recentfile->new_from_file("t/RECENT-6h.yaml");
my $recent_events = $rf->recent_events;
my $recent_events_cnt = scalar @$recent_events;
is (
92,
$recent_events_cnt,
"found $recent_events_cnt events",
);
$rf->interval("5s");
$rf->localroot($root_from);
$rf->comment("produced during the test 02-operation.t");
$rf->aggregator([qw(10s 30s 1m 1h Z)]);
$rf->verbose(0);
my $start = Time::HiRes::time;
for my $e (@$recent_events) {
for my $pass (0,1) {
my $file = sprintf
(
"%s/%s",
$pass==0 ? $root_from : $root_to,
$e->{path},
);
mkpath dirname $file;
open my $fh, ">", $file or die "Could not open '$file': $!";
print $fh time, ":", $file, "\n";
close $fh or die "Could not close '$file': $!";
if ($pass==0) {
$rf->update($file,$e->{type});
}
}
}
$rf->aggregate;
my $took = Time::HiRes::time - $start;
ok $took > 0, "creating the tree and aggregate took $took seconds";
my $dagg1 = $rf->_debug_aggregate;
for my $i (1..5) {
my $file_from = "$root_from/anotherfilefromtesting$i";
open my $fh, ">", $file_from or die "Could not open: $!";
print $fh time, ":", $file_from;
close $fh or die "Could not close: $!";
$rf->update($file_from,"new");
}
$rf->aggregate;
my $dagg2 = $rf->_debug_aggregate;
undef $rf;
ok($dagg1->[0]{size} < $dagg2->[0]{size}, "The second 5s file size larger: $dagg1->[0]{size} < $dagg2->[0]{size}");
ok($dagg1->[1]{mtime} <= $dagg2->[1]{mtime}, "The second 30s file timestamp larger: $dagg1->[1]{mtime} <= $dagg2->[1]{mtime}");
is $dagg1->[2]{size}, $dagg2->[2]{size}, "The 1m file size unchanged";
is $dagg1->[3]{mtime}, $dagg2->[3]{mtime}, "The 1h file timestamp unchanged";
ok -l "t/ta/RECENT.recent", "found the symlink";
my $have_slept = my $have_worked = 0;
$start = Time::HiRes::time;
my $debug = +[];
for my $i (0..99) {
my $file = sprintf
(
"%s/secscnt%03d",
$root_from,
($i<25) ? ($i%12) : $i,
);
open my $fh, ">", $file or die "Could not open '$file': $!";
print $fh time, ":", $file, "\n";
close $fh or die "Could not close '$file': $!";
my $another_rf = File::Rsync::Mirror::Recentfile->new
(
interval => "5s",
localroot => $root_from,
aggregator => [qw(10s 30s 1m Z)],
);
$another_rf->update($file,"new");
my $should_have = 97 + (($i<25) ? ($i < 12 ? ($i+1) : 12) : ($i-12));
my($news,$filtered_news);
if ($i < 50) {
$another_rf->aggregate;
}
{
my $recc = File::Rsync::Mirror::Recent->new
(
local => "$root_from/RECENT-5s.yaml",
);
$news = $recc->news ();
$filtered_news = [ uniq map { $_->{path} } @$news ];
}
is scalar @$filtered_news, $should_have, "should_have[$should_have]" or die;
$debug->[$i] = $news;
my $rf2 = File::Rsync::Mirror::Recentfile->new_from_file("$root_from/RECENT-5s.yaml");
my $rece = $rf2->recent_events;
my $rececnt = @$rece;
my $span = $rece->[0]{epoch} - $rece->[-1]{epoch};
$have_worked = Time::HiRes::time - $start - $have_slept;
ok($rececnt > 0
&& ($i<50 ? $span <= 5 # we have run aggregate, so it guaranteed(*)
: $i < 90 ? 1 # we have not yet spent 5 seconds, so cannot predict
: $span > 5 # we have certainly written enough files now, must happen
),
sprintf
("i[%s]cnt[%s]span[%s]worked[%6.4f]",
$i,
$rececnt,
$span,
$have_worked,
));
$have_slept += Time::HiRes::sleep 0.2;
}
# (*) "<=" instead of "<" because of rounding errors
}
{
# running mirror
my $test_counter;
BEGIN {
$test_counter = $tests;
$tests += 3;
}
printf "#test_counter[%d]\n", $test_counter;
my $rf = File::Rsync::Mirror::Recentfile->new
(
filenameroot => "RECENT",
interval => q(30s),
localroot => $root_to,
max_rsync_errors => 0,
remote_dir => $root_from,
# verbose => 1,
max_files_per_connection => 65,
rsync_options => {
compress => 0,
links => 1,
times => 1,
# not available in rsync 3.0.3: 'omit-dir-times' => 1,
checksum => 0,
},
);
my $somefile_epoch;
for my $pass (0,1) {
my $success;
if (0 == $pass) {
$success = $rf->mirror;
my $re = $rf->recent_events;
$somefile_epoch = $re->[24]{epoch};
} elsif (1 == $pass) {
$success = $rf->mirror(after => $somefile_epoch);
}
ok($success, "mirrored pass[$pass] without dying");
}
{
my $recc = File::Rsync::Mirror::Recent->new
( # ($root_from, $root_to)
local => "$root_from/RECENT-5s.yaml",
);
diag "\n" if $Opt{verbose};
diag $recc->overview if $Opt{verbose};
}
{
my $recc = File::Rsync::Mirror::Recent->new
(
# ignore_link_stat_errors => 1,
localroot => $root_to,
remote => "$root_from/RECENT-5s.yaml",
# verbose => 1,
max_files_per_connection => 512,
rsync_options => {
links => 1,
times => 1,
compress => 1,
checksum => 1,
},
);
$recc->rmirror;
}
{
my $recc = File::Rsync::Mirror::Recent->new
( # ($root_from, $root_to)
local => "$root_to/RECENT-5s.yaml",
);
diag "\n" if $Opt{verbose};
diag $recc->overview if $Opt{verbose};
}
{
BEGIN {
$tests += 2;
}
my $recc = File::Rsync::Mirror::Recent->new
(
# order matters!
# ignore_link_stat_errors => 1,
localroot => $root_to,
remote => "$root_from/RECENT.recent",
max_files_per_connection => 65,
rsync_options =>
{
links => 1,
times => 1,
compress => 1,
checksum => 1,
},
_runstatusfile => $statusfile,
verbose => $Opt{verbose},
);
$recc->rmirror;
my $rf2 = File::Rsync::Mirror::Recentfile->new_from_file("$root_from/RECENT-5s.yaml");
my $file = "$root_from/about-re-mirroring.txt";
open my $fh, ">", $file or die "Could not open '$file': $!";
print $fh time;
close $fh or die "Could not close '$file': $!";
$rf2->update($file, "new");
$recc->rmirror;
ok -e "$root_to/about-re-mirroring.txt", "picked up the update";
$file = "$root_from/about-re2-mirroring.txt";
undef $fh;
open $fh, ">", $file or die "Could not open '$file': $!";
print $fh time;
close $fh or die "Could not close '$file': $!";
$rf2->update($file, "new", 123456789);
$rf2->aggregate(force => 1);
$rf2->aggregate(force => 1);
$recc->verbose(1) if $Opt{verbose};
# { no warnings 'once'; $DB::single++; }
# x map { $_->dirtymark } @{$self->recentfiles}
# x map { $_->_seeded } @{$self->recentfiles}
# x sort keys %$rf
# $recc->verbose(1)
$recc->rmirror;
ok -e "$root_to/about-re2-mirroring.txt", "picked up a dirty update";
}
{
my $recc = File::Rsync::Mirror::Recent->new
( # ($root_from, $root_to)
local => "$root_to/RECENT-5s.yaml",
);
diag "\n" if $Opt{verbose};
diag $recc->overview if $Opt{verbose};
}
{
my $recc = File::Rsync::Mirror::Recent->new
(
# ignore_link_stat_errors => 1,
localroot => $root_to,
local => "$root_to/RECENT.recent",
);
my %seen;
for my $rf (@{$recc->recentfiles}) {
my $dirtymark = $rf->dirtymark or next;
$seen{ $dirtymark }++;
}
is scalar keys %seen, 1, "all recentfiles have the same dirtymark or we don't know it";
}
}
rmtree [$root_from, $root_to, $statusfile] unless $Opt{verbose};
BEGIN {
plan tests => $tests
}
# Local Variables:
# mode: cperl
# cperl-indent-level: 4
# End: