#!/usr/bin/perl
use strict;
use warnings;
use Search::Tools::Tokenizer;
use Search::Tools::UTF8;
use Benchmark qw(:all);
use File::Slurp;
my $ascii = read_file('t/docs/test.txt');
my $ascii_utf8 = to_utf8($ascii);
my $ascii_xs = $ascii; # no utf8 flag!
my $tokenizer = Search::Tools::Tokenizer->new();
cmpthese(
1000,
{ 'ascii-pp' => sub {
my $tokens = $tokenizer->tokenize_pp($ascii);
},
'ascii_utf8-pp' => sub {
my $tokens = $tokenizer->tokenize_pp($ascii_utf8);
},
'ascii-xs' => sub {
my $tokens = $tokenizer->tokenize($ascii_xs);
},
'ascii_utf8-xs' => sub {
my $tokens = $tokenizer->tokenize($ascii_utf8);
},
}
);
sub heat_seeker {
# trivial case
}