#!/usr/bin/perl
use strict;
use warnings;
use Search::Tools;
use Search::Tools::Tokenizer;
use Benchmark qw(:all);
use File::Slurp;
my $ascii = read_file('t/docs/test.txt');
my $tokenizer = Search::Tools::Tokenizer->new;
my $q = qr/^(recense|checkerbreast|cannon)$/i;
cmpthese(
1000,
{ 'token_overload' => sub {
my $tokens = $tokenizer->tokenize( $ascii, $q );
while ( my $tok = $tokens->next ) {
"$tok";
}
},
'token_str' => sub {
my $tokens = $tokenizer->tokenize( $ascii, $q );
while ( my $tok = $tokens->next ) {
$tok->str;
}
},
}
);