#!/usr/bin/env perl
use strict;
use Test::More tests => 3;
use lib 't';
use Data::Dump qw( dump );
use_ok('Search::Tools::Tokenizer');
my $file = 't/docs/snip-phrases.txt';
my $buf = Search::Tools->slurp($file);
# test sentence detection in Tokenizer first
my $tokenizer = Search::Tools::Tokenizer->new();
my $tokens = $tokenizer->tokenize( $buf, qr/john/ );
my $sentence_starts = $tokens->get_sentence_starts();
my $heat = $tokens->get_heat();
#dump($sentence_starts);
is_deeply( $sentence_starts, [108], "sentence_starts" );
#undef $tokens;
#diag("Search::Tools::XS_DEBUG=$Search::Tools::XS_DEBUG");
my $tokenizer2 = Search::Tools::Tokenizer->new();
my $tokens2 = $tokenizer2->tokenize( $buf, qr/john/ );
my $sentence_starts2 = $tokens2->get_sentence_starts();
my $heat2 = $tokens2->get_heat();
#dump($sentence_starts2);
is_deeply( $sentence_starts2, [108], "sentence_starts" );