@@ -1,76 +1,83 @@
Revision history for Perl extension Net::Google::SafeBrowsing2.
-0.1 Wed Jul 21 16:35:48 2010
+0.1 Wed Jul 21 2011 16:35:48 2010
- original version
-0.2 Fri Nov 05 14:45:48 2010
+0.2 Fri Nov 05 2011 14:45:48 2010
- add support for MAC
-0.5 Mon Mar 07 16:55:00 2011
- - add MySQl back-end storage
- - introduce base class Net::Google::SafeBrowsing2::DBI
- - update documentation
+0.5 Mon Mar 07 2012 16:55:00 2011
+ - add MySQl back-end storage
+ - introduce base class Net::Google::SafeBrowsing2::DBI
+ - update documentation
- Tue Apr 05 14:40:00 2011
- - fix Net::Google::SafeBrowsing2::DBI to work with all database types
+Tue Apr 05 2012 14:40:00 2011
+ - fix Net::Google::SafeBrowsing2::DBI to work with all database types
-0.6 Mon Jul 25 13:21:00 2011
- - Handle local database reset
+0.6 Mon Jul 25 2012 13:21:00 2011
+ - Handle local database reset
-0.7 Mon Jul 25 15:18:00 2011
- - Add local_lookup to perform a lookup against the local database only for debugging purpose.
- - Fix duplicate insert of add chunks and sub chunks.
+0.7 Mon Jul 25 2012 15:18:00 2011
+ - Add local_lookup to perform a lookup against the local database only for debugging purpose.
+ - Fix duplicate insert of add chunks and sub chunks.
-0.8 Tue Aug 02 13:05:00 2011
- - Fix chunk retrieval with MySQL backend (VARCHAR to VARBINARY)
- - Reduce the number of full hashes requested
+0.8 Tue Aug 02 2012 13:05:00 2011
+ - Fix chunk retrieval with MySQL backend (VARCHAR to VARBINARY)
+ - Reduce the number of full hashes requested
-0.9 Tue Aug 09 15:26:00 2011
- - Fix bug with local whitelisting (sub chunks).
- - Fix the parsing of full hashes.
+0.9 Tue Aug 09 2012 15:26:00 2011
+ - Fix bug with local whitelisting (sub chunks).
+ - Fix the parsing of full hashes.
-1.0 Wed Aug 10 15:11:00
- - Storage: add keep_all argument to keep expired full hashes in the database
- - Separate the error output from debug output
- - A lot of testing
+1.0 Wed Aug 10 2012 15:11:00
+ - Storage: add keep_all argument to keep expired full hashes in the database
+ - Separate the error output from debug output
+ - A lot of testing
-1.01 Wed Aug 26 13:49:00
- - Use String::HexConvert for faster hex_to_ascii
+1.01 Wed Aug 26 2012 13:49:00
+ - Use String::HexConvert for faster hex_to_ascii
-1.02 Thu Nov 10 10:00:00
- - Fix uninitialized $self->{errors} variable
+1.02 Thu Nov 10 2012 10:00:00
+ - Fix uninitialized $self->{errors} variable
-1.03 Thu Nov 21 17:24:00
- - no code change
- - source code available on github at https://github.com/juliensobrier/Net-Google-SafeBrowsing2
- - minor documentation updates
+1.03 Thu Nov 21 2012 17:24:00
+ - no code change
+ - source code available on github at https://github.com/juliensobrier/Net-Google-SafeBrowsing2
+ - minor documentation updates
-1.04 Wed Dec 07 10:26:00
- - add Net::Google::SafeBrowsing2::Lookup (http://code.google.com/apis/safebrowsing/lookup_guide.html)
+1.04 Wed Dec 07 2012 10:26:00
+ - add Net::Google::SafeBrowsing2::Lookup (http://code.google.com/apis/safebrowsing/lookup_guide.html)
- Wed Dec 07 10:49:00
- - Net::Google::SafeBrowsing2::Sqlite, index s_chunks_unique was created at the wrong place
+Wed Dec 07 2012 10:49:00
+ - Net::Google::SafeBrowsing2::Sqlite, index s_chunks_unique was created at the wrong place
- Tue Dec 12 15:41:00
- - Net::Google::SafeBrowsing2::Lookup 0.2: documentation update
+Tue Dec 12 2012 15:41:00
+ - Net::Google::SafeBrowsing2::Lookup 0.2: documentation update
-1.05 Thu Jan 12 14:26:00
- - No code change. Move C<local_lookup> to PRIVATE FUNCTIONS to avoid confusions.
+1.05 Thu Jan 12 2013 14:26:00
+ - No code change. Move C<local_lookup> to PRIVATE FUNCTIONS to avoid confusions.
-1.05 Mon May 07 10:22:00
- - Added Postgres storage from njohnston
+1.05 Mon May 07 2013 10:22:00
+ - Added Postgres storage from njohnston
-1.06 Sun Jun 03 21:40:00
- - Update DBI and MySQL back-end to keep empty sub chunks. Fix index for s_chunks table
- - MySQL: truncate prefixes longer than 8 bytes
- - Add a warning about the number of updates required with an empty database
- - Storage: reset function is required
+1.06 Sun Jun 03 2013 21:40:00
+ - Update DBI and MySQL back-end to keep empty sub chunks. Fix index for s_chunks table
+ - MySQL: truncate prefixes longer than 8 bytes
+ - Add a warning about the number of updates required with an empty database
+ - Storage: reset function is required
-1.07 Fri Jun 08 18:38:00
- - Import/Export feature for add chunks and sub chunks. Can be used to move from one back-end storage to another.
+1.07 Fri Jun 08 2013 18:38:00
+ - Import/Export feature for add chunks and sub chunks. Can be used to move from one back-end storage to another.
1.08 Mon Apr 08 2013 07:48:00
- - Fix Po documentation (contribution)
+ - Fix Po documentation (contribution)
1.09 Wed Jun 05 2013 20:29:00
- - Use HTTPS to access safebrowsing.clients.google.com/.
\ No newline at end of file
+ - Use HTTPS to access safebrowsing.clients.google.com/.
+
+1.10 Tue Dec 31 2013 22:15:00
+ - Force IPv4 to solve bug on CentOS.
+
+1.11 Fri Jul 11 2014 21:41:00
+ - Add dependency on IO::Socket::SSL
+ - Remove dependency on Net::IPAddress
\ No newline at end of file
@@ -35,11 +35,11 @@
"Digest::HMAC_SHA1" : "0",
"Digest::SHA" : "0",
"File::Slurp" : "0",
+ "IO::Socket::SSL" : "0",
"LWP::UserAgent" : "0",
"List::Util" : "0",
"MIME::Base64" : "0",
"MIME::Base64::URLSafe" : "0",
- "Net::IPAddress" : "0",
"String::HexConvert" : "0",
"Test::More" : "0",
"Text::Trim" : "0",
@@ -49,5 +49,5 @@
}
},
"release_status" : "stable",
- "version" : "1.09"
+ "version" : "1.11"
}
@@ -21,14 +21,14 @@ requires:
Digest::HMAC_SHA1: 0
Digest::SHA: 0
File::Slurp: 0
+ IO::Socket::SSL: 0
LWP::UserAgent: 0
List::Util: 0
MIME::Base64: 0
MIME::Base64::URLSafe: 0
- Net::IPAddress: 0
String::HexConvert: 0
Test::More: 0
Text::Trim: 0
URI: 0
constant: 0
-version: 1.09
+version: 1.11
@@ -10,7 +10,6 @@ WriteMakefile(
Digest::SHA => 0,
List::Util => 0,
constant => 0,
- Net::IPAddress => 0,
Test::More => 0,
Text::Trim => 0,
Digest::HMAC_SHA1 => 0,
@@ -18,6 +17,7 @@ WriteMakefile(
MIME::Base64 => 0,
String::HexConvert => 0,
File::Slurp => 0,
+ IO::Socket::SSL => 0,
},
($] >= 5.005 ? ## Add these new keywords supported since 5.005
(ABSTRACT_FROM => 'lib/Net/Google/SafeBrowsing2.pm', # retrieve abstract from module
@@ -1,4 +1,4 @@
-Net-Google-SafeBrowsing2 version 1.08
+Net-Google-SafeBrowsing2 version 1.11
=====================================
Net::Google::SafeBrowsing2 implements the Google Safe Browsing v2 API.
@@ -33,7 +33,6 @@ This module requires these other modules and libraries:
Digest::SHA
List::Util
constant
- Net::IPAddress
Test::More
Text::Trim
Digest::HMAC_SHA1
@@ -41,6 +40,7 @@ This module requires these other modules and libraries:
MIME::Base64
String::HexConvert
File::Slurp
+ IO::Socket::SSL
COPYRIGHT AND LICENCE
@@ -51,4 +51,3 @@ This library is free software; you can redistribute it and/or modify
it under the same terms as Perl itself, either Perl version 5.8.8 or,
at your option, any later version of Perl 5 you may have available.
-
@@ -8,20 +8,19 @@ use LWP::UserAgent;
use URI;
use Digest::SHA qw(sha256);
use List::Util qw(first);
-use Net::IPAddress;
use Text::Trim;
use Digest::HMAC_SHA1 qw(hmac_sha1 hmac_sha1_hex);
use MIME::Base64::URLSafe;
use MIME::Base64;
use String::HexConvert;
use File::Slurp;
-use IO::Socket::SSL;
+use IO::Socket::SSL 'inet4' ;
use Exporter 'import';
our @EXPORT = qw(DATABASE_RESET MAC_ERROR MAC_KEY_ERROR INTERNAL_ERROR SERVER_ERROR NO_UPDATE NO_DATA SUCCESSFUL MALWARE PHISHING);
-our $VERSION = '1.09';
+our $VERSION = '1.11';
BEGIN {
IO::Socket::SSL::set_ctx_defaults(
@@ -158,6 +157,14 @@ Arguments
=over 4
+=item server
+
+Safe Browsing Server. https://safebrowsing.clients.google.com/safebrowsing/ by default
+
+=item mac_server
+
+Safe Browsing MAC Server. https://sb-ssl.google.com/safebrowsing/ by default
+
=item key
Required. Your Google Safe browsing API key
@@ -200,13 +207,15 @@ sub new {
my ($class, %args) = @_;
my $self = { # default arguments
+ server => 'https://safebrowsing.clients.google.com/safebrowsing/',
+ mac_server => 'https://sb-ssl.google.com/safebrowsing/',
list => ['googpub-phish-shavar', 'goog-malware-shavar'],
- key => '',
+ key => '',
version => '2.2',
debug => 0,
errors => 0,
last_error => '',
- mac => 0,
+ mac => 0,
perf => 0,
%args,
@@ -268,10 +277,10 @@ Be careful if you set this option to 1 as too frequent updates might result in t
sub update {
my ($self, %args) = @_;
-# my @lists = @{[$args{list}]} || @{$self->{list}} || croak "Missing list name\n";
- my $list = $args{list};
- my $force = $args{force} || 0;
- my $mac = $args{mac} || $self->{mac} || 0;
+# my @lists = @{[$args{list}]} || @{$self->{list}} || croak "Missing list name\n";
+ my $list = $args{list};
+ my $force = $args{force} || 0;
+ my $mac = $args{mac} || $self->{mac} || 0;
my @lists = @{$self->{list}};
@@ -318,7 +327,7 @@ sub update {
my $ua = $self->ua;
- my $url = "https://safebrowsing.clients.google.com/safebrowsing/downloads?client=api&apikey=" . $self->{key} . "&appver=$VERSION&pver=" . $self->{version};
+ my $url = $self->{server} . "downloads?client=api&apikey=" . $self->{key} . "&appver=$VERSION&pver=" . $self->{version};
$url .= "&wrkey=$wrapped_key" if ($mac);
my $body = '';
@@ -607,7 +616,7 @@ NOTE: this function is useless in practice because Google includes some lists wh
sub get_lists {
my ($self, %args) = @_;
- my $url = "https://safebrowsing.clients.google.com/safebrowsing/list?client=api&apikey=" . $self->{key} . "&appver=$VERSION&pver=" . $self->{version};
+ my $url = $self->{server} . "list?client=api&apikey=" . $self->{key} . "&appver=$VERSION&pver=" . $self->{version};
my $res = $self->ua->get($url);
@@ -917,7 +926,7 @@ sub request_mac_keys {
my $client_key = '';
my $wrapped_key = '';
- my $url = "https://sb-ssl.google.com/safebrowsing/newkey?client=api&apikey=" . $self->{key} . "&appver=$VERSION&pver=" . $self->{version};
+ my $url = $self->{mac_server} . "newkey?client=api&apikey=" . $self->{key} . "&appver=$VERSION&pver=" . $self->{version};
my $res = $self->ua->get($url);
@@ -1457,16 +1466,14 @@ sub canonical_uri {
$escape =~ s/^([a-z]+:\/\/[^\/]+)(\?.*)$/$1\/$2/gi;
# $self->debug("\t$escape\n");
- # other weird case if domain = digits only, try to translte it to IP address
+ # other weird case if domain = digits only, try to translate it to IP address
if ((my $domain = URI->new($escape)->host) =~/^\d+$/) {
- my $ip = num2ip($domain);
+ my $ip = Socket::inet_ntoa(Socket::inet_aton($domain));
- if (validaddr($ip)) {
- $uri = URI->new($escape);
- $uri->host($ip);
+ $uri = URI->new($escape);
+ $uri->host($ip);
- $escape = $uri->as_string;
- }
+ $escape = $uri->as_string;
}
# $self->debug("1. $url => $escape\n");
@@ -1600,7 +1607,7 @@ sub request_full_hash {
}
}
- my $url = "https://safebrowsing.clients.google.com/safebrowsing/gethash?client=api&apikey=" . $self->{key} . "&appver=$VERSION&pver=" . $self->{version};
+ my $url = $self->{server} . "gethash?client=api&apikey=" . $self->{key} . "&appver=$VERSION&pver=" . $self->{version};
my $prefix_list = join('', @$prefixes);
my $header = "$size:" . scalar @$prefixes * $size;
@@ -1777,6 +1784,15 @@ sub expand_range {
=over 4
+=item 1.11
+
+Add dependency on IO::Socket::SSL.
+Remove dependency on Net::IPAddress.
+
+=item 1.10
+
+Force IPv4 to solve bug on CentOS.
+
=item 1.09
Use HTTPS to access safebrowsing.clients.google.com/.
@@ -1850,12 +1866,16 @@ Add support for Message Authentication Code (MAC)
=head1 SEE ALSO
+Source code available at L<https://github.com/juliensobrier/Net-Google-SafeBrowsing2>.
+
See L<Net::Google::SafeBrowsing2::Storage>, L<Net::Google::SafeBrowsing2::Sqlite> and L<Net::Google::SafeBrowsing2::MySQL> for information on storing and managing the Google Safe Browsing database.
Google Safe Browsing v2 API: L<http://code.google.com/apis/safebrowsing/developers_guide_v2.html>
L<Net::Google::SafeBrowsing> (Google Safe Browsing v1) is deprecated by Google since 12/01/2011.
+L<Net::Google::SafeBrowsing2> (Google Safe Browsing v2) will deprecated by Google on 12/01/2014.
+
=head1 AUTHOR
Julien Sobrier, E<lt>jsobrier@zscaler.comE<gt> or E<lt>julien@sobrier.netE<gt>
@@ -67,6 +67,8 @@ is( $gsb->canonical_uri('http://www.google.com/a/../b/..')->as_string, 'http://w
is( $gsb->canonical_uri('http://www.google.com/a/../b/..?foo')->as_string, 'http://www.google.com/?foo', 'canonicalization "http://www.google.com/a/../b/..?foo" is OK');
is( $gsb->canonical_uri('http://www.google.com/#a#b')->as_string, 'http://www.google.com/', 'canonicalization "http://www.google.com/#a#b" is OK');
is( $gsb->canonical_uri('http://www.google.com/#a#b#c')->as_string, 'http://www.google.com/', 'canonicalization "http://www.google.com/#a#b#c" is OK');
+is( $gsb->canonical_uri('http://16843009/index.html')->as_string, 'http://1.1.1.1/index.html', 'canonicalization "http://16843009/index.html" is OK');
+is( $gsb->canonical_uri('http://1/index.html')->as_string, 'http://0.0.0.1/index.html', 'canonicalization "http://16843009/index.html" is OK');
# Form Google API doc, possible strings for lookup
my @values = $gsb->canonical('http://a.b.c/1/2.html?param=1');