#!perl -w
#
# WebSearch
# Copyright (C) 1996-1997 by USC/ISI
# $Id: WebSearch,v 1.1 2008/07/14 03:28:13 Martin Exp $
#
# Complete copyright notice follows below.
sub usage
{
my $msg = shift;
defined($msg) && ($msg ne '') && print STDERR "$0: $msg\n";
print STDERR <<"END";
usage: WebSearch [--engine <name>] [--gui] [--max <integer>] [--options <key=value>]... [--count] [--terse] [--all] [--raw] [--list] [--verbose] [--VERSION] [--help] [--host <hostname>] [--port <portnum>] [--username bbunny --password c4rr0t5] [--lwpdebug] [--debug] query
Make a query to a web search engine, printing to STDOUT the URLs which match (one per line).
See `perldoc WebSearch` for details.
END
exit 1;
} # usage
=head1 NAME
WebSearch - a web-searching application demonstrating WWW::Search
=head1 SYNOPSIS
B<WebSearch [-m MaxCount] [-e SearchEngine] [-o option] [-o option...] [-ardvV] query>
=head1 DESCRIPTION
This program is provides a command-line interface to web search engines,
listing all URLs found for a given query. This program also provides
a simple demonstration of the WWW::Search Perl library for web searches.
The program supports a number of search engines;
use WebSearch --list to see which backends are installed.
A more sophisticated client is L<AutoSearch>
which maintains a change list of found objects.
For examples and hints about searches,
see L<AutoSearch>.
=head1 OPTIONS
WebSearch uses Getopt::Long, so you can use double-minus with long
option names, or single-minus with one-letter abbreviations.
=over 8
=item --engine e_name, -e e_name
The string e_name is the name of (the module for) the desired search
engine. Capitalization matters. See `perldoc WWW::Search` to find
out what the default is (probably Null).
Use --list to get a list of installed backends.
=item --gui, -g
Perform the search to mimic the default browser-based search.
Not implemented for all backends, see the documentation for each backend.
=item --list
Prints to STDERR a \n-separated list of installed backends.
=item --max max_count, -m max_count
Specify the maximum number of hits to retrieve.
=item --option o_string, -o o_string
Specify a search-engine option in the form 'key=value' (or just
'key'). Can be repeated for as many options are needed. Keys can be
repeated.
=item --count, -c
As the first line of output, print the approximate hit count.
As the last line of output, print the actual hit count.
=item --terse, -t
Do not print any URLs.
Only useful if you also specify --count.
If you specify --terse but not --count,
there will be no output no matter how many hits are found!
=item --all, -a
For each hit result, print all the URLs that the search engine
indicated were equivalent to it. (Some URLs may refer to the same
object.) Can be combined with --verbose; can not be combined with
--raw.
=item --raw, -r
For each hit result, print the raw HTML.
Not implemented for all backends.
=item --verbose, -v
Verbose mode. Enumerate the returned URLs and show the description,
score, date, etc. for each.
=item --VERSION, -V
Print version information and exit immediately.
=item --debug <i>, -d <i>
Display back-end debugging information (with debug level <i>)
=item --host <hostname.sub.domain>
Set the _host option for the WWW::Search object (backend-dependent).
=item --port <i>
Set the _port option for the WWW::Search object (backend-dependent).
=item --username <bbunny>
Set the username with which to login to the backend.
=item --password <c4rr0t5>
Set the password with which to login to the backend.
=item --lwpdebug, -l
Display low-level libwww-perl debugging information
=back
=head1 ENVIRONMENT VARIABLES
The environment variable F<http_proxy> (or F<HTTP_PROXY>)
specifies a proxy, if any.
=head1 SEE ALSO
For the library, see L<WWW::Search>.
For a more sophisticated client, see L<AutoSearch>.
=head1 AUTHOR
C<WebSearch> was written by John Heidemann, <johnh@isi.edu>.
C<WebSearch> is maintained by Martin Thurn, <mthurn@cpan.org>.
=head1 COPYRIGHT
Copyright (c) 1996-1997 University of Southern California.
All rights reserved.
Redistribution and use in source and binary forms are permitted
provided that the above copyright notice and this paragraph are
duplicated in all such forms and that any documentation, advertising
materials, and other materials related to such distribution and use
acknowledge that the software was developed by the University of
Southern California, Information Sciences Institute. The name of the
University may not be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
=cut
use strict;
&usage('no arguments given') if ($#ARGV == -1);
# &usage if ($#ARGV >= 0 && $ARGV[0] eq '-?');
use Getopt::Long;
use WWW::Search;
use vars qw( $VERSION );
$VERSION = '2.14';
use vars qw( $sEngine $all $verbose $raw $iMax $print_version $debuglwp $debug );
use vars qw( @options $_host $_port $help $iShowCount $iGui $opt_list );
use vars qw( $opt_username $opt_password );
# Set default values:
$iShowCount = 0;
$iGui = 0;
my $iTerse = 0;
$_host = '';
$_port = '';
$sEngine = '';
undef $debug;
$debuglwp = 0;
$opt_list = 0;
# Get the command-line options:
&Getopt::Long::config(qw(no_ignore_case no_getopt_compat));
&usage('getopt failed') unless &GetOptions('all', \$all,
'count', \$iShowCount,
'debug:i', \$debug,
'engine=s', \$sEngine,
'gui', \$iGui,
'help', \$help,
'host=s', \$_host,
'list',
'lwpdebug', \$debuglwp,
'max=i', \$iMax,
'options=s@', \@options,
'password=s',
'port=s', \$_port,
'raw', \$raw,
'terse', \$iTerse,
'username=s',
'verbose', \$verbose,
'VERSION', \$print_version,
);
&usage('user requested help') if $help;
$debug = 1 if (defined($debug) and ($debug < 1));
&print_version() if defined($print_version);
&list_engines() if ($opt_list);
# there MUST be some argument(s) left, the query:
&usage('no query found on command line') if (scalar(@ARGV) <= 0);
&main(join(" ", @ARGV));
exit 0;
sub print_version
{
print "$0 version $VERSION\nWWW::Search version $WWW::Search::VERSION\n";
if ($sEngine ne '')
{
my $mod = "WWW::Search::$sEngine";
if (eval "use $mod; 1;")
{
my $e = '$'. $mod .'::VERSION';
# print STDERR " + e is ===$e===\n";
my $iVersion = eval $e;
$iVersion ||= 'unknown';
print "$mod version $iVersion\n";
} # if
} # if
exit 0;
} # print_version
sub list_engines
{
my @as = WWW::Search::installed_engines();
$, = "\n";
print STDERR (sort @as), '';
exit 0;
} # list_engines
my $verbose_code;
sub print_result {
my($result, $count) = @_;
return if $iTerse;
my($prefix) = "";
if ($verbose) {
my(@attribs) = ();
$prefix = "$count. ";
if (!defined($verbose_code)) {
$verbose_code = "";
foreach (qw(title description score normalized_score size change_date index_date)) {
$verbose_code .= "push(\@attribs, \"$_: \" . \$result->$_())\n" .
"\tif (defined(\$result->$_()));\n";
};
};
eval $verbose_code;
$prefix .= "(" . join(",\n\t", @attribs) . ")\n\t"
if ($#attribs >= 0);
};
if (defined($all)) {
foreach ($result->urls()) {
print "$prefix$_\n";
$prefix = "\t";
};
} else {
if (defined($raw)) {
print $result->raw(), "\n";
} else {
print $prefix, $result->url, "\n";
};
};
}
sub print_error {
my ($msg, $count) = @_;
my $error = $verbose ? sprintf("[%3d] ", $count) : '';
$error .= $msg;
print STDERR $error, "\n" if $error ne '';
}
sub main {
my $query = shift;
my $count = 0;
my $search = new WWW::Search($sEngine);
$search->{_host} = $_host if $_host ne '';
$search->{_port} = $_port if $_port ne '';
my %hsOptions = ();
if (0 < $debuglwp) {
require LWP::Debug;
LWP::Debug::level('+');
}
$search->http_proxy($ENV{'HTTP_PROXY'}) if ($ENV{'HTTP_PROXY'});
$search->http_proxy($ENV{'http_proxy'}) if ($ENV{'http_proxy'});
if (0 < scalar(@options))
{
foreach my $sPair (@options)
{
if ($sPair =~ m/^([^=]+)=(.*)$/)
{
my ($key, $value) = ($1, $2);
# This is a bit of a hack. A set of CGI options is not
# strictly a hash, because multiple values for the same key
# can be specified. To get around this, we rely on the fact
# that this hash of options is only used to construct a CGI
# parameter list. If we see multiple values for the same key,
# we append the multiple values onto the value of the key in
# CGI '?key=value' format.
if (exists($hsOptions{$key}) && $hsOptions{$key} ne '')
{
# There was already an option of this key given; append
# multiple values as CGI arguments:
$hsOptions{$key} .= '&'.$key.'='.WWW::Search::escape_query($value);
} # if exists
else
{
# This is the only instance of this key; just insert the
# hash value:
$hsOptions{$key} = WWW::Search::escape_query($value);
}
} # if option is of the form key=value
else
{
$hsOptions{$sPair} = '';
}
} # foreach $sPair
} # if @options
if (defined($iMax))
{
$search->maximum_to_retrieve($iMax);
}
else
{
$iMax = 10000;
}
$hsOptions{'search_debug'} = $debug if (defined($debug) and (0 < $debug));
$iGui
? $search->gui_query(WWW::Search::escape_query($query), \%hsOptions)
: $search->native_query(WWW::Search::escape_query($query), \%hsOptions);
$search->login($opt_username, $opt_password);
my($way) = 0; # 0=piecemeal, 1=all at once
my($result);
if (($iShowCount) && defined($search->approximate_result_count))
{
print "There are approximately " if $verbose;
print $search->approximate_result_count;
print " results." if $verbose;
print "\n";
} # if
my $iNumPrinted = 0;
if ($way) # return all at once.
{
foreach $result ($search->results())
{
print_result($result, ++$count);
$iNumPrinted++;
last if ($iMax < $count);
};
}
else # return page by page
{
while ($result = $search->next_result())
{
print_result($result, ++$count);
$iNumPrinted++;
last if ($iMax < $count);
} # while
} # else page by page
if ($iShowCount)
{
print "There are exactly " if $verbose;
print $iNumPrinted;
print " results (maximum of $iMax were requested)." if $verbose;
print "\n";
} # if
# handle errors
if ($count == 0) {
my($response) = $search->response();
my $nothing = $verbose ? "Nothing found." : '';
if ($response->is_success) {
print_error($nothing, $count);
} else {
print_error("Error: " . $response->as_string(), $count);
};
};
};