@@ -1,11 +1,37 @@
Revision history for AnyEvent::HTTP
TODO: provide lwp_request function that takes an lwp http requets and returns a http response.
-TODO: httpbis: $location = URI->new_abs($location, "$scheme://$host:$port$path_query")->as_string;
TODO: set_proxy hook
TODO: use proxy hook
-TODO: ip6 literals in url
TODO: maybe read big chunks in smaller portions for chunked-encoding + on_body.
+TODO: on_upgrade, for 101 responses?
+TODO: document session vs. sessionid correctly.
+TODO: support proxy username:password in both proxy switch and set_proxy string (dzagashev@gmail.com)
+TODO: remove "unexpectedly got a destructed handle"
+
+TODO: callback as body (Kostirya)
+TODO: infinite recursion(?) (Kostirya)
+
+2.22 Thu May 14 04:04:03 CEST 2015
+ - ipv6 literals were not correctly parsed (analyzed by Raphael Geissert).
+ - delete the body when mutating request to GET request when
+ redirecting (reported by joe trader).
+ - send proxy-authorization header to proxy when using CONNECT
+ (reported by dzagashev@gmail.com).
+ - do not send Proxy-Authroization header when not using a proxy.
+ - when retrying a persistent request, switch persistency off.
+ - added t/02_ip_literals.t.
+
+2.21 Mon Jun 9 01:35:54 CEST 2014
+ - correctly keep body when redirecting POSTs, instead of
+ deleting them.
+
+2.2 Mon Jun 9 01:31:46 CEST 2014
+ - connection header was malformed (patch by Raphael Geissert).
+ - add lots of known idempotent methods from httpbis.
+ - implement relative location headers (rfc 7231), with fallback on URI.
+ - add support for status code 308 from rfc 7238.
+ - recommend URI.
2.15 Wed Nov 14 23:22:07 CET 2012
- use the recurse parameter to also limit the number of retries to be
@@ -48,7 +48,7 @@ use AnyEvent::Handle ();
use base Exporter::;
-our $VERSION = '2.15';
+our $VERSION = 2.22;
our @EXPORT = qw(http_get http_post http_head http_request);
@@ -91,7 +91,7 @@ object at least alive until the callback get called. If the object gets
destroyed before the callback is called, the request will be cancelled.
The callback will be called with the response body data as first argument
-(or C<undef> if an error occured), and a hash-ref with response headers
+(or C<undef> if an error occurred), and a hash-ref with response headers
(and trailers) as second argument.
All the headers in that hash are lowercased. In addition to the response
@@ -125,7 +125,7 @@ message. Currently the following status codes are used:
=over 4
-=item 595 - errors during connection etsbalishment, proxy handshake.
+=item 595 - errors during connection establishment, proxy handshake.
=item 596 - errors during TLS negotiation, request sending and header processing.
@@ -159,6 +159,11 @@ include:
Whether to recurse requests or not, e.g. on redirects, authentication and
other retries and so on, and how often to do so.
+Only redirects to http and https URLs are supported. While most common
+redirection forms are handled entirely within this module, some require
+the use of the optional L<URI> module. If it is required but missing, then
+the request will fail with an error.
+
=item headers => hashref
The request headers to use. Currently, C<http_request> may provide its own
@@ -192,6 +197,9 @@ C<$scheme> must be either missing or must be C<http> for HTTP.
If not specified, then the default proxy is used (see
C<AnyEvent::HTTP::set_proxy>).
+Currently, if your proxy requires authorization, you have to specify an
+appropriate "Proxy-Authorization" header in every request.
+
=item body => $string
The request body, usually empty. Will be sent as-is (future versions of
@@ -244,7 +252,7 @@ context) - only connections using the same unique ID will be reused.
=item on_prepare => $callback->($fh)
In rare cases you need to "tune" the socket before it is used to
-connect (for exmaple, to bind it on a given IP address). This parameter
+connect (for example, to bind it on a given IP address). This parameter
overrides the prepare callback passed to C<AnyEvent::Socket::tcp_connect>
and behaves exactly the same way (e.g. it has to provide a
timeout). See the description for the C<$prepare_cb> argument of
@@ -691,6 +699,44 @@ sub _error(\%$$) {
()
}
+our %IDEMPOTENT = (
+ DELETE => 1,
+ GET => 1,
+ HEAD => 1,
+ OPTIONS => 1,
+ PUT => 1,
+ TRACE => 1,
+
+ ACL => 1,
+ "BASELINE-CONTROL" => 1,
+ BIND => 1,
+ CHECKIN => 1,
+ CHECKOUT => 1,
+ COPY => 1,
+ LABEL => 1,
+ LINK => 1,
+ MERGE => 1,
+ MKACTIVITY => 1,
+ MKCALENDAR => 1,
+ MKCOL => 1,
+ MKREDIRECTREF => 1,
+ MKWORKSPACE => 1,
+ MOVE => 1,
+ ORDERPATCH => 1,
+ PROPFIND => 1,
+ PROPPATCH => 1,
+ REBIND => 1,
+ REPORT => 1,
+ SEARCH => 1,
+ UNBIND => 1,
+ UNCHECKOUT => 1,
+ UNLINK => 1,
+ UNLOCK => 1,
+ UPDATE => 1,
+ UPDATEREDIRECTREF => 1,
+ "VERSION-CONTROL" => 1,
+);
+
sub http_request($$@) {
my $cb = pop;
my ($method, $url, %arg) = @_;
@@ -729,7 +775,7 @@ sub http_request($$@) {
: $uscheme eq "https" ? 443
: return $cb->(undef, { @pseudo, Status => 599, Reason => "Only http and https URL schemes supported" });
- $uauthority =~ /^(?: .*\@ )? ([^\@:]+) (?: : (\d+) )?$/x
+ $uauthority =~ /^(?: .*\@ )? ([^\@]+?) (?: : (\d+) )?$/x
or return $cb->(undef, { @pseudo, Status => 599, Reason => "Unparsable URL" });
my $uhost = lc $1;
@@ -775,7 +821,7 @@ sub http_request($$@) {
$hdr{"content-length"} = length $arg{body}
if length $arg{body} || $method ne "GET";
- my $idempotent = $method =~ /^(?:GET|HEAD|PUT|DELETE|OPTIONS|TRACE)$/;
+ my $idempotent = $IDEMPOTENT{$method};
# default value for keepalive is true iff the request is for an idempotent method
my $persistent = exists $arg{persistent} ? !!$arg{persistent} : $idempotent;
@@ -785,7 +831,7 @@ sub http_request($$@) {
# the key to use in the keepalive cache
my $ka_key = "$uscheme\x00$uhost\x00$uport\x00$arg{sessionid}";
- $hdr{connection} = ($persistent ? $keepalive ? "keep-alive " : "" : "close ") . "Te"; #1.1
+ $hdr{connection} = ($persistent ? $keepalive ? "keep-alive, " : "" : "close, ") . "Te"; #1.1
$hdr{te} = "trailers" unless exists $hdr{te}; #1.1
my %state = (connect_guard => 1);
@@ -805,10 +851,10 @@ sub http_request($$@) {
"$method $rpath HTTP/1.1\015\012"
. (join "", map "\u$_: $hdr{$_}\015\012", grep defined $hdr{$_}, keys %hdr)
. "\015\012"
- . (delete $arg{body})
+ . $arg{body}
);
- # return if error occured during push_write()
+ # return if error occurred during push_write()
return unless %state;
# reduce memory usage, save a kitten, also re-use it for the response headers.
@@ -845,19 +891,42 @@ sub http_request($$@) {
}
# redirect handling
- # microsoft and other shitheads don't give a shit for following standards,
- # try to support some common forms of broken Location headers.
- if ($hdr{location} !~ /^(?: $ | [^:\/?\#]+ : )/x) {
- $hdr{location} =~ s/^\.\/+//;
+ # relative uri handling forced by microsoft and other shitheads.
+ # we give our best and fall back to URI if available.
+ if (exists $hdr{location}) {
+ my $loc = $hdr{location};
+
+ if ($loc =~ m%^//%) { # //
+ $loc = "$rscheme:$loc";
+
+ } elsif ($loc eq "") {
+ $loc = $url;
+
+ } elsif ($loc !~ /^(?: $ | [^:\/?\#]+ : )/x) { # anything "simple"
+ $loc =~ s/^\.\/+//;
+
+ if ($loc !~ m%^[.?#]%) {
+ my $prefix = "$rscheme://$uhost:$uport";
+
+ unless ($loc =~ s/^\///) {
+ $prefix .= $upath;
+ $prefix =~ s/\/[^\/]*$//;
+ }
- my $url = "$rscheme://$uhost:$uport";
+ $loc = "$prefix/$loc";
- unless ($hdr{location} =~ s/^\///) {
- $url .= $upath;
- $url =~ s/\/[^\/]*$//;
+ } elsif (eval { require URI }) { # uri
+ $loc = URI->new_abs ($loc, $url)->as_string;
+
+ } else {
+ return _error %state, $cb, { @pseudo, Status => 599, Reason => "Cannot parse Location (URI module missing)" };
+ #$hdr{Status} = 599;
+ #$hdr{Reason} = "Unparsable Redirect (URI module missing)";
+ #$recurse = 0;
+ }
}
- $hdr{location} = "$url/$hdr{location}";
+ $hdr{location} = $loc;
}
my $redirect;
@@ -869,12 +938,16 @@ sub http_request($$@) {
# 301, 302 and 303, in contrast to HTTP/1.0 and 1.1.
# also, the UA should ask the user for 301 and 307 and POST,
# industry standard seems to be to simply follow.
- # we go with the industry standard.
+ # we go with the industry standard. 308 is defined
+ # by rfc7538
if ($status == 301 or $status == 302 or $status == 303) {
- # HTTP/1.1 is unclear on how to mutate the method
- $method = "GET" unless $method eq "HEAD";
$redirect = 1;
- } elsif ($status == 307) {
+ # HTTP/1.1 is unclear on how to mutate the method
+ unless ($method eq "HEAD") {
+ $method = "GET";
+ delete $arg{body};
+ }
+ } elsif ($status == 307 or $status == 308) {
$redirect = 1;
}
}
@@ -1052,10 +1125,10 @@ sub http_request($$@) {
%state = ();
$state{recurse} =
http_request (
- $method => $url,
+ $method => $url,
%arg,
- recurse => $recurse - 1,
- keepalive => 0,
+ recurse => $recurse - 1,
+ persistent => 0,
sub {
%state = ();
&$cb
@@ -1111,8 +1184,12 @@ sub http_request($$@) {
if ($proxy && $uscheme eq "https") {
# oh dear, we have to wrap it into a connect request
+ my $auth = exists $hdr{"proxy-authorization"}
+ ? "proxy-authorization: " . (delete $hdr{"proxy-authorization"}) . "\015\012"
+ : "";
+
# maybe re-use $uauthority with patched port?
- $state{handle}->push_write ("CONNECT $uhost:$uport HTTP/1.0\015\012\015\012");
+ $state{handle}->push_write ("CONNECT $uhost:$uport HTTP/1.0\015\012$auth\015\012");
$state{handle}->push_read (line => $qr_nlnl, sub {
$_[1] =~ /^HTTP\/([0-9\.]+) \s+ ([0-9]{3}) (?: \s+ ([^\015\012]*) )?/ix
or return _error %state, $cb, { @pseudo, Status => 599, Reason => "Invalid proxy connect response ($_[1])" };
@@ -1125,6 +1202,8 @@ sub http_request($$@) {
}
});
} else {
+ delete $hdr{"proxy-authorization"} unless $proxy;
+
$handle_actual_request->();
}
};
@@ -1196,7 +1275,7 @@ string of the form C<http://host:port>, croaks otherwise.
To clear an already-set proxy, use C<undef>.
-When AnyEvent::HTTP is laoded for the first time it will query the
+When AnyEvent::HTTP is loaded for the first time it will query the
default proxy from the operating system, currently by looking at
C<$ENV{http_proxy>}.
@@ -1208,21 +1287,22 @@ cookies.
You should call this function (with a true C<$session_end>) before you
save cookies to disk, and you should call this function after loading them
-again. If you have a long-running program you can additonally call this
+again. If you have a long-running program you can additionally call this
function from time to time.
A cookie jar is initially an empty hash-reference that is managed by this
-module. It's format is subject to change, but currently it is like this:
+module. Its format is subject to change, but currently it is as follows:
The key C<version> has to contain C<1>, otherwise the hash gets
emptied. All other keys are hostnames or IP addresses pointing to
hash-references. The key for these inner hash references is the
server path for which this cookie is meant, and the values are again
-hash-references. The keys of those hash-references is the cookie name, and
+hash-references. Each key of those hash-references is a cookie name, and
the value, you guessed it, is another hash-reference, this time with the
key-value pairs from the cookie, except for C<expires> and C<max-age>,
which have been replaced by a C<_expires> key that contains the cookie
-expiry timestamp.
+expiry timestamp. Session cookies are indicated by not having an
+C<_expires> key.
Here is an example of a cookie jar with a single cookie, so you have a
chance of understanding the above paragraph:
@@ -1266,7 +1346,7 @@ C<Mozilla/5.0 (compatible; U; AnyEvent-HTTP/$VERSION; +http://software.schmorp.d
=item $AnyEvent::HTTP::MAX_PER_HOST
The maximum number of concurrent connections to the same host (identified
-by the hostname). If the limit is exceeded, then the additional requests
+by the hostname). If the limit is exceeded, then additional requests
are queued until previous connections are closed. Both persistent and
non-persistent connections are counted in this limit.
@@ -1274,13 +1354,13 @@ The default value for this is C<4>, and it is highly advisable to not
increase it much.
For comparison: the RFC's recommend 4 non-persistent or 2 persistent
-connections, older browsers used 2, newers (such as firefox 3) typically
-use 6, and Opera uses 8 because like, they have the fastest browser and
-give a shit for everybody else on the planet.
+connections, older browsers used 2, newer ones (such as firefox 3)
+typically use 6, and Opera uses 8 because like, they have the fastest
+browser and give a shit for everybody else on the planet.
=item $AnyEvent::HTTP::PERSISTENT_TIMEOUT
-The time after which idle persistent conenctions get closed by
+The time after which idle persistent connections get closed by
AnyEvent::HTTP (default: C<3>).
=item $AnyEvent::HTTP::ACTIVE
@@ -1355,7 +1435,7 @@ eval {
=head2 SHOWCASE
-This section contaisn some more elaborate "real-world" examples or code
+This section contains some more elaborate "real-world" examples or code
snippets.
=head2 HTTP/1.1 FILE DOWNLOAD
@@ -1369,7 +1449,7 @@ HTTP/1.0 servers as well, and usually falls back to a complete re-download
on older servers.
It calls the completion callback with either C<undef>, which means a
-nonretryable error occured, C<0> when the download was partial and should
+nonretryable error occurred, C<0> when the download was partial and should
be retried, and C<1> if it was successful.
use AnyEvent::HTTP;
@@ -6,5 +6,6 @@ Makefile.PL
HTTP.pm
t/00_load.t
t/01_basic.t
+t/02_ip_literals.t
META.yml Module YAML meta-data (added by MakeMaker)
META.json Module JSON meta-data (added by MakeMaker)
@@ -4,7 +4,7 @@
"unknown"
],
"dynamic_config" : 1,
- "generated_by" : "ExtUtils::MakeMaker version 6.62, CPAN::Meta::Converter version 2.112150",
+ "generated_by" : "ExtUtils::MakeMaker version 6.98, CPAN::Meta::Converter version 2.142060",
"license" : [
"unknown"
],
@@ -22,15 +22,18 @@
"prereqs" : {
"build" : {
"requires" : {
- "ExtUtils::MakeMaker" : 0
+ "ExtUtils::MakeMaker" : "0"
}
},
"configure" : {
"requires" : {
- "ExtUtils::MakeMaker" : 0
+ "ExtUtils::MakeMaker" : "0"
}
},
"runtime" : {
+ "recommends" : {
+ "URI" : "0"
+ },
"requires" : {
"AnyEvent" : "5.33",
"common::sense" : "3.3"
@@ -38,5 +41,5 @@
}
},
"release_status" : "stable",
- "version" : "2.15"
+ "version" : 2.22
}
@@ -3,21 +3,23 @@ abstract: unknown
author:
- unknown
build_requires:
- ExtUtils::MakeMaker: 0
+ ExtUtils::MakeMaker: '0'
configure_requires:
- ExtUtils::MakeMaker: 0
+ ExtUtils::MakeMaker: '0'
dynamic_config: 1
-generated_by: 'ExtUtils::MakeMaker version 6.62, CPAN::Meta::Converter version 2.112150'
+generated_by: 'ExtUtils::MakeMaker version 6.98, CPAN::Meta::Converter version 2.142060'
license: unknown
meta-spec:
url: http://module-build.sourceforge.net/META-spec-v1.4.html
- version: 1.4
+ version: '1.4'
name: AnyEvent-HTTP
no_index:
directory:
- t
- inc
+recommends:
+ URI: '0'
requires:
- AnyEvent: 5.33
- common::sense: 3.3
-version: 2.15
+ AnyEvent: '5.33'
+ common::sense: '3.3'
+version: 2.22
@@ -12,6 +12,11 @@ my $mm = MM->new({
AnyEvent => 5.33,
common::sense => 3.3,
},
+ META_MERGE => {
+ recommends => {
+ URI => 0,
+ },
+ },
});
$mm->flush;
@@ -52,7 +52,7 @@ DESCRIPTION
cancelled.
The callback will be called with the response body data as first
- argument (or "undef" if an error occured), and a hash-ref with
+ argument (or "undef" if an error occurred), and a hash-ref with
response headers (and trailers) as second argument.
All the headers in that hash are lowercased. In addition to the
@@ -84,7 +84,7 @@ DESCRIPTION
590-599 and the "Reason" pseudo-header will contain an error
message. Currently the following status codes are used:
- 595 - errors during connection etsbalishment, proxy handshake.
+ 595 - errors during connection establishment, proxy handshake.
596 - errors during TLS negotiation, request sending and header
processing.
597 - errors during body receiving or processing.
@@ -111,6 +111,12 @@ DESCRIPTION
authentication and other retries and so on, and how often to do
so.
+ Only redirects to http and https URLs are supported. While most
+ common redirection forms are handled entirely within this
+ module, some require the use of the optional URI module. If it
+ is required but missing, then the request will fail with an
+ error.
+
headers => hashref
The request headers to use. Currently, "http_request" may
provide its own "Host:", "Content-Length:", "Connection:" and
@@ -143,6 +149,10 @@ DESCRIPTION
If not specified, then the default proxy is used (see
"AnyEvent::HTTP::set_proxy").
+ Currently, if your proxy requires authorization, you have to
+ specify an appropriate "Proxy-Authorization" header in every
+ request.
+
body => $string
The request body, usually empty. Will be sent as-is (future
versions of this module might offer more options).
@@ -191,7 +201,7 @@ DESCRIPTION
on_prepare => $callback->($fh)
In rare cases you need to "tune" the socket before it is used to
- connect (for exmaple, to bind it on a given IP address). This
+ connect (for example, to bind it on a given IP address). This
parameter overrides the prepare callback passed to
"AnyEvent::Socket::tcp_connect" and behaves exactly the same way
(e.g. it has to provide a timeout). See the description for the
@@ -374,7 +384,7 @@ DESCRIPTION
To clear an already-set proxy, use "undef".
- When AnyEvent::HTTP is laoded for the first time it will query the
+ When AnyEvent::HTTP is loaded for the first time it will query the
default proxy from the operating system, currently by looking at
"$ENV{http_proxy"}.
@@ -386,21 +396,22 @@ DESCRIPTION
You should call this function (with a true $session_end) before you
save cookies to disk, and you should call this function after
loading them again. If you have a long-running program you can
- additonally call this function from time to time.
+ additionally call this function from time to time.
A cookie jar is initially an empty hash-reference that is managed by
- this module. It's format is subject to change, but currently it is
- like this:
+ this module. Its format is subject to change, but currently it is as
+ follows:
The key "version" has to contain 1, otherwise the hash gets emptied.
All other keys are hostnames or IP addresses pointing to
hash-references. The key for these inner hash references is the
server path for which this cookie is meant, and the values are again
- hash-references. The keys of those hash-references is the cookie
- name, and the value, you guessed it, is another hash-reference, this
- time with the key-value pairs from the cookie, except for "expires"
- and "max-age", which have been replaced by a "_expires" key that
- contains the cookie expiry timestamp.
+ hash-references. Each key of those hash-references is a cookie name,
+ and the value, you guessed it, is another hash-reference, this time
+ with the key-value pairs from the cookie, except for "expires" and
+ "max-age", which have been replaced by a "_expires" key that
+ contains the cookie expiry timestamp. Session cookies are indicated
+ by not having an "_expires" key.
Here is an example of a cookie jar with a single cookie, so you have
a chance of understanding the above paragraph:
@@ -440,7 +451,7 @@ DESCRIPTION
$AnyEvent::HTTP::MAX_PER_HOST
The maximum number of concurrent connections to the same host
- (identified by the hostname). If the limit is exceeded, then the
+ (identified by the hostname). If the limit is exceeded, then
additional requests are queued until previous connections are
closed. Both persistent and non-persistent connections are counted
in this limit.
@@ -449,12 +460,12 @@ DESCRIPTION
increase it much.
For comparison: the RFC's recommend 4 non-persistent or 2 persistent
- connections, older browsers used 2, newers (such as firefox 3)
+ connections, older browsers used 2, newer ones (such as firefox 3)
typically use 6, and Opera uses 8 because like, they have the
fastest browser and give a shit for everybody else on the planet.
$AnyEvent::HTTP::PERSISTENT_TIMEOUT
- The time after which idle persistent conenctions get closed by
+ The time after which idle persistent connections get closed by
AnyEvent::HTTP (default: 3).
$AnyEvent::HTTP::ACTIVE
@@ -464,7 +475,7 @@ DESCRIPTION
load-leveling.
SHOWCASE
- This section contaisn some more elaborate "real-world" examples or code
+ This section contains some more elaborate "real-world" examples or code
snippets.
HTTP/1.1 FILE DOWNLOAD
@@ -477,7 +488,7 @@ DESCRIPTION
re-download on older servers.
It calls the completion callback with either "undef", which means a
- nonretryable error occured, 0 when the download was partial and should
+ nonretryable error occurred, 0 when the download was partial and should
be retried, and 1 if it was successful.
use AnyEvent::HTTP;
@@ -625,6 +636,6 @@ AUTHOR
Marc Lehmann <schmorp@schmorp.de>
http://home.schmorp.de/
- With many thanks to Дмитрий Шалашов, who provided
- countless testcases and bugreports.
+ With many thanks to Дмитрий Шалашов, who provided countless testcases
+ and bugreports.
@@ -0,0 +1,34 @@
+BEGIN { $| = 1; print "1..23\n" }
+
+use AnyEvent::Impl::Perl;
+
+require AnyEvent::HTTP;
+
+print "ok 1\n";
+
+my $t = 2;
+
+for my $auth (qw(
+ 0.42.42.42
+ [0.42.42.42]:81
+ [::0.42.42.42]:81
+ [::0:2]
+ [0:0::2]:80
+ [::0:2]:81
+ [0:0::2]:81
+)) {
+ my $cv = AE::cv;
+
+ AnyEvent::HTTP::http_get ("http://$auth/", timeout => 1/128, sub {
+ print $_[1]{Status} == 599 ? "not ": "", "ok ", $t + 1, " # $_[1]{Status} $auth\n";
+ $cv->send;
+ });
+
+ print "ok ", $t, "\n";
+ $cv->recv;
+ print "ok ", $t + 2, "\n";
+
+ $t += 3;
+}
+
+print "ok 23\n";