@@ -118,8 +118,11 @@ Encode_XSEncoding(pTHX_ encode_t *enc)
{
dSP;
HV *stash = gv_stashpv("Encode::XS", TRUE);
- SV *sv = sv_bless(newRV_noinc(newSViv(PTR2IV(enc))),stash);
+ SV *iv = newSViv(PTR2IV(enc));
+ SV *sv = sv_bless(newRV_noinc(iv),stash);
int i = 0;
+ SvFLAGS(iv) |= SVp_POK;
+ SvPVX(iv) = enc->name[0];
PUSHMARK(sp);
XPUSHs(sv);
while (enc->name[i])
@@ -94,8 +94,11 @@ Encode_XSEncoding(pTHX_ encode_t *enc)
{
dSP;
HV *stash = gv_stashpv("Encode::XS", TRUE);
- SV *sv = sv_bless(newRV_noinc(newSViv(PTR2IV(enc))),stash);
+ SV *iv = newSViv(PTR2IV(enc));
+ SV *sv = sv_bless(newRV_noinc(iv),stash);
int i = 0;
+ SvFLAGS(iv) |= SVp_POK;
+ SvPVX(iv) = enc->name[0];
PUSHMARK(sp);
XPUSHs(sv);
while (enc->name[i])
@@ -1,8 +1,63 @@
# Revision history for Perl extension Encode.
#
-# $Id: Changes,v 2.55 2013/09/14 07:51:59 dankogai Exp dankogai $
+# $Id: Changes,v 2.59 2014/04/06 17:23:55 dankogai Exp $
#
-$Revision: 2.55 $ $Date: 2013/09/14 07:51:59 $
+$Revision: 2.59 $ $Date: 2014/04/06 17:23:55 $
+! Byte/Makefile.PL
+ CN/Makefile.PL
+ EBCDIC/Makefile.PL
+ Encode.pm
+ Encode.xs
+ Encode/Makefile_PL.e2x
+ JP/Makefile.PL
+ KR/Makefile.PL
+ Symbol/Makefile.PL
+ TW/Makefile.PL
+ bin/enc2xs
+ Restored the signature of Encode_XSEncoding() to address RT#94478
+ * While https://github.com/dankogai/p5-encode/pull/20
+ pulls the symnames via argument thus breaks the compatibility
+ with Encode::XX modules with *.ucm, the restored version
+ pulls the symanmes via enc->name[0] so the added 2nd argument
+ is no longer needed.
+ https://rt.cpan.org/Public/Bug/Display.html?id=94478
+
+2.58 2014/03/28 02:37:42
+! bin/piconv
+ Addressed: piconv bug of decoding UTF-16 (with fix)
+ https://github.com/dankogai/p5-encode/issues/19
+! Byte/Makefile.PL
+ CN/Makefile.PL
+ EBCDIC/Makefile.PL
+ Encode.pm
+ Encode.xs
+ Encode/Makefile_PL.e2x
+ JP/Makefile.PL
+ KR/Makefile.PL
+ Symbol/Makefile.PL
+ TW/Makefile.PL
+ bin/enc2xs
+ Pulled: Remap symname [RT #94221]
+ https://github.com/dankogai/p5-encode/pull/20
+ https://rt.cpan.org/Public/Bug/Display.html?id=94221
+! Encode.pm
+ Pulled: [doc] clarify that CHECK coderefs return octets #18
+ https://github.com/dankogai/p5-encode/pull/18
+
+2.57 2014/01/03 04:52:36
+! encengine.c
+ Pulled: sun compiler (maybe others) doesn't like UTF-8 in the source
+ https://github.com/dankogai/p5-encode/pull/17
+! bin/enc2xs
+ Merged RT#91763: POD errors
+ https://rt.cpan.org/Ticket/Display.html?id=91763
+
+2.56 2013/12/22 13:40:00
+! Encode.pm t/Encode.t
+ Merged RT#91569: [PATCH] decode_utf8 and non-PVs
+ https://rt.cpan.org/Ticket/Display.html?id=91569
+
+2.55 2013/09/14 07:51:59
! Encode.pm
Makefile.PL
Unicode/Unicode.pm
@@ -75,8 +75,11 @@ Encode_XSEncoding(pTHX_ encode_t *enc)
{
dSP;
HV *stash = gv_stashpv("Encode::XS", TRUE);
- SV *sv = sv_bless(newRV_noinc(newSViv(PTR2IV(enc))),stash);
+ SV *iv = newSViv(PTR2IV(enc));
+ SV *sv = sv_bless(newRV_noinc(iv),stash);
int i = 0;
+ SvFLAGS(iv) |= SVp_POK;
+ SvPVX(iv) = enc->name[0];
PUSHMARK(sp);
XPUSHs(sv);
while (enc->name[i])
@@ -111,8 +111,11 @@ Encode_XSEncoding(pTHX_ encode_t *enc)
{
dSP;
HV *stash = gv_stashpv("Encode::XS", TRUE);
- SV *sv = sv_bless(newRV_noinc(newSViv(PTR2IV(enc))),stash);
+ SV *iv = newSViv(PTR2IV(enc));
+ SV *sv = sv_bless(newRV_noinc(iv),stash);
int i = 0;
+ SvFLAGS(iv) |= SVp_POK;
+ SvPVX(iv) = enc->name[0];
PUSHMARK(sp);
XPUSHs(sv);
while (enc->name[i])
@@ -1,10 +1,10 @@
#
-# $Id: Encode.pm,v 2.55 2013/09/14 07:51:59 dankogai Exp dankogai $
+# $Id: Encode.pm,v 2.59 2014/04/06 17:30:41 dankogai Exp $
#
package Encode;
use strict;
use warnings;
-our $VERSION = sprintf "%d.%02d", q$Revision: 2.55 $ =~ /(\d+)/g;
+our $VERSION = sprintf "%d.%02d", q$Revision: 2.59 $ =~ /(\d+)/g;
use constant DEBUG => !!$ENV{PERL_ENCODE_DEBUG};
use XSLoader ();
XSLoader::load( __PACKAGE__, $VERSION );
@@ -209,7 +209,7 @@ my $utf8enc;
sub decode_utf8($;$) {
my ( $octets, $check ) = @_;
return undef unless defined $octets;
- $octets .= '' if ref $octets;
+ $octets .= '';
$check ||= 0;
$utf8enc ||= find_encoding('utf8');
my $string = $utf8enc->decode( $octets, $check );
@@ -801,13 +801,24 @@ If you're not interested in this, then bitwise-OR it with the bitmask.
=head2 coderef for CHECK
As of C<Encode> 2.12, C<CHECK> can also be a code reference which takes the
-ordinal value of the unmapped character as an argument and returns a string
-that represents the fallback character. For instance:
+ordinal value of the unmapped character as an argument and returns
+octets that represent the fallback character. For instance:
$ascii = encode("ascii", $utf8, sub{ sprintf "<U+%04X>", shift });
Acts like C<FB_PERLQQ> but U+I<XXXX> is used instead of C<\x{I<XXXX>}>.
+Even the fallback for C<decode> must return octets, which are
+then decoded with the character encoding that C<decode> accepts. So for
+example if you wish to decode octests as UTF-8, and use ISO-8859-15 as
+a fallback for bytes that are not valid UTF-8, you could write
+
+ $str = decode 'UTF-8', $octets, sub {
+ my $tmp = chr shift;
+ from_to $tmp, 'ISO-8859-15', 'UTF-8';
+ return $tmp;
+ };
+
=head1 Defining Encodings
To define a new encoding, use:
@@ -1,5 +1,5 @@
/*
- $Id: Encode.xs,v 2.24 2013/08/29 16:47:39 dankogai Exp $
+ $Id: Encode.xs,v 2.26 2014/04/06 17:23:55 dankogai Exp $
*/
#define PERL_NO_GET_CONTEXT
@@ -45,8 +45,11 @@ Encode_XSEncoding(pTHX_ encode_t * enc)
{
dSP;
HV *stash = gv_stashpv("Encode::XS", TRUE);
- SV *sv = sv_bless(newRV_noinc(newSViv(PTR2IV(enc))), stash);
+ SV *iv = newSViv(PTR2IV(enc));
+ SV *sv = sv_bless(newRV_noinc(iv),stash);
int i = 0;
+ SvFLAGS(iv) |= SVp_POK;
+ SvPVX(iv) = enc->name[0];
PUSHMARK(sp);
XPUSHs(sv);
while (enc->name[i]) {
@@ -94,8 +94,11 @@ Encode_XSEncoding(pTHX_ encode_t *enc)
{
dSP;
HV *stash = gv_stashpv("Encode::XS", TRUE);
- SV *sv = sv_bless(newRV_noinc(newSViv(PTR2IV(enc))),stash);
+ SV *iv = newSViv(PTR2IV(enc));
+ SV *sv = sv_bless(newRV_noinc(iv),stash);
int i = 0;
+ SvFLAGS(iv) |= SVp_POK;
+ SvPVX(iv) = enc->name[0];
PUSHMARK(sp);
XPUSHs(sv);
while (enc->name[i])
@@ -92,8 +92,11 @@ Encode_XSEncoding(pTHX_ encode_t *enc)
{
dSP;
HV *stash = gv_stashpv("Encode::XS", TRUE);
- SV *sv = sv_bless(newRV_noinc(newSViv(PTR2IV(enc))),stash);
+ SV *iv = newSViv(PTR2IV(enc));
+ SV *sv = sv_bless(newRV_noinc(iv),stash);
int i = 0;
+ SvFLAGS(iv) |= SVp_POK;
+ SvPVX(iv) = enc->name[0];
PUSHMARK(sp);
XPUSHs(sv);
while (enc->name[i])
@@ -4,7 +4,7 @@
"unknown"
],
"dynamic_config" : 1,
- "generated_by" : "ExtUtils::MakeMaker version 6.72, CPAN::Meta::Converter version 2.132140",
+ "generated_by" : "ExtUtils::MakeMaker version 6.94, CPAN::Meta::Converter version 2.140640",
"license" : [
"perl_5"
],
@@ -43,5 +43,5 @@
"url" : "https://github.com/dankogai/p5-encode"
}
},
- "version" : "2.55"
+ "version" : "2.59"
}
@@ -3,23 +3,23 @@ abstract: unknown
author:
- unknown
build_requires:
- ExtUtils::MakeMaker: 0
+ ExtUtils::MakeMaker: '0'
configure_requires:
- ExtUtils::MakeMaker: 0
+ ExtUtils::MakeMaker: '0'
dynamic_config: 1
-generated_by: 'ExtUtils::MakeMaker version 6.72, CPAN::Meta::Converter version 2.132140'
+generated_by: 'ExtUtils::MakeMaker version 6.94, CPAN::Meta::Converter version 2.140640'
license: perl
meta-spec:
url: http://module-build.sourceforge.net/META-spec-v1.4.html
- version: 1.4
+ version: '1.4'
name: Encode
no_index:
directory:
- t
- inc
requires:
- Exporter: 5.57
- parent: 0.221
+ Exporter: '5.57'
+ parent: '0.221'
resources:
repository: https://github.com/dankogai/p5-encode
-version: 2.55
+version: '2.59'
@@ -1,5 +1,5 @@
#
-# $Id: Makefile.PL,v 2.12 2013/09/14 07:51:59 dankogai Exp dankogai $
+# $Id: Makefile.PL,v 2.12 2013/09/14 07:51:59 dankogai Exp $
#
use 5.007003;
use strict;
@@ -80,8 +80,11 @@ Encode_XSEncoding(pTHX_ encode_t *enc)
{
dSP;
HV *stash = gv_stashpv("Encode::XS", TRUE);
- SV *sv = sv_bless(newRV_noinc(newSViv(PTR2IV(enc))),stash);
+ SV *iv = newSViv(PTR2IV(enc));
+ SV *sv = sv_bless(newRV_noinc(iv),stash);
int i = 0;
+ SvFLAGS(iv) |= SVp_POK;
+ SvPVX(iv) = enc->name[0];
PUSHMARK(sp);
XPUSHs(sv);
while (enc->name[i])
@@ -90,8 +90,11 @@ Encode_XSEncoding(pTHX_ encode_t *enc)
{
dSP;
HV *stash = gv_stashpv("Encode::XS", TRUE);
- SV *sv = sv_bless(newRV_noinc(newSViv(PTR2IV(enc))),stash);
+ SV *iv = newSViv(PTR2IV(enc));
+ SV *sv = sv_bless(newRV_noinc(iv),stash);
int i = 0;
+ SvFLAGS(iv) |= SVp_POK;
+ SvPVX(iv) = enc->name[0];
PUSHMARK(sp);
XPUSHs(sv);
while (enc->name[i])
@@ -10,7 +10,7 @@ use warnings;
use Getopt::Std;
use Config;
my @orig_ARGV = @ARGV;
-our $VERSION = do { my @r = (q$Revision: 2.8 $ =~ /\d+/g); sprintf "%d."."%02d" x $#r, @r };
+our $VERSION = do { my @r = (q$Revision: 2.11 $ =~ /\d+/g); sprintf "%d."."%02d" x $#r, @r };
# These may get re-ordered.
# RAW is a do_now as inserted by &enter
@@ -314,8 +314,11 @@ Encode_XSEncoding(pTHX_ encode_t *enc)
{
dSP;
HV *stash = gv_stashpv("Encode::XS", TRUE);
- SV *sv = sv_bless(newRV_noinc(newSViv(PTR2IV(enc))),stash);
+ SV *iv = newSViv(PTR2IV(enc));
+ SV *sv = sv_bless(newRV_noinc(iv),stash);
int i = 0;
+ SvFLAGS(iv) |= SVp_POK;
+ SvPVX(iv) = enc->name[0];
PUSHMARK(sp);
XPUSHs(sv);
while (enc->name[i])
@@ -1080,7 +1083,7 @@ add a new encoding, just read this chapter and forget the rest.
=over 4
-=item 0.
+=item 0.Z<>
Have a .ucm file ready. You can get it from somewhere or you can write
your own from scratch or you can grab one from the Encode distribution
@@ -1091,7 +1094,7 @@ in I<my.ucm>. C<$> is a shell prompt.
$ ls -F
my.ucm
-=item 1.
+=item 1.Z<>
Issue a command as follows;
@@ -1114,7 +1117,7 @@ The following files were created.
=over 4
-=item 1.1.
+=item 1.1.Z<>
If you want *.ucm installed together with the modules, do as follows;
@@ -1124,20 +1127,20 @@ If you want *.ucm installed together with the modules, do as follows;
=back
-=item 2.
+=item 2.Z<>
Edit the files generated. You don't have to if you have no time AND no
intention to give it to someone else. But it is a good idea to edit
the pod and to add more tests.
-=item 3.
+=item 3.Z<>
Now issue a command all Perl Mongers love:
$ perl Makefile.PL
Writing Makefile for Encode::My
-=item 4.
+=item 4.Z<>
Now all you have to do is make.
@@ -1158,7 +1161,7 @@ The time it takes varies depending on how fast your machine is and
how large your encoding is. Unless you are working on something big
like euc-tw, it won't take too long.
-=item 5.
+=item 5.Z<>
You can "make install" already but you should test first.
@@ -1171,11 +1174,11 @@ You can "make install" already but you should test first.
Files=1, Tests=2, 0 wallclock secs
( 0.09 cusr + 0.01 csys = 0.09 CPU)
-=item 6.
+=item 6.Z<>
If you are content with the test result, just "make install"
-=item 7.
+=item 7.Z<>
If you want to add your encoding to Encode's demand-loading list
(so you don't have to "use Encode::YourEncoding"), run
@@ -1,5 +1,5 @@
#!./perl
-# $Id: piconv,v 2.5 2013/04/26 18:30:46 dankogai Exp $
+# $Id: piconv,v 2.6 2014/03/28 02:37:42 dankogai Exp $
#
use 5.8.0;
use strict;
@@ -68,14 +68,16 @@ To: $to => $cto
EOT
}
-my %use_bom = map { $_ => 1 } qw/UTF-16 UTF-32/;
+my %use_bom =
+ map { $_ => 1 } qw/UTF-16 UTF-16BE UTF-16LE UTF-32 UTF-32BE UTF-32LE/;
# we do not use <> (or ARGV) for the sake of binmode()
@ARGV or push @ARGV, \*STDIN;
unless ( $scheme eq 'perlio' ) {
binmode STDOUT;
- my $need2slurp = $use_bom{ find_encoding($to)->name };
+ my $need2slurp = $use_bom{ find_encoding($to)->name }
+ || $use_bom{ find_encoding($from)->name };
for my $argv (@ARGV) {
my $ifh = ref $argv ? $argv : undef;
$ifh or open $ifh, "<", $argv or warn "Can't open $argv: $!" and next;
@@ -169,7 +171,7 @@ The following are mainly of interest to Encode hackers:
-D,--debug show debug information
-S,--scheme scheme use the scheme for conversion
Those are handy when you can only see ASCII characters:
- -p,--perlqq transliterate characters missing in encoding to \x{HHHH}
+ -p,--perlqq transliterate characters missing in encoding to \\x{HHHH}
where HHHH is the hexadecimal Unicode code point
--htmlcref transliterate characters missing in encoding to &#NNN;
where NNN is the decimal Unicode code point
@@ -81,7 +81,7 @@ This scheme can also handle shift encodings.
A slight enhancement to the scheme also allows for look-ahead - if
we add a flag to re-add the removed byte to the source we could handle
- a" -> รค
+ a" -> U+00E4 (LATIN SMALL LETTER A WITH DIAERESIS)
ab -> a (and take b back please)
*/
@@ -1,5 +1,5 @@
#
-# $Id: Encoder.pm,v 2.3 2013/09/14 07:51:59 dankogai Exp dankogai $
+# $Id: Encoder.pm,v 2.3 2013/09/14 07:51:59 dankogai Exp $
#
package Encode::Encoder;
use strict;
@@ -1,5 +1,5 @@
#
-# $Id: GSM0338.pm,v 2.5 2013/09/14 07:51:59 dankogai Exp dankogai $
+# $Id: GSM0338.pm,v 2.5 2013/09/14 07:51:59 dankogai Exp $
#
package Encode::GSM0338;
@@ -1,5 +1,5 @@
#
-# $Id: UTF7.pm,v 2.8 2013/09/14 07:51:59 dankogai Exp dankogai $
+# $Id: UTF7.pm,v 2.8 2013/09/14 07:51:59 dankogai Exp $
#
package Encode::Unicode::UTF7;
use strict;
@@ -25,7 +25,7 @@ my @character_set = ('0'..'9', 'A'..'Z', 'a'..'z');
my @source = qw(ascii iso8859-1 cp1250);
my @destiny = qw(cp1047 cp37 posix-bc);
my @ebcdic_sets = qw(cp1047 cp37 posix-bc);
-plan test => 38+$n*@encodings + 2*@source*@destiny*@character_set + 2*@ebcdic_sets*256 + 6 + 4;
+plan test => 38+$n*@encodings + 2*@source*@destiny*@character_set + 2*@ebcdic_sets*256 + 6 + 5;
my $str = join('',map(chr($_),0x20..0x7E));
my $cpy = $str;
ok(length($str),from_to($cpy,'iso8859-1','Unicode'),"Length Wrong");
@@ -150,6 +150,10 @@ package main;
ok(decode(latin1 => Encode::Dummy->new("foobar")), "foobar");
ok(encode(utf8 => Encode::Dummy->new("foobar")), "foobar");
+# RT#91569
+# decode_utf8 with non-string arguments
+ok(decode_utf8(*1), "*main::1");
+
# hash keys
my $key = (keys %{{ "whatever\x{100}" => '' }})[0];
my $kopy = $key;
@@ -1,5 +1,5 @@
#
-# $Id: Encoder.t,v 2.1 2013/09/14 07:51:59 dankogai Exp dankogai $
+# $Id: Encoder.t,v 2.1 2013/09/14 07:51:59 dankogai Exp $
#
BEGIN {