@@ -1,6 +1,75 @@
+ -*-coding:utf-8;fill-column:79-*- ············································
+ Last Modified Time-stamp: "2014-12-07 05:39:39 MST sburke@cpan.org"
+#==============================================================================
+
Revision history for Perl module Text::Unidecode
- Time-stamp: "2001-07-14 02:28:13 MDT"
+2014-12-07 Sean M. Burke sburke@cpan.org
+ * RELEASE 1.23. Just a bugfix version.
+ * The bug in question: https://rt.cpan.org/Ticket/Display.html?id=97456
+ * Thank you very much to superstar Dagfinn Ilmari Mannsaker for noting
+ it first *and* for providing a patch for a problem that would baffle
+ me completely:
+ "On perls 5.8.8 through 5.12.x, regex matches against UTF-16
+ surrogate characters emits a fatal "Malformed UTF-8 character"
+ warning if warnings are enabled. ExtUtils::MakeMaker prior to 6.78
+ runs the test suite with -w, causing the installation to fail.
+ The attached patch [which I applied -SMB] disables utf8
+ warnings while doing the regex substitution and converting the
+ character number to a character in the test."
+ And thank you very much to Ricardo Signes and Tim Bunce for reminding
+ me to actually release this thang! I was stupid and forgot... for
+ several MONTHS.
+ * Doc: Adding mention of Tom Christiansen's "Perl Unicode Cookbook":
+ http://www.perl.com/pub/2012/04/perlunicook-standard-preamble.html
+ * Doc: Adding a suggestion of "use utf8;" in German example.
+
+2014-08-15 Sean M. Burke sburke@cpan.org
+ * RELEASE 1.22. (The dev release works, so this is a version bump.)
+ * See notes for 2014-07-25, because this is the first public release
+ with significant changes since 2001!
+
+2014-07-25 Sean M. Burke sburke@cpan.org
+ * !DEVELOPER RELEASE!
+ * !Release 1.20_01!
+ * Many bugfixes. Thanks especially to Tomaž Šolc!
+ * Yet more *.t files added for improved sanity checking.
+ * Shuffling around the internals of Unidecode.pm
+ * Putting in some vacuous 0x__.pm files where
+ previously there would just be a load failure
+
+2014-06-30 Sean M. Burke sburke@cpan.org
+ * Release 1.01 -- first official Unidecode release since 2001!!!
+ * There are no real changes since the 2014-06-23 developer
+ release. I'm just making this all official now.
+
+
+2014-06-23 Sean M. Burke sburke@cpan.org
+ * !DEVELOPER RELEASE!
+ * Release 1.00_03
+ * Now asserting that we need at least Perl 5.8.0
+ An automated test system that tried running the t/*.t
+ under a 5.6.2 spewed all kinds of crazy error messages.
+ Hence the bump-up.
+ So, I added assertions for the version.
+ * I added some tests for more basic sanity assertions.
+
+2014-06-17 Sean M. Burke sburke@cpan.org
+ v1.00_02 - Not released. Just internal rearranging.
+
+
+2014-06-13 Sean M. Burke sburke@cpan.org
+ * !DEVELOPER RELEASE!
+ * Release 1.00(_01!)- so many years later, finally we bump up to 1.*!
+
+ * My documentation is now BRILLIANT.
+ * Minor bugfixes.
+ * Some code comments for clarity.
+ * A modern test suite.
+
+ * A proper release will follow in a few days.
+
+
2001-07-14 Sean M. Burke sburke@cpan.org
* Release 0.04 -- forgot to put TODO.txt in 0.03. Now including
@@ -0,0 +1,7 @@
+
+ [Last Modified Time-stamp: "2014-01-25 15:44:56 MST sburke@cpan.org"]
+
+
+This module, Text::Unidecode (along with its documentation and its
+data tables) is distributed under the same terms as Perl itself.
+
@@ -1,9 +1,4 @@
ChangeLog
-MANIFEST
-MANIFEST.SKIP
-Makefile.PL
-README
-TODO.txt
lib/Text/Unidecode.pm
lib/Text/Unidecode/x00.pm
lib/Text/Unidecode/x01.pm
@@ -13,6 +8,7 @@ lib/Text/Unidecode/x04.pm
lib/Text/Unidecode/x05.pm
lib/Text/Unidecode/x06.pm
lib/Text/Unidecode/x07.pm
+lib/Text/Unidecode/x08.pm
lib/Text/Unidecode/x09.pm
lib/Text/Unidecode/x0a.pm
lib/Text/Unidecode/x0b.pm
@@ -29,6 +25,11 @@ lib/Text/Unidecode/x15.pm
lib/Text/Unidecode/x16.pm
lib/Text/Unidecode/x17.pm
lib/Text/Unidecode/x18.pm
+lib/Text/Unidecode/x19.pm
+lib/Text/Unidecode/x1a.pm
+lib/Text/Unidecode/x1b.pm
+lib/Text/Unidecode/x1c.pm
+lib/Text/Unidecode/x1d.pm
lib/Text/Unidecode/x1e.pm
lib/Text/Unidecode/x1f.pm
lib/Text/Unidecode/x20.pm
@@ -40,12 +41,42 @@ lib/Text/Unidecode/x25.pm
lib/Text/Unidecode/x26.pm
lib/Text/Unidecode/x27.pm
lib/Text/Unidecode/x28.pm
+lib/Text/Unidecode/x29.pm
+lib/Text/Unidecode/x2a.pm
+lib/Text/Unidecode/x2b.pm
+lib/Text/Unidecode/x2c.pm
+lib/Text/Unidecode/x2d.pm
lib/Text/Unidecode/x2e.pm
lib/Text/Unidecode/x2f.pm
lib/Text/Unidecode/x30.pm
lib/Text/Unidecode/x31.pm
lib/Text/Unidecode/x32.pm
lib/Text/Unidecode/x33.pm
+lib/Text/Unidecode/x34.pm
+lib/Text/Unidecode/x35.pm
+lib/Text/Unidecode/x36.pm
+lib/Text/Unidecode/x37.pm
+lib/Text/Unidecode/x38.pm
+lib/Text/Unidecode/x39.pm
+lib/Text/Unidecode/x3a.pm
+lib/Text/Unidecode/x3b.pm
+lib/Text/Unidecode/x3c.pm
+lib/Text/Unidecode/x3d.pm
+lib/Text/Unidecode/x3e.pm
+lib/Text/Unidecode/x3f.pm
+lib/Text/Unidecode/x40.pm
+lib/Text/Unidecode/x41.pm
+lib/Text/Unidecode/x42.pm
+lib/Text/Unidecode/x43.pm
+lib/Text/Unidecode/x44.pm
+lib/Text/Unidecode/x45.pm
+lib/Text/Unidecode/x46.pm
+lib/Text/Unidecode/x47.pm
+lib/Text/Unidecode/x48.pm
+lib/Text/Unidecode/x49.pm
+lib/Text/Unidecode/x4a.pm
+lib/Text/Unidecode/x4b.pm
+lib/Text/Unidecode/x4c.pm
lib/Text/Unidecode/x4d.pm
lib/Text/Unidecode/x4e.pm
lib/Text/Unidecode/x4f.pm
@@ -134,6 +165,13 @@ lib/Text/Unidecode/xa1.pm
lib/Text/Unidecode/xa2.pm
lib/Text/Unidecode/xa3.pm
lib/Text/Unidecode/xa4.pm
+lib/Text/Unidecode/xa5.pm
+lib/Text/Unidecode/xa6.pm
+lib/Text/Unidecode/xa7.pm
+lib/Text/Unidecode/xa8.pm
+lib/Text/Unidecode/xa9.pm
+lib/Text/Unidecode/xaa.pm
+lib/Text/Unidecode/xab.pm
lib/Text/Unidecode/xac.pm
lib/Text/Unidecode/xad.pm
lib/Text/Unidecode/xae.pm
@@ -178,6 +216,39 @@ lib/Text/Unidecode/xd4.pm
lib/Text/Unidecode/xd5.pm
lib/Text/Unidecode/xd6.pm
lib/Text/Unidecode/xd7.pm
+lib/Text/Unidecode/xd8.pm
+lib/Text/Unidecode/xd9.pm
+lib/Text/Unidecode/xda.pm
+lib/Text/Unidecode/xdb.pm
+lib/Text/Unidecode/xdc.pm
+lib/Text/Unidecode/xdd.pm
+lib/Text/Unidecode/xde.pm
+lib/Text/Unidecode/xdf.pm
+lib/Text/Unidecode/xe0.pm
+lib/Text/Unidecode/xe1.pm
+lib/Text/Unidecode/xe2.pm
+lib/Text/Unidecode/xe3.pm
+lib/Text/Unidecode/xe4.pm
+lib/Text/Unidecode/xe5.pm
+lib/Text/Unidecode/xe6.pm
+lib/Text/Unidecode/xe7.pm
+lib/Text/Unidecode/xe8.pm
+lib/Text/Unidecode/xe9.pm
+lib/Text/Unidecode/xea.pm
+lib/Text/Unidecode/xeb.pm
+lib/Text/Unidecode/xec.pm
+lib/Text/Unidecode/xed.pm
+lib/Text/Unidecode/xee.pm
+lib/Text/Unidecode/xef.pm
+lib/Text/Unidecode/xf0.pm
+lib/Text/Unidecode/xf1.pm
+lib/Text/Unidecode/xf2.pm
+lib/Text/Unidecode/xf3.pm
+lib/Text/Unidecode/xf4.pm
+lib/Text/Unidecode/xf5.pm
+lib/Text/Unidecode/xf6.pm
+lib/Text/Unidecode/xf7.pm
+lib/Text/Unidecode/xf8.pm
lib/Text/Unidecode/xf9.pm
lib/Text/Unidecode/xfa.pm
lib/Text/Unidecode/xfb.pm
@@ -185,4 +256,28 @@ lib/Text/Unidecode/xfc.pm
lib/Text/Unidecode/xfd.pm
lib/Text/Unidecode/xfe.pm
lib/Text/Unidecode/xff.pm
-test.pl
+LICENSE
+Makefile.PL
+MANIFEST
+MANIFEST.SKIP
+README
+t/00010_hello_world.t
+t/00050_assert_perl_version.t
+t/00055_can_utf8.t
+t/00100_can_binmode.t
+t/00200_xbrace_syntax.t
+t/00300_wideness.t
+t/00400_just_load_module.t
+t/00500_test_loading.t
+t/00510_load_many_tables.t
+t/01000_note_incs.t
+t/01500_generate_no_warnings.t
+t/02000_uniform_table_sizes.t
+t/03000_misc_unihan.t
+t/04000_misc_errors_fixed.t
+t/04010_misc_errors_fixed.t
+t/05000_main.t
+t/09900_bye.t
+TODO.txt
+META.yml Module YAML meta-data (added by MakeMaker)
+META.json Module JSON meta-data (added by MakeMaker)
@@ -1,3 +1,17 @@
+#
+# Last Modified Time-stamp: "2014-08-15 04:48:23 MDT sburke@cpan.org"
+#
+make_unidecode_dist.*
+make_manifest
+\.bak$
+\.old$
Makefile$
~$
.tar.gz$
+~
+#
+CACHEDIR.TAG
+_*00_ABOUT_THIS_DIR.txt
+MYMETA.*
+SMB_.*
+/[0-9]{1,3}$
@@ -0,0 +1,41 @@
+{
+ "abstract" : "Provide plain ASCII transliterations of Unicode text",
+ "author" : [
+ "Sean Burke <sburke@cpan.org>"
+ ],
+ "dynamic_config" : 1,
+ "generated_by" : "ExtUtils::MakeMaker version 6.62, CPAN::Meta::Converter version 2.112150",
+ "license" : [
+ "perl_5"
+ ],
+ "meta-spec" : {
+ "url" : "http://search.cpan.org/perldoc?CPAN::Meta::Spec",
+ "version" : "2"
+ },
+ "name" : "Text-Unidecode",
+ "no_index" : {
+ "directory" : [
+ "t",
+ "inc"
+ ]
+ },
+ "prereqs" : {
+ "build" : {
+ "requires" : {
+ "ExtUtils::MakeMaker" : 0
+ }
+ },
+ "configure" : {
+ "requires" : {
+ "ExtUtils::MakeMaker" : 0
+ }
+ },
+ "runtime" : {
+ "requires" : {
+ "perl" : "5.008"
+ }
+ }
+ },
+ "release_status" : "stable",
+ "version" : "1.23"
+}
@@ -0,0 +1,22 @@
+---
+abstract: 'Provide plain ASCII transliterations of Unicode text'
+author:
+ - 'Sean Burke <sburke@cpan.org>'
+build_requires:
+ ExtUtils::MakeMaker: 0
+configure_requires:
+ ExtUtils::MakeMaker: 0
+dynamic_config: 1
+generated_by: 'ExtUtils::MakeMaker version 6.62, CPAN::Meta::Converter version 2.112150'
+license: perl
+meta-spec:
+ url: http://module-build.sourceforge.net/META-spec-v1.4.html
+ version: 1.4
+name: Text-Unidecode
+no_index:
+ directory:
+ - t
+ - inc
+requires:
+ perl: 5.008
+version: 1.23
@@ -1,20 +1,56 @@
-require 5.006;
+# This -*-perl-*- script writes the Makefile for installing this distribution.
+# ( Last Modified Time-stamp: "2014-07-22 16:46:15 MDT sburke@cpan.org" )
+#======================================================================
+#
+# For info on instaling this module and how to control it, see:
+# perldoc cpan
+# perldoc cpanp
+# perldoc perlmodinstall
+# perldoc ExtUtils::MakeMaker
+#
+#
+#======================================================================
+
+require 5.008_000; # # <==========!!!!!!!!!!
+my $min_perl_version = 5.008_000; # # <==========!!!!!!!!!!
+
+# For the module author: see lib/ExtUtils/MakeMaker.pm for details
+# of how to influence the contents of the Makefile that is written.
+# Incidentally. Perldoc perlfunc says "use " is compile-time, but
+# "require 5.6" is runtime.
+
use ExtUtils::MakeMaker;
-# See lib/ExtUtils/MakeMaker.pm for details of how to influence
-# the contents of the Makefile that is written.
+#if( $ExtUtils::MakeMaker::VERSION < 6.47 ) {
+# die "I need at least perl version $min_perl_version";
+#}
+
WriteMakefile(
- 'NAME' => 'Text::Unidecode',
- 'VERSION_FROM' => 'lib/Text/Unidecode.pm', # finds $VERSION
- 'dist' => { COMPRESS => 'gzip -9f', SUFFIX => 'gz', },
-);
+ 'NAME' => 'Text::Unidecode',
+ 'ABSTRACT' => 'Provide plain ASCII transliterations of Unicode text',
-package MY;
+ 'VERSION_FROM' => 'lib/Text/Unidecode.pm', # finds $VERSION
-sub libscan
-{ # Determine things that should *not* be installed
- my($self, $path) = @_;
- return '' if $path =~ m/~/;
- $path;
-}
+ 'AUTHOR' => 'Sean Burke <sburke@cpan.org>',
+ 'LICENSE' => 'perl',
+ # "...under the same terms as Perl itself"
+
+ ($ExtUtils::MakeMaker::VERSION >= 6.47
+ ? ('MIN_PERL_VERSION' => $min_perl_version )
+
+ : () # Otherwise we can't express that metadata about this
+ # whole module suite, and we just have to let the version
+ # "require" thing up there block this whole Makefile.PL
+ # from running.
+ ),
+
+ 'dist' => {
+ 'COMPRESS' => 'gzip -4f',
+ # from 4 to 6 is where there's a sudden doubling
+ # of processing time, So 4 is cool.
+ 'SUFFIX' => 'gz',
+ 'TARFLAGS' => 'cfh', # h=follow symlinks when making dist
+ },
+
+);
__END__
@@ -1,98 +1,94 @@
-README for Text::Unidecode
- Time-stamp: "2001-07-14 02:03:33 MDT"
+ # -*- coding:utf-8; mode:Text; fill-column:79 -*-
+ # Time-stamp: "2014-06-17 17:23:26 MDT sburke@cpan.org"
+# (This page is in UTF-8!) |··················································|
- Text::Unidecode
+Module: Text::Unidecode-- make ASCII transliterations of Unicode text
-NAME
- Text::Unidecode -- US-ASCII transliterations of Unicode text
+Unidecode makes ASCII transliterations of Unicode text. Sometimes it's dumb,
+but it's better than looking at "???" or "\15BA\15A0\1610...".
+If you have smarter text-handling subroutines, Unidecode might be useful
+as a fallthrough for them.
-SYNOPSIS
- use utf8;
- use Text::Unidecode;
- print unidecode(
- "\x{5317}\x{4EB0}\n"
- # those are the Chinese characters for Beijing
- );
-
- # That prints: Bei Jing
+Example:
+ print unidecode( "北亰\n" );
+ prints: Bei Jing
-DESCRIPTION
-
- It often happens that you have non-Roman text data in Unicode, but
- you can't display it -- usually because you're trying to show it
- to a user via an application that doesn't support Unicode, or
- because the fonts you need aren't accessible. You could represent
- the Unicode characters as "???????" or "\15BA\15A0\1610...", but
- that's nearly useless to the user who actually wants to read what
- the text says.
-
- What Text::Unidecode provides is a function, `unidecode(...)' that
- takes Unicode data and tries to represent it in US-ASCII
- characters (i.e., the universally displayable characters between
- 0x00 and 0x7F). The representation is almost always an attempt at
- *transliteration* -- i.e., conveying, in Roman letters, the
- pronunciation expressed by the text in some other writing
- system. (See the example in the synopsis.)
-
-
-See the POD for more information.
+See more examples below.
+For full documentation, run:
+ perldoc Unidecode
+Or read:see:
+ http://search.cpan.org/perldoc?Text::Unidecode
+An article about how Unidecode runs:
+ http://interglacial.com/tpj/22/
REQUIREMENTS
-
-This module requires Perl 5.6.0 or higher. (Earlier Perls apparently
-lack the "use utf8" pragma that Text::Unidecode needs.)
-I have also observed that 5.6.0's Unicode support is shakey; strongly
-consider upgrading to 5.6.1 at least.
-
-
+This module requires Perl 5.8.0 at the very least. That's probably not a
+problem for you, since that's from a decade ago!
INSTALLATION
+* For using the "CPAN Plus" system, read: perldoc cpanp
+* For old-style "make" interface, read: perldoc perlmodinstall
+
+~~~ EXAMPLE UNIDECODE INPUT AND OUTPUT ~~~
+(Just two or three lines, from a few languages.)
+
+La décennie voit le début des biotechnologies avec le premier clonage,
+les organismes génétiquement modifiés, le début du séquençage du
+génome humain
+=> La decennie voit le debut des biotechnologies avec le premier clonage,
+ les organismes genetiquement modifies, le debut du sequencage du
+ genome humain
+
+Wśród nocnej ciszy głos się rozchodzi:
+Wstańcie, pasterze, Bóg się nam rodzi!
+=> Wsrod nocnej ciszy glos sie rozchodzi:
+ Wstancie, pasterze, Bog sie nam rodzi!
+
+Καθαίρονται δ᾽ ἄλλως αἵματι μιαινόμενοι οἷον εἴ τις εἰς πηλὸν ἐμβὰς
+πηλῷ ἀπονίζοντο. μαίνεσθαι δ᾽ ἂν δοκοίη, εἴ τίς μιν ἀνθρώπων
+=> Kathairontai d' allos aimati miainomenoi oion ei tis eis pelon embas
+ pelo aponizonto. mainesthai d' an dokoie, ei tis min anthropon
+
+На другой день к завтраку подавали очень вкусные пирожки, раков и
+бараньи котлеты; и пока ели, приходил наверх повар Никанор справиться,
+=> Na drughoi dien' k zavtraku podavali ochien' vkusnyie pirozhki, rakov i
+ baran'i kotliety; i poka ieli, prikhodil navierkh povar Nikanor spravit'sia,
+
+Nước trà (hay nước chè) là đồ uống phổ biến thứ hai trên thế giới (sau
+nước uống). Nó làm bằng cách ngâm lá, chồi, hay cành của cây chè
+=> Nuoc tra (hay nuoc che) la do uong pho bien thu hai tren the gioi (sau
+ nuoc uong). No lam bang cach ngam la, choi, hay canh cua cay che
+
+#### And Then Things Get A Bit Suboptimal
+# But remember the Unidecode motto: "It's better than nothing!"
+
+유자차(柚子茶)는 유자청을 찬물이나 더운 물에 희석하여 마시는 한국의
+전통 차이다. 유자청은 얇게 자른 유자를 꿀이나 설탕과 섞은 뒤 3~4개월
+=> yujaca(You Zi Cha )neun yujaceongeul canmulina deoun mule hyiseoghayeo masineun hangugyi
+ jeontong caida. yujaceongeun yalbge jareun yujareul ggulina seoltanggwa seoggeun dwi 3~4gaeweol
+
+* The Gayatri Mantra- Sanskrit
+ॐ भूर्भुवः॒ स्वः । तत्स॑वितुर्वरे॑णियं ।
+भ॒र्गो॑ दे॒वस्य॑ धीमहि। । धियो॒ यो नः॑ प्रचो॒दया॑त्॥ ।
+=> AUM bhuurbhuvH' svH / tts'viturvre'nniyN /
+ bh'rgo' de'vsy' dhiimhi / / dhiyo' yo nH' prco'dyaa't // /
+
+道可道，非常道。名可名，非常名。無名天地之始；有名萬物之母。故常無欲，
+以觀其妙；常有欲，以觀其徼。此兩者，同出而異名，同謂之玄。玄之又玄，衆
+=> Dao Ke Dao ,Fei Chang Dao . Ming Ke Ming ,Fei Chang Ming . Wu Ming Tian Di Zhi Shi ;You Ming Wan Wu Zhi Mu . Gu Chang Wu Yu ,
+ Yi Guan Qi Miao ;Chang You Yu ,Yi Guan Qi Jiao . Ci Liang Zhe ,Tong Chu Er Yi Ming ,Tong Wei Zhi Xuan . Xuan Zhi You Xuan ,Zhong
+
+#Yiddish. Directionality and ligature might come out wrong in your browser/editor:]
+טיי איז א געטראנק וואס מען טרינקט איבער דער גארער וועלט. טיי ווערט
+געמאכט דורך ווייקן די געטרוקנטע בלעטער אדער בלומען פון דעם פלאנץ
+=> tyy yz g`trnq vvs m`n trynqt yb`r d`r gr`r vv`lt. tyy vv`rt
+ g`mkt dvrk vvyyqn dy g`trvqnt` bl`t`r d`r blvm`n pvn d`m plnts
+
+
+#Urdu. Directionality and ligature might come out wrong in your browser/editor:]
+چائے دنیا کی پسندیدہ مشروب ہے۔ یہ چاۓ کے پودے کی پتیوں کو چند منٹ گرم
+پانی میں ابالنے سے تیار ہوتی ہے۔
+=> chy'y dny khy psndydh mshrwb hy. yh chy' khy pwdy khy ptywN khw chnd mntt grm
+ pny myN blny sy tyr hwty hy.
-You install Text::Unidecode, as you would install any perl module
-library, by running these commands:
-
- perl Makefile.PL
- make
- make test
- make install
-
-If you want to install a private copy of Text::Unidecode in your home
-directory, then you should try to produce the initial Makefile with
-something like this command:
-
- perl Makefile.PL LIB=~/perl
-
-See perldoc perlmodinstall for more information on installing modules.
-
-
-DOCUMENTATION
-
-POD-format documentation is included in Unidecode.pm. POD is readable
-with the 'perldoc' utility. See ChangeLog for recent changes.
-
-
-SUPPORT
-
-Questions, bug reports, useful code bits, and suggestions for
-Text::Unidecode should just be sent to me at sburke@cpan.org
-
-
-AVAILABILITY
-
-The latest version of Text::Unidecode is available from the
-Comprehensive Perl Archive Network (CPAN). Visit
-<http://www.perl.com/CPAN/> to find a CPAN site near you.
-
-
-COPYRIGHT
-
-Copyright 2001, Sean M. Burke <sburke@cpan.org>, all rights reserved.
-
-The programs and documentation in this dist are distributed in the
-hope that they will be useful, but without any warranty; without even
-the implied warranty of merchantability or fitness for a particular
-purpose.
-
-This library is free software; you can redistribute it and/or modify
-it under the same terms as Perl itself.
@@ -1,10 +1,59 @@
-Text::Unidecode Transliteration Notes -- TODO FILE
+# -*-coding:utf-8;-*- ··················································
+# Last Modified Time-stamp: "2014-11-10 17:30:04 MST sburke@cpan.org"
+#======================================================================
+ ~~ Text::Unidecode TODO file ~~
+TODO GTD:
+Keep plugging in suggestions from Tomaž Šolc's message
-============================== BLOCK 03 ==============================
-Coptic things I'm unsure of: Khei=kh, Gangia=g, Shima=ch, Dei=ti.
+TODO:
+Figure out how to courteously declare dependencies in my Makefile.PL
+
+TODO:
+Bundle with a "unidecode" util that calls a routine in Unidecode.pm?
+
+TODO:
+Plow through more bug reports, applying patches etc,
+especially stuff from that nice Tomaž Šolc man.
+
+TODO:
+
+Populate 0x80-0x9F with their values from Win-1252.
+
+TODO:
+
+Make it handle stuff in Astral Plane (over U+FFFF)
+
+TODO:
+
+THEN re-run the thing that reads the Unihan database and generates
+all the Text/Unidecode/__.pm files.
+(Do the hyperspace handling first, because lots of Unihan
+stuff is up there.)
+
+TODO:
+
+Look to see whether I need to deal with the Arabic stuff in
+the U+FBxx etc blocks. I thought it was always just
+font-internal stuff, but I'm starting to suspect that it may
+be encountered in the real world
+
+TODO:
+
+Of course:
+- Check for new glyphs in existing tables.
+- Look at whole new tables (like the Philippine scripts) in
+normal space (x < U+FFFF)
+- Look at the wild wilderness in hyperspace (x > U+FFFF
+
+
+======================================================================
+======================================================================
+
+########################################################################
+Below here is TODOs from the Unicode version in 2001. Yes, that long ago.
============================== BLOCK 09 ==============================
@@ -98,6 +147,5 @@ Arabic Presentation Forms-B (FE70-FEFF) -- do I need to
do these, or are they never actually found in text files?
-
---
-26 TODOs at Sat Jul 14 00:27:44 2001
+======================================================================
+(end)
@@ -1,20 +1,55 @@
-# Time-stamp: "Sat Jul 14 00:27:20 2001 by Automatic Bizooty (__blocks2pm.plx)"
+# Time-stamp: "2014-07-25 02:57:26 MDT sburke@cpan.org"
$Text::Unidecode::Char[0x00] = [
+# BLOCK U+0000
+
qq{\x00}, qq{\x01}, qq{\x02}, qq{\x03}, qq{\x04}, qq{\x05}, qq{\x06}, qq{\x07}, qq{\x08}, qq{\x09}, qq{\x0a}, qq{\x0b}, qq{\x0c}, qq{\x0d}, qq{\x0e}, qq{\x0f},
+#^00 ^01 ^02 ^03 ^04 ^05 ^06 ^07 ^08 ^09 ^0a ^0b ^0c ^0d ^0e ^0f
+
qq{\x10}, qq{\x11}, qq{\x12}, qq{\x13}, qq{\x14}, qq{\x15}, qq{\x16}, qq{\x17}, qq{\x18}, qq{\x19}, qq{\x1a}, qq{\x1b}, qq{\x1c}, qq{\x1d}, qq{\x1e}, qq{\x1f},
+#^10 ^11 ^12 ^13 ^14 ^15 ^16 ^17 ^18 ^19 ^1a ^1b ^1c ^1d ^1e ^1f
+
' ', qq{!}, qq{"}, qq{#}, qq{\$}, qq{%}, qq{&}, qq{'}, qq{(}, qq{)}, qq{*}, qq{+}, qq{,}, qq{-}, qq{.}, qq{/},
+#^20 ^21 ^22 ^23 ^24 ^25 ^26 ^27 ^28 ^29 ^2a ^2b ^2c ^2d ^2e ^2f ^30
+
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', qq{:}, qq{;}, qq{<}, qq{=}, qq{>}, qq{?},
+#^30 ^31 ^32 ^33 ^34 ^35 ^36 ^37 ^38 ^39 ^3a ^3b ^3c ^3d ^3e ^3f
+
qq{\@}, 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O',
-'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', qq{]}, qq{\\}, qq{]}, qq{^}, qq{_},
+#^40 ^41 ^42 ^43 ^44 ^45 ^46 ^47 ^48 ^49 ^4a ^4b ^4c ^4d ^4e ^4f
+
+'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', qq{[}, qq{\\}, qq{]}, qq{^}, qq{_},
+#^50 ^51 ^52 ^53 ^54 ^55 ^56 ^57 ^58 ^59 ^5a ^5b ^5c ^5d ^5e ^5f
+
qq{`}, 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o',
+#^60 ^61 ^62 ^63 ^64 ^65 ^66 ^67 ^68 ^69 ^6a ^6b ^6c ^6d ^6e ^6f
+
'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', qq{\{}, qq{|}, qq{\}}, qq{~}, qq{\x7f},
+#^70 ^71 ^72 ^73 ^74 ^75 ^76 ^77 ^78 ^79 ^7a ^7b ^7c ^7d ^7e ^7f
+
+
"", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "",
+#^80 ^81 ^82 ^83 ^84 ^85 ^86 ^87 ^88 ^89 ^8a ^8b ^8c ^8d ^8e ^8f
+
"", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "",
+#^90 ^91 ^92 ^93 ^94 ^95 ^96 ^97 ^98 ^99 ^9a ^9b ^9c ^9d ^9e ^9f
+
+
' ', qq{!}, qq{C/}, 'PS', qq{\$?}, qq{Y=}, qq{|}, 'SS', qq{"}, qq{(c)}, 'a', qq{<<}, qq{!}, "", qq{(r)}, qq{-},
+#^a0 ^a1 ^a2 ^a3 ^a4 ^a5 ^a6 ^a7 ^a8 ^a9 ^aa ^ab ^ac ^ad ^ae ^af
+
'deg', qq{+-}, '2', '3', qq{'}, 'u', 'P', qq{*}, qq{,}, '1', 'o', qq{>>}, qq{1/4}, qq{1/2}, qq{3/4}, qq{?},
+#^b0 ^b1 ^b2 ^b3 ^b4 ^b5 ^b6 ^b7 ^b8 ^b9 ^ba ^bb ^bc ^bd ^be ^bf ^c0
+
'A', 'A', 'A', 'A', 'A', 'A', 'AE', 'C', 'E', 'E', 'E', 'E', 'I', 'I', 'I', 'I',
-'D', 'N', 'O', 'O', 'O', 'O', 'O', 'x', 'O', 'U', 'U', 'U', 'U', 'U', 'Th', 'ss',
+#^c0 ^c1 ^c2 ^c3 ^c4 ^c5 ^c6 ^c7 ^c8 ^c9 ^ca ^cb ^cc ^cd ^ce ^cf
+
+'D', 'N', 'O', 'O', 'O', 'O', 'O', 'x', 'O', 'U', 'U', 'U', 'U', 'Y', 'Th', 'ss',
+#^d0 ^d1 ^d2 ^d3 ^d4 ^d5 ^d6 ^d7 ^d8 ^d9 ^da ^db ^dc ^dd ^de ^df
+
'a', 'a', 'a', 'a', 'a', 'a', 'ae', 'c', 'e', 'e', 'e', 'e', 'i', 'i', 'i', 'i',
+#^e0 ^e1 ^e2 ^e3 ^e4 ^e5 ^e6 ^e7 ^e8 ^e9 ^ea ^eb ^ec ^ed ^ee ^ef
+
'd', 'n', 'o', 'o', 'o', 'o', 'o', qq{/}, 'o', 'u', 'u', 'u', 'u', 'y', 'th', 'y',
+#^f0 ^f1 ^f2 ^f3 ^f4 ^f5 ^f6 ^f7 ^f8 ^f9 ^fa ^fb ^fc ^fd ^fe ^ff
];
1;
@@ -1,9 +1,9 @@
-# Time-stamp: "Sat Jul 14 00:27:20 2001 by Automatic Bizooty (__blocks2pm.plx)"
+# Time-stamp: "2014-07-09 03:12:21 MDT sburke@cpan.org"
$Text::Unidecode::Char[0x01] = [
'A', 'a', 'A', 'a', 'A', 'a', 'C', 'c', 'C', 'c', 'C', 'c', 'C', 'c', 'D', 'd',
'D', 'd', 'E', 'e', 'E', 'e', 'E', 'e', 'E', 'e', 'E', 'e', 'G', 'g', 'G', 'g',
'G', 'g', 'G', 'g', 'H', 'h', 'H', 'h', 'I', 'i', 'I', 'i', 'I', 'i', 'I', 'i',
-'I', 'i', 'IJ', "", 'J', 'j', 'K', 'k', 'k', 'L', 'l', 'L', 'l', 'L', 'l', 'L',
+'I', 'i', 'IJ', 'ij', 'J', 'j', 'K', 'k', 'k', 'L', 'l', 'L', 'l', 'L', 'l', 'L',
'l', 'L', 'l', 'N', 'n', 'N', 'n', 'N', 'n', qq{'n}, 'ng', 'NG', 'O', 'o', 'O', 'o',
'O', 'o', 'OE', 'oe', 'R', 'r', 'R', 'r', 'R', 'r', 'S', 's', 'S', 's', 'S', 's',
'S', 's', 'T', 't', 'T', 't', 'T', 't', 'U', 'u', 'U', 'u', 'U', 'u', 'U', 'u',
@@ -15,6 +15,6 @@ $Text::Unidecode::Char[0x01] = [
qq{|}, qq{||}, qq{|=}, qq{!}, 'DZ', 'Dz', 'dz', 'LJ', 'Lj', 'lj', 'NJ', 'Nj', 'nj', 'A', 'a', 'I',
'i', 'O', 'o', 'U', 'u', 'U', 'u', 'U', 'u', 'U', 'u', 'U', 'u', qq{\@}, 'A', 'a',
'A', 'a', 'AE', 'ae', 'G', 'g', 'G', 'g', 'K', 'k', 'O', 'o', 'O', 'o', 'ZH', 'zh',
-'j', 'DZ', 'D', 'dz', 'G', 'g', 'HV', 'W', 'N', 'n', 'A', 'a', 'AE', 'ae', 'O', 'o',
+'j', 'DZ', 'Dz', 'dz', 'G', 'g', 'HV', 'W', 'N', 'n', 'A', 'a', 'AE', 'ae', 'O', 'o',
];
1;
@@ -1,17 +1,53 @@
-# Time-stamp: "Sat Jul 14 00:27:20 2001 by Automatic Bizooty (__blocks2pm.plx)"
+# Time-stamp: "2014-07-09 03:56:04 MDT sburke@cpan.org"
$Text::Unidecode::Char[0x02] = [
'A', 'a', 'A', 'a', 'E', 'e', 'E', 'e', 'I', 'i', 'I', 'i', 'O', 'o', 'O', 'o',
'R', 'r', 'R', 'r', 'U', 'u', 'U', 'u', 'S', 's', 'T', 't', 'Y', 'y', 'H', 'h',
-'[?]', '[?]', 'OU', 'ou', 'Z', 'z', 'A', 'a', 'E', 'e', 'O', 'o', 'O', 'o', 'O', 'o',
-'O', 'o', 'Y', 'y', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
-'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'N', 'd', 'OU', 'ou', 'Z', 'z', 'A', 'a', 'E', 'e', 'O', 'o', 'O', 'o', 'O', 'o',
+'O', 'o', 'Y', 'y',
+
+'l', # 0x34
+'n', # 0x35
+'t', # 0x36
+'j', # 0x37
+'db', # 0x38
+'qp', # 0x39
+'A', # 0x3a
+'C', # 0x3b
+'c', # 0x3c
+'L', # 0x3d
+'T', # 0x3e
+'s', # 0x3f
+'z', # 0x40
+
+'[?]', '[?]',
+
+'B', # 0x43
+'U', # 0x44
+'^', # 0x45
+'E', # 0x46
+'e', # 0x47
+'J', # 0x48
+'j', # 0x49
+'q', # 0x4a
+'q', # 0x4b
+'R', # 0x4c
+'r', # 0x4d
+'Y', # 0x4e
+'y', # 0x4f
+
+
'a', 'a', 'a', 'b', 'o', 'c', 'd', 'd', 'e', qq{\@}, qq{\@}, 'e', 'e', 'e', 'e', 'j',
'g', 'g', 'g', 'g', 'u', 'Y', 'h', 'h', 'i', 'i', 'I', 'l', 'l', 'l', 'lZ', 'W',
-'W', 'm', 'n', 'n', 'n', 'o', 'OE', 'O', 'F', 'R', 'R', 'R', 'R', 'r', 'r', 'R',
-'R', 'R', 's', 'S', 'j', 'S', 'S', 't', 't', 'U', 'U', 'v', qq{^}, 'W', 'Y', 'Y',
+'W', 'm', 'n', 'n', 'n', 'o', 'OE', 'O', 'F',
+
+'r', 'r', 'r', 'r',
+'r', 'r', 'r',
+
+'R', 'R', 's', 'S', 'j', 'S', 'S', 't', 't', 'u', 'U', 'v', qq{^}, 'w', 'y', 'Y',
'z', 'z', 'Z', 'Z', qq{?}, qq{?}, qq{?}, 'C', qq{\@}, 'B', 'E', 'G', 'H', 'j', 'k', 'L',
-'q', qq{?}, qq{?}, 'dz', 'dZ', 'dz', 'ts', 'tS', 'tC', 'fN', 'ls', 'lz', 'WW', qq{]]}, '[?]', '[?]',
-'k', 'h', 'j', 'r', 'r', 'r', 'r', 'w', 'y', qq{'}, qq{"}, qq{`}, qq{'}, qq{`}, qq{`}, qq{'},
+'q', qq{?}, qq{?}, 'dz', 'dZ', 'dz', 'ts', 'tS', 'tC', 'fN', 'ls', 'lz', 'WW', qq{]]},
+'h', 'h', 'h',
+'h', 'j', 'r', 'r', 'r', 'r', 'w', 'y', qq{'}, qq{"}, qq{`}, qq{'}, qq{`}, qq{`}, qq{'},
qq{?}, qq{?}, qq{<}, qq{>}, qq{^}, 'V', qq{^}, 'V', qq{'}, qq{-}, qq{/}, qq{\\}, qq{,}, qq{_}, qq{\\}, qq{/},
qq{:}, qq{.}, qq{`}, qq{'}, qq{^}, 'V', qq{+}, qq{-}, 'V', qq{.}, qq{\@}, qq{,}, qq{~}, qq{"}, 'R', 'X',
'G', 'l', 's', 'x', qq{?}, "", "", "", "", "", "", "", 'V', qq{=}, qq{"}, '[?]',
@@ -1,4 +1,4 @@
-# Time-stamp: "Sat Jul 14 00:27:20 2001 by Automatic Bizooty (__blocks2pm.plx)"
+# Time-stamp: "2014-07-09 04:09:13 MDT sburke@cpan.org"
$Text::Unidecode::Char[0x03] = [
"", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "",
"", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "",
@@ -6,7 +6,24 @@ $Text::Unidecode::Char[0x03] = [
"", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "",
"", "", "", "", "", "", "", "", "", "", "", "", "", "", "", '[?]',
'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
-"", "", "", '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+"", "", "",
+
+#'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'a', # 0x63
+'e', # 0x64
+'i', # 0x65
+'o', # 0x66
+'u', # 0x67
+'c', # 0x68
+'d', # 0x69
+'h', # 0x6a
+'m', # 0x6b
+'r', # 0x6c
+'t', # 0x6d
+'v', # 0x6e
+'x', # 0x6f
+
+
'[?]', '[?]', '[?]', '[?]', qq{'}, qq{,}, '[?]', '[?]', '[?]', '[?]', "", '[?]', '[?]', '[?]', qq{?}, '[?]',
'[?]', '[?]', '[?]', '[?]', "", "", 'A', qq{;}, 'E', 'E', 'I', '[?]', 'O', '[?]', 'U', 'O',
'I', 'A', 'B', 'G', 'D', 'E', 'Z', 'E', 'Th', 'I', 'K', 'L', 'M', 'N', 'Ks', 'O',
@@ -1,9 +1,9 @@
-# Time-stamp: "Sat Jul 14 00:27:21 2001 by Automatic Bizooty (__blocks2pm.plx)"
+# Time-stamp: "2014-07-09 04:18:09 MDT sburke@cpan.org"
$Text::Unidecode::Char[0x04] = [
-'Ie', 'Io', 'Dj', 'Gj', 'Ie', 'Dz', 'I', 'Yi', 'J', 'Lj', 'Nj', 'Tsh', 'Kj', 'I', 'U', 'Dzh',
-'A', 'B', 'V', 'G', 'D', 'Ie', 'Zh', 'Z', 'I', 'I', 'K', 'L', 'M', 'N', 'O', 'P',
+'Ie', 'Io', 'Dj', 'Gj', 'E', 'Dz', 'I', 'Yi', 'J', 'Lj', 'Nj', 'Tsh', 'Kj', 'I', 'U', 'Dzh',
+'A', 'B', 'V', 'G', 'D', 'E', 'Zh', 'Z', 'I', 'I', 'K', 'L', 'M', 'N', 'O', 'P',
'R', 'S', 'T', 'U', 'F', 'Kh', 'Ts', 'Ch', 'Sh', 'Shch', "", 'Y', qq{'}, 'E', 'Iu', 'Ia',
-'a', 'b', 'v', 'gh', 'd', 'ie', 'zh', 'z', 'i', 'i', 'k', 'l', 'm', 'n', 'o', 'p',
+'a', 'b', 'v', 'g', 'd', 'e', 'zh', 'z', 'i', 'i', 'k', 'l', 'm', 'n', 'o', 'p',
'r', 's', 't', 'u', 'f', 'kh', 'ts', 'ch', 'sh', 'shch', "", 'y', qq{'}, 'e', 'iu', 'ia',
'ie', 'io', 'dj', 'gj', 'ie', 'dz', 'i', 'yi', 'j', 'lj', 'nj', 'tsh', 'kj', 'i', 'u', 'dzh',
'O', 'o', 'E', 'e', 'Ie', 'ie', 'E', 'e', 'Ie', 'ie', 'O', 'o', 'Io', 'io', 'Ks', 'ks',
@@ -1,4 +1,4 @@
-# Time-stamp: "Sat Jul 14 00:27:21 2001 by Automatic Bizooty (__blocks2pm.plx)"
+# Time-stamp: "2014-07-09 04:28:06 MDT sburke@cpan.org"
$Text::Unidecode::Char[0x05] = [
'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
@@ -12,7 +12,11 @@ $Text::Unidecode::Char[0x05] = [
'[?]', "", "", "", "", "", "", "", "", "", "", "", "", "", "", "",
"", "", '[?]', "", "", "", "", "", "", "", "", "", "", "", "", "",
qq{\@}, 'e', 'a', 'o', 'i', 'e', 'e', 'a', 'a', 'o', '[?]', 'u', qq{'}, "", "", "",
-"", "", "", qq{:}, "", '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+
+# u+05c0:
+"|", "", "", qq{:}, "", '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+
+
"", 'b', 'g', 'd', 'h', 'v', 'z', 'kh', 't', 'y', 'k', 'k', 'l', 'm', 'm', 'n',
'n', 's', qq{`}, 'p', 'p', 'ts', 'ts', 'q', 'r', 'sh', 't', '[?]', '[?]', '[?]', '[?]', '[?]',
'V', 'oy', 'i', qq{'}, qq{"}, '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:02:31 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0x08 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,20 @@
+# Time-stamp: "2014-07-09 04:45:32 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0x19 ] = [
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+];
+1;
@@ -0,0 +1,20 @@
+# Time-stamp: "2014-07-09 04:45:01 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0x1a ] = [
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+];
+1;
@@ -0,0 +1,20 @@
+# Time-stamp: "2014-07-09 04:45:07 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0x1b ] = [
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+];
+1;
@@ -0,0 +1,20 @@
+# Time-stamp: "2014-07-09 04:44:34 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0x1c ] = [
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+];
+1;
@@ -0,0 +1,20 @@
+# Time-stamp: "2014-07-09 04:44:20 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0x1d ] = [
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
+];
+1;
@@ -1,4 +1,4 @@
-# Time-stamp: "Sat Jul 14 00:27:22 2001 by Automatic Bizooty (__blocks2pm.plx)"
+# Time-stamp: "2014-07-22 05:34:54 MDT sburke@cpan.org"
$Text::Unidecode::Char[0x1e] = [
'A', 'a', 'B', 'b', 'B', 'b', 'B', 'b', 'C', 'c', 'D', 'd', 'D', 'd', 'D', 'd',
'D', 'd', 'D', 'd', 'E', 'e', 'E', 'e', 'E', 'e', 'E', 'e', 'E', 'e', 'F', 'f',
@@ -9,7 +9,7 @@ $Text::Unidecode::Char[0x1e] = [
'S', 's', 'S', 's', 'S', 's', 'S', 's', 'S', 's', 'T', 't', 'T', 't', 'T', 't',
'T', 't', 'U', 'u', 'U', 'u', 'U', 'u', 'U', 'u', 'U', 'u', 'V', 'v', 'V', 'v',
'W', 'w', 'W', 'w', 'W', 'w', 'W', 'w', 'W', 'w', 'X', 'x', 'X', 'x', 'Y', 'y',
-'Z', 'z', 'Z', 'z', 'Z', 'z', 'h', 't', 'w', 'y', 'a', 'S', '[?]', '[?]', '[?]', '[?]',
+'Z', 'z', 'Z', 'z', 'Z', 'z', 'h', 't', 'w', 'y', 'a', 's', 's', 's', 'Ss', 'd',
'A', 'a', 'A', 'a', 'A', 'a', 'A', 'a', 'A', 'a', 'A', 'a', 'A', 'a', 'A', 'a',
'A', 'a', 'A', 'a', 'A', 'a', 'A', 'a', 'E', 'e', 'E', 'e', 'E', 'e', 'E', 'e',
'E', 'e', 'E', 'e', 'E', 'e', 'E', 'e', 'I', 'i', 'I', 'i', 'O', 'o', 'O', 'o',
@@ -1,4 +1,4 @@
-# Time-stamp: "Sat Jul 14 00:27:22 2001 by Automatic Bizooty (__blocks2pm.plx)"
+# Time-stamp: "2013-09-22 03:48:31 MDT sburke@cpan.org"
$Text::Unidecode::Char[0x20] = [
' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', "", "", "", "",
qq{-}, qq{-}, qq{-}, qq{-}, qq{--}, qq{--}, qq{||}, qq{_}, qq{'}, qq{'}, qq{,}, qq{'}, qq{"}, qq{"}, qq{,,}, qq{"},
@@ -10,7 +10,7 @@ qq{-}, qq{^}, qq{***}, qq{--}, qq{/}, qq{-[}, qq{]-}, '[?]', qq{?!}, qq{!?}, '7'
'0', "", "", "", '4', '5', '6', '7', '8', '9', qq{+}, qq{-}, qq{=}, qq{(}, qq{)}, 'n',
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', qq{+}, qq{-}, qq{=}, qq{(}, qq{)}, '[?]',
'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
-'ECU', 'CL', 'Cr', 'FF', 'L', 'mil', 'N', 'Pts', 'Rs', 'W', 'NS', 'D', 'EU', 'K', 'T', 'Dr',
+'ECU', 'CL', 'Cr', 'FF', 'L', 'mil', 'N', 'Pts', 'Rs', 'W', 'NS', 'D', 'EUR', 'K', 'T', 'Dr',
'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
"", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "",
@@ -1,17 +1,21 @@
-# Time-stamp: "Sat Jul 14 00:27:22 2001 by Automatic Bizooty (__blocks2pm.plx)"
+# Time-stamp: "2013-10-21 16:18:35 MDT sburke@cpan.org"
$Text::Unidecode::Char[0x21] = [
+
"", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "",
"", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "",
+"", "", "tm", "", "", "", "", "", "", "", "", "", "", "", "", "",
"", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "",
-"", "", "", "", "", "", "", "", "", "", "", '[?]', '[?]', '[?]', '[?]', '[?]',
+
'[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
'[?]', '[?]', '[?]', qq{ 1/3 }, qq{ 2/3 }, qq{ 1/5 }, qq{ 2/5 }, qq{ 3/5 }, qq{ 4/5 }, qq{ 1/6 }, qq{ 5/6 }, qq{ 1/8 }, qq{ 3/8 }, qq{ 5/8 }, qq{ 7/8 }, qq{ 1/},
'I', 'II', 'III', 'IV', 'V', 'VI', 'VII', 'VIII', 'IX', 'X', 'XI', 'XII', 'L', 'C', 'D', 'M',
'i', 'ii', 'iii', 'iv', 'v', 'vi', 'vii', 'viii', 'ix', 'x', 'xi', 'xii', 'l', 'c', 'd', 'm',
+
qq{(D}, qq{D)}, qq{((|))}, qq{)}, '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]', '[?]',
qq{-}, qq{|}, qq{-}, qq{|}, qq{-}, qq{|}, qq{\\}, qq{/}, qq{\\}, qq{/}, qq{-}, qq{-}, qq{~}, qq{~}, qq{-}, qq{|},
qq{-}, qq{|}, qq{-}, qq{-}, qq{-}, qq{|}, qq{-}, qq{|}, qq{|}, qq{-}, qq{-}, qq{-}, qq{-}, qq{-}, qq{-}, qq{|},
qq{|}, qq{|}, qq{|}, qq{|}, qq{|}, qq{|}, qq{^}, 'V', qq{\\}, qq{=}, 'V', qq{^}, qq{-}, qq{-}, qq{|}, qq{|},
+
qq{-}, qq{-}, qq{|}, qq{|}, qq{=}, qq{|}, qq{=}, qq{=}, qq{|}, qq{=}, qq{|}, qq{=}, qq{=}, qq{=}, qq{=}, qq{=},
qq{=}, qq{|}, qq{=}, qq{|}, qq{=}, qq{|}, qq{\\}, qq{/}, qq{\\}, qq{/}, qq{=}, qq{=}, qq{~}, qq{~}, qq{|}, qq{|},
qq{-}, qq{|}, qq{-}, qq{|}, qq{-}, qq{-}, qq{-}, qq{|}, qq{-}, qq{|}, qq{|}, qq{|}, qq{|}, qq{|}, qq{|}, qq{|},
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:03:09 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0x29 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0x2a ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0x2b ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0x2c ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0x2d ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0x34 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0x35 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0x36 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0x37 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0x38 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0x39 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0x3a ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0x3b ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0x3c ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0x3d ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0x3e ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0x3f ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0x40 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0x41 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0x42 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0x43 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0x44 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0x45 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0x46 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0x47 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0x48 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0x49 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0x4a ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0x4b ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0x4c ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -1,6 +1,6 @@
-# Time-stamp: "Sat Jul 14 00:27:30 2001 by Automatic Bizooty (__blocks2pm.plx)"
+# Time-stamp: "2014-01-24 23:59:56 MST sburke@cpan.org"
$Text::Unidecode::Char[0x4e] = [
-qq{[?] }, 'Ding ', 'Kao ', 'Qi ', 'Shang ', 'Xia ', qq{[?] }, 'Mo ', 'Zhang ', 'San ', 'Shang ', 'Xia ', 'Ji ', 'Bu ', 'Yu ', 'Mian ',
+'Yi ', 'Ding ', 'Kao ', 'Qi ', 'Shang ', 'Xia ', qq{[?] }, 'Mo ', 'Zhang ', 'San ', 'Shang ', 'Xia ', 'Ji ', 'Bu ', 'Yu ', 'Mian ',
'Gai ', 'Chou ', 'Chou ', 'Zhuan ', 'Qie ', 'Pi ', 'Shi ', 'Shi ', 'Qiu ', 'Bing ', 'Ye ', 'Cong ', 'Dong ', 'Si ', 'Cheng ', 'Diu ',
'Qiu ', 'Liang ', 'Diu ', 'You ', 'Liang ', 'Yan ', 'Bing ', 'Sang ', 'Gun ', 'Jiu ', 'Ge ', 'Ya ', 'Qiang ', 'Zhong ', 'Ji ', 'Jie ',
'Feng ', 'Guan ', 'Chuan ', 'Chan ', 'Lin ', 'Zhuo ', 'Zhu ', 'Ha ', 'Wan ', 'Dan ', 'Wei ', 'Zhu ', 'Jing ', 'Li ', 'Ju ', 'Pie ',
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xa5 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xa6 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xa7 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xa8 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xa9 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xaa ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xab ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xd8 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xd9 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xda ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xdb ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xdc ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xdd ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xde ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xdf ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xe0 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xe1 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xe2 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xe3 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xe4 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xe5 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xe6 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xe7 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xe8 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xe9 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xea ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xeb ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xec ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xed ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xee ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xef ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xf0 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xf1 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xf2 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xf3 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xf4 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xf5 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xf6 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xf7 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -0,0 +1,3 @@
+# Time-stamp: "2014-07-22 05:15:15 MDT sburke@cpan.org"
+$Text::Unidecode::Char[ 0xf8 ] = Text::Unidecode::make_placeholder_map();
+1;
@@ -1,18 +1,21 @@
+;;;;# -*-coding:utf-8;-*- µ ← col73
-require 5.006;
-package Text::Unidecode; # Time-stamp: "2001-07-14 02:29:41 MDT"
+require 5;
+use 5.8.0;
+package Text::Unidecode; # Time-stamp: "2014-12-07 05:15:04 MST sburke@cpan.org"
use utf8;
use strict;
use integer; # vroom vroom!
-use vars qw($VERSION @ISA @EXPORT @Char $NULLMAP);
-$VERSION = '0.04';
+use vars qw($VERSION @ISA @EXPORT @Char $UNKNOWN $NULLMAP $TABLE_SIZE);
+$VERSION = '1.23';
require Exporter;
@ISA = ('Exporter');
@EXPORT = ('unidecode');
BEGIN { *DEBUG = sub () {0} unless defined &DEBUG }
-
-$NULLMAP = [('[?] ') x 0x100]; # for blocks we can't load
+$UNKNOWN = '[?] ';
+$TABLE_SIZE = 256;
+$NULLMAP = [( $UNKNOWN ) x $TABLE_SIZE]; # for blocks we can't load
#--------------------------------------------------------------------------
{
@@ -25,23 +28,81 @@ $NULLMAP = [('[?] ') x 0x100]; # for blocks we can't load
sub unidecode {
# Destructive in void context -- in other contexts, nondestructive.
- unless(@_) {
- # Nothing coming in
+ unless(@_) { # Sanity: Nothing coming in!
return() if wantarray;
return '';
}
- @_ = map $_, @_ if defined wantarray;
- # We're in list or scalar context, NOT void context.
- # So make @_'s items no longer be aliases.
- # Otherwise, let @_ be aliases, and alter in-place.
-
- foreach my $x (@_) {
- next unless defined $x;
- $x =~ s~([^\x00-\x7f])~${$Char[ord($1)>>8]||t($1)}[ord($1)&255]~egs;
- # Replace character 0xABCD with $Char[0xAB][0xCD], loading
- # the table as needed.
+
+ if( defined wantarray ) {
+ # We're in list or scalar context (i.e., just not void context.)
+ # So make @_'s items no longer be aliases.
+ @_ = map $_, @_;
+ } else {
+ # Otherwise (if we're in void context), then just let @_ stay
+ # aliases, and alter their elements IN-PLACE!
}
+ foreach my $n (@_) {
+ next unless defined $n;
+
+ # Shut up potentially fatal warnings about UTF-16 surrogate
+ # characters when running under perl -w
+ # This is per https://rt.cpan.org/Ticket/Display.html?id=97456
+ no warnings 'utf8';
+
+ $n =~ s~([^\x00-\x7f])~${$Char[ord($1)>>8]||t($1)}[ord($1)&255]~egs;
+ }
+ # That means:
+ # Replace character 0xABCD with $Char[0xAB][0xCD], loading
+ # the table 0xAB as needed.
+ #
+ #======================================================================
+ #
+ # Yes, that's dense code. It's the warp core!
+ # Here is an expansion into pseudocode... as best as I can manage it...
+ #
+ # $character = $1;
+ # $charnum = ord($character);
+ # $charnum_lowbits = $charnum & 255;
+ # $charnum_highbits = $charnum >> 8;
+ #
+ # $table_ref = $Char->[$charnum_highbits];
+ #
+ # if($table_ref) {
+ # # As expected, we got the arrayref for this table.
+ # } else {
+ # # Uhoh, we couldn't find the arrayref for this table.
+ # # So we call t($character).
+ # # It loads a table. Namely, it does:
+ # Load_Table_For( $charnum_highbits );
+ # # ...which does magic, and puts something in
+ # # $Char->[$charnum_highbits],
+ # # so NOW we actually CAN do:
+ # $table_ref = $Char->[$charnum_highbits];
+ # }
+ #
+ # $for_this_char
+ # = $table_ref->[ $charnum_lowbits ];
+ #
+ # # Although the syntax we actually use is the odd
+ # but COMPLETE EQUIVALENT to this syntax:
+ #
+ # $for_this_char
+ # = ${ $table_ref }[ $charnum_lowbits ];
+ #
+ # and $for_this_char is the replacement text for this
+ # character, in:
+ # $n =~ s~(char)~replacement~egs
+ #
+ # (And why did I use s~x~y~ instead of s/x/y/ ?
+ # It's all the same for Perl: perldoc perlretut says:
+ # As with the match "m//" operator, "s///" can
+ # use other delimiters, such as "s!!!" and "s{}{}",
+ # I didn't do it for sake of obscurity. I think it's just to
+ # keep my editor's syntax highlighter from crashing,
+ # which was a problem with s/// when the insides are as gory
+ # as we have here.
+
return unless defined wantarray; # void context
return @_ if wantarray; # normal list context -- return the copies
# Else normal scalar context:
@@ -49,60 +110,129 @@ sub unidecode {
return join '', @_; # rarer fallthru: a list in, but a scalar out.
}
-sub t {
- # load (and return) a char table for this character
- # this should get called only once per table per session.
- my $bank = ord($_[0]) >> 8;
- return $Char[$bank] if $Char[$bank];
+#======================================================================
+
+sub make_placeholder_map {
+ return [( $UNKNOWN ) x $TABLE_SIZE ];
+}
+sub make_placeholder_map_nulls {
+ return [( "" ) x $TABLE_SIZE ];
+}
+
+#======================================================================
+
+sub t { # "t" is for "t"able.
+ # Load (and return) a char table for this character
+ # this should get called only once per table per session.
+ my $bank = ord($_[0]) >> 8;
+ return $Char[$bank] if $Char[$bank];
- {
- DEBUG and printf "Loading %s::x%02x\n", __PACKAGE__, $bank;
- local $SIG{'__DIE__'};
- eval(sprintf 'require %s::x%02x;', __PACKAGE__, $bank);
- }
+ load_bank($bank);
- # Now see how that fared...
- if(ref($Char[$bank] || '') ne 'ARRAY') {
- DEBUG > 1 and print
- " Loading failed for bank $bank (err $@). Using null map.\n";
- return $Char[$bank] = $NULLMAP;
- } else {
- DEBUG > 1 and print " Succeeded.\n";
- if(DEBUG) {
- # Sanity-check it:
- my $cb = $Char[$bank];
- unless(@$cb == 256) {
- printf "Block x%02x is of size %d -- chopping to 256\n",
- scalar(@$cb);
- $#$cb = 255; # pre-extend the array, or chop it to size.
- }
- for(my $i = 0; $i < 256; ++$i) {
- unless(defined $cb->[$i]) {
- printf "Undef at position %d in block x%02x\n",
- $i, $bank;
- $cb->[$i] = '';
- }
- }
- }
- return $Char[$bank];
- }
+ # Now see how that fared...
+
+ if(ref($Char[$bank] || '') ne 'ARRAY') {
+ DEBUG > 1 and print
+ " Loading failed for bank $bank (err $@). Using null map.\n";
+ return $Char[$bank] = $NULLMAP;
+ }
+
+
+ DEBUG > 1 and print " Loading succeeded.\n";
+ my $cb = $Char[$bank];
+
+ # Sanity-check it:
+ if(@$cb == $TABLE_SIZE) {
+ # As expected. Fallthru.
+
+ } else {
+ if(@$cb > $TABLE_SIZE) {
+ DEBUG and print "Bank $bank is too large-- it has ", scalar @$cb,
+ " entries in it. Pruning.\n";
+ splice @$cb, $TABLE_SIZE;
+ # That two-argument form splices everything off into nowhere,
+ # starting with the first overage character.
+
+ } elsif( @$cb < $TABLE_SIZE) {
+ DEBUG and print "Bank $bank is too small-- it has ", scalar @$cb,
+ " entries in it. Now padding it.\n";
+ if(@$cb == 0) {
+ DEBUG and print " (Yes, ZERO entries!)\n";
+ }
+ push @$cb,
+ ( $UNKNOWN ) x ( $TABLE_SIZE - @$cb)
+ # i.e., however many items, times the deficit
+ ;
+ # And fallthru...
+
+ } else {
+ die "UNREACHABLE CODE HERE (INSANE)";
+ }
+ }
+
+ # Check for undefness in block:
+
+ for(my $i = 0; $i < $TABLE_SIZE; ++$i) {
+ unless(defined $cb->[$i]) {
+ DEBUG and printf "Undef at position %d in block x%02x\n",
+ $i, $bank;
+ $cb->[$i] = '';
+ }
+ }
+
+ return $Char[$bank];
}
-#--------------------------------------------------------------------------
+#-----------------------------------------------------------------------
+
+our $eval_loaded_okay;
+
+sub load_bank {
+
+ # This is in its own sub, for sake of sweeping the scary thing
+ # (namely, a call to eval) under the rug.
+ # I.e., to paraphrase what Larry Wall once said to me: if
+ # you're going to do something odd, maybe you should do it
+ # in private.
+
+ my($banknum) = @_; # just as an integer value
+
+ DEBUG and printf
+ "# Eval-loading %s::x%02x ...\n",
+
+ $eval_loaded_okay = 0;
+ my $code =
+ sprintf( "require %s::x%02x; \$eval_loaded_okay = 1;\n",
+ __PACKAGE__,
+ $banknum);
+
+ {
+ local $SIG{'__DIE__'};
+ eval($code);
+ }
+
+ return 1 if $eval_loaded_okay;
+ return 0;
+}
+
+#======================================================================
+
1;
__END__
+=encoding utf8
+
=head1 NAME
-Text::Unidecode -- US-ASCII transliterations of Unicode text
+Text::Unidecode -- plain ASCII transliterations of Unicode text
=head1 SYNOPSIS
use utf8;
use Text::Unidecode;
print unidecode(
- "\x{5317}\x{4EB0}\n"
- # those are the Chinese characters for Beijing
+ "北亰\n"
+ # Chinese characters for Beijing (U+5317 U+4EB0)
);
# That prints: Bei Jing
@@ -110,7 +240,7 @@ Text::Unidecode -- US-ASCII transliterations of Unicode text
=head1 DESCRIPTION
It often happens that you have non-Roman text data in Unicode, but
-you can't display it -- usually because you're trying to
+you can't display it-- usually because you're trying to
show it to a user via an application that doesn't support Unicode,
or because the fonts you need aren't accessible. You could
represent the Unicode characters as "???????" or
@@ -121,19 +251,34 @@ What Text::Unidecode provides is a function, C<unidecode(...)> that
takes Unicode data and tries to represent it in US-ASCII characters
(i.e., the universally displayable characters between 0x00 and
0x7F). The representation is
-almost always an attempt at I<transliteration> -- i.e., conveying,
+almost always an attempt at I<transliteration>-- i.e., conveying,
in Roman letters, the pronunciation expressed by the text in
some other writing system. (See the example in the synopsis.)
-Unidecode's ability to transliterate is limited by two factors:
+
+NOTE:
+
+To make sure your perldoc/Pod viewing setup for viewing this page is
+working: The six-letter word "résumé" should look like "resume" with
+an "/" accent on each "e".
+
+For further tests, and help if that doesn't work, see below,
+L</A POD ENCODING TEST>.
+
+
+=head1 DESIGN PHILOSOPHY
+
+Unidecode's ability to transliterate from a given language is limited
+by two factors:
=over
-=item * The amount and quality of data in the original
+=item * The amount and quality of data in the written form of the
+original language
So if you have Hebrew data
that has no vowel points in it, then Unidecode cannot guess what
-vowels should appear in a pronounciation.
+vowels should appear in a pronunciation.
S f y hv n vwls n th npt, y wn't gt ny vwls
n th tpt. (This is a specific application of the general principle
of "Garbage In, Garbage Out".)
@@ -176,22 +321,22 @@ is exported by default. It can be used in a variety of calling contexts:
=over
-=item C<$out = unidecode($in);> # scalar context
+=item C<$out = unidecode( $in );> # scalar context
This returns a copy of $in, transliterated.
-=item C<$out = unidecode(@in);> # scalar context
+=item C<$out = unidecode( @in );> # scalar context
-This is the same as C<$out = unidecode(join '', @in);>
+This is the same as C<$out = unidecode(join "", @in);>
-=item C<@out = unidecode(@in);> # list context
+=item C<@out = unidecode( @in );> # list context
This returns a list consisting of copies of @in, each transliterated. This
is the same as C<@out = map scalar(unidecode($_)), @in;>
-=item C<unidecode(@items);> # void context
+=item C<unidecode( @items );> # void context
-=item C<unidecode(@bar, $foo, @baz);> # void context
+=item C<unidecode( @bar, $foo, @baz );> # void context
Each item on input is replaced with its transliteration. This
is the same as C<for(@bar, $foo, @baz) { $_ = unidecode($_) }>
@@ -201,9 +346,10 @@ is the same as C<for(@bar, $foo, @baz) { $_ = unidecode($_) }>
You should make a minimum of assumptions about the output of
C<unidecode(...)>. For example, if you assume an all-alphabetic
(Unicode) string passed to C<unidecode(...)> will return an all-alphabetic
-string, you're wrong -- some alphabetic Unicode characters are
+string, you're wrong-- some alphabetic Unicode characters are
transliterated as strings containing punctuation (e.g., the
-Armenian letter at 0x0539 currently transliterates as C<T`>.
+Armenian letter "Թ" (U+0539), currently transliterates as "T`"
+(capital-T then a backtick).
However, these are the assumptions you I<can> make:
@@ -217,7 +363,7 @@ C<unidecode(...)> is 7-bit pure.
=item *
The output of C<unidecode(...)> always consists entirely of US-ASCII
-characters -- i.e., characters 0x0000 - 0x007F.
+characters-- i.e., characters 0x0000 - 0x007F.
=item *
@@ -228,9 +374,9 @@ you have a "\x01" on input, you'll get a "\x01" in output.)
=item *
-Yes, some transliterations produce a "\n" -- but just a few, and only
-with good reason. Note that the value of newline ("\n") varies
-from platform to platform -- see L<perlport/perlport>.
+Yes, some transliterations produce a "\n" but it's just a few, and
+only with good reason. Note that the value of newline ("\n") varies
+from platform to platform-- see L<perlport>.
=item *
@@ -239,31 +385,42 @@ Some Unicode characters may transliterate to nothing (i.e., empty string).
=item *
Very many Unicode characters transliterate to multi-character sequences.
-E.g., Han character 0x5317 transliterates as the four-character string
+E.g., Unihan character U+5317, "北", transliterates as the four-character string
"Bei ".
=item *
-Within these constraints, I may change the transliteration of characters
+Within these constraints, I<I may change> the transliteration of characters
in future versions. For example, if someone convinces me that
-the Armenian letter at 0x0539, currently transliterated as "T`", would
-be better transliterated as "D", I may well make that change.
+that the Armenian letter "Թ", currently transliterated as "T`", would
+be better transliterated as "D", I I<may> well make that change.
+
+=item *
+
+Unfortunately, there are many characters that Unidecode doesn't know a
+transliteration for. This is generally because the character has been
+added since I last revised the Unidecode data tables. I'm always
+catching up!
=back
=head1 DESIGN GOALS AND CONSTRAINTS
-Text::Unidecode is meant to be a transliterator-of-last resort,
+Text::Unidecode is meant to be a transliterator of last resort,
to be used once you've decided that you can't just display the
-Unicode data as is, and once you've decided you don't have a
-more clever, language-specific transliterator available. It
-transliterates context-insensitively -- that is, a given character is
+Unicode data as is, I<and once you've decided you don't have a
+more clever, language-specific transliterator available,> or once
+you've I<already applied> smarter algorithms or mappings that you prefer
+and you now just want Unidecode to do cleanup.
+
+Unidecode
+transliterates context-insensitively-- that is, a given character is
replaced with the same US-ASCII (7-bit ASCII) character or characters,
-no matter what the surrounding character are.
+no matter what the surrounding characters are.
The main reason I'm making Text::Unidecode work with only
context-insensitive substitution is that it's fast, dumb, and
-straightforward enough to be feasable. It doesn't tax my
+straightforward enough to be feasible. It doesn't tax my
(quite limited) knowledge of world languages. It doesn't require
me writing a hundred lines of code to get the Thai syllabification
right (and never knowing whether I've gotten it wrong, because I
@@ -275,11 +432,11 @@ moreover, context-insensitive substitution is still mostly useful,
but still clearly couldn't be mistaken for authoritative.
Text::Unidecode is an example of the 80/20 rule in
-action -- you get 80% of the usefulness using just 20% of a
+action-- you get 80% of the usefulness using just 20% of a
"real" solution.
A "real" approach to transliteration for any given language can
-involve such increasingly tricky contextual factors as these
+involve such increasingly tricky contextual factors as these:
=over
@@ -292,7 +449,7 @@ by some diacritic character.
=item Syllables
A character "X" at end of a syllable could mean something
-different from when it's at the start -- which is especially problematic
+different from when it's at the start-- which is especially problematic
when the language involved doesn't explicitly mark where one syllable
stops and the next starts.
@@ -332,12 +489,80 @@ Out of a desire to avoid being mired in I<any> of these kinds of
contextual factors, I chose to exclude I<all of them> and just stick
with context-insensitive replacement.
+
+=head1 A POD ENCODING TEST
+
+=over
+
+=item *
+
+"Brontë" is six characters that should look like "Bronte", but
+with double-dots on the "e" character.
+
+=item *
+
+"Résumé" is six characters that should look like "Resume", but
+with /-shaped accents on the "e" characters.
+
+=item *
+
+"læti" should be I<four> letters long-- the second letter should not
+be two letters "ae", but should be a single letter that
+looks like an "a" entirely fused with an "e".
+
+=item *
+
+"χρονος" is six Greek characters that should look kind of like: xpovoc
+
+=item *
+
+"КАК ВАС ЗОВУТ" is three short Russian words that should look a
+lot like: KAK BAC 3OBYT
+
+=item *
+
+"ടധ" is two Malayalam characters that should look like: sw
+
+=item *
+
+"丫二十一" is four Chinese characters that should look like: C<Y=+->
+
+=back
+
+If all of those come out right, your Pod viewing setup is working
+fine-- welcome to the 2010s! If those are full of garbage characters,
+consider viewing this page as HTML at
+L<https://metacpan.org/pod/Text::Unidecode>
+or
+L<http://search.cpan.org/perldoc?Text::Unidecode>
+
+
+If things look mostly okay, but the Malayalam and/or the Chinese are
+just question-marks or empty boxes, it's probably just that your
+computer lacks the fonts for those.
+
=head1 TODO
-Things that need tending to are detailed in the TODO.txt file, included
-in this distribution. Normal installs probably don't leave the TODO.txt
-lying around, but if nothing else, you can see it at
-http://search.cpan.org/search?dist=Text::Unidecode
+Lots:
+
+* Rebuild the Unihan database. (Talk about hitting a moving target!)
+
+* Add tone-numbers for Mandarin hanzi? Namely: In Unihan, when tone
+marks are present (like in "kMandarin: dào", should I continue to
+transliterate as just "Dao", or should I put in the tone number:
+"Dao4"? It would be pretty jarring to have digits appear where
+previously there was just alphabetic stuff-- But tone numbers
+make Chinese more readable.
+
+* Start dealing with characters over U+FFFF.
+
+* Fill in all the little characters that've crept into the Misc Symbols
+Etc blocks.
+
+* More things that need tending to are detailed in the TODO.txt file,
+included in this distribution. Normal installs probably don't leave
+the TODO.txt lying around, but if nothing else, you can see it at
+L<http://search.cpan.org/search?dist=Text::Unidecode>
=head1 MOTTO
@@ -345,26 +570,150 @@ The Text::Unidecode motto is:
It's better than nothing!
-...in both meanings: 1) seeing the output of C<unidecode(...)> is
+...in I<both> meanings: 1) seeing the output of C<unidecode(...)> is
better than just having all font-unavailable Unicode characters
replaced with "?"'s, or rendered as gibberish; and 2) it's the
worst, i.e., there's nothing that Text::Unidecode's algorithm is
-better than.
+better than. All sensible transliteration algorithms (like for
+German, see below) are going to be smarter than Unidecode's.
+
+=head1 WHEN YOU DON'T LIKE WHAT UNIDECODE DOES
+
+I will repeat the above, because some people miss it:
+
+Text::Unidecode is meant to be a transliterator of I<last resort,>
+to be used once you've decided that you can't just display the
+Unicode data as is, I<and once you've decided you don't have a
+more clever, language-specific transliterator available>-- or once
+you've I<already applied> a smarter algorithm and now just want Unidecode
+to do cleanup.
+
+In other words, when you don't like what Unidecode does, I<do it
+yourself.> Really, that's what the above says. Here's how
+you would do this for German, for example:
+
+In German, there's the typographical convention that an umlaut (the
+double-dots on: ä ö ü) can be written as an "-e", like with "Schön"
+becoming "Schoen". But Unidecode doesn't do that-- I have Unidecode
+simply drop the umlaut accent and give back "Schon".
+
+(I chose this not because I'm a big meanie, but because
+I<generally> changing "ü" to "ue" is disastrous for all text
+that's I<not in German>. Finnish "Hyvää päivää" would turn
+into "Hyvaeae paeivaeae". And I discourage you from being I<yet
+another> German who emails me, trying to impel me to consider
+a typographical nicety of German to be more important than
+I<all other languages>.)
+
+If you know that the text you're handling is probably in German, and
+you want to apply the "umlaut becomes -e" rule, here's how to do it
+for yourself (and then use Unidecode as I<the fallback> afterwards):
+
+ use utf8; # <-- probably necessary.
+
+ our( %German_Characters ) = qw(
+ Ä AE ä ae
+ Ö OE ö oe
+ Ü UE ü ue
+ ß ss
+ );
+
+ use Text::Unidecode qw(unidecode);
+
+ sub german_to_ascii {
+ my($german_text) = @_;
+
+ $german_text =~
+ s/([ÄäÖöÜüß])/$German_Characters{$1}/g;
+
+ # And now, as a *fallthrough*:
+ $german_text = unidecode( $german_text );
+ return $german_text;
+ }
+
+To pick another example, here's something that's not about a
+specific language, but simply having a preference that may or
+may not agree with Unidecode's (i.e., mine). Consider the "¥"
+symbol. Unidecode changes that to "Y=". If you want "¥" as
+"YEN", then...
+
+ use Text::Unidecode qw(unidecode);
+
+ sub my_favorite_unidecode {
+ my($text) = @_;
+
+ $text =~ s/¥/YEN/g;
+
+ # ...and anything else you like, such as:
+ $text =~ s/€/Euro/g;
+
+ # And then, as a fallback,...
+ $text = unidecode($text);
+
+ return $text;
+ }
+
+Then if you do:
+
+ print my_favorite_unidecode("You just won ¥250,000 and €40,000!!!");
+
+...you'll get:
+
+ You just won YEN250,000 and Euro40,000!!!
+
+...just as you like it.
+
+(By the way, the reason I<I> don't have Unidecode just turn "¥" into "YEN"
+is that the same symbol also stands for yuan, the Chinese
+currency. A "Y=" is nicely, I<safely> neutral as to whether
+we're talking about yen or yuan-- Japan, or China.)
+
+Another example: for hanzi/kanji/hanja, I have designed
+Unidecode to transliterate according to the value that that
+character has in Mandarin (otherwise Cantonese,...). Some
+users have complained that applying Unidecode to Japanese
+produces gibberish.
+
+To make a long story short: transliterating from Japanese is
+I<difficult> and it requires a I<lot> of context-sensitivity.
+If you have text that you're fairly sure is in
+Japanese, you're going to have to use a Japanese-specific
+algorithm to transliterate Japanese into ASCII. (And then
+you can call Unidecode on the output from that-- it is useful
+for, for example, turning ｆｕｌｌｗｉｄｔｈ characters into
+their normal (ASCII) forms.
=head1 CAVEATS
If you get really implausible nonsense out of C<unidecode(...)>, make
sure that the input data really is a utf8 string. See
-L<perlunicode/perlunicode>.
+L<perlunicode> and L<perlunitut>.
=head1 THANKS
-Thanks to Harald Tveit Alvestrand,
-Abhijit Menon-Sen, and Mark-Jason Dominus.
+Thanks to (in only the sloppiest of sorta-chronological order):
+Jordan Lachler, Harald Tveit Alvestrand, Melissa Axelrod,
+Abhijit Menon-Sen, Mark-Jason Dominus, Joe Johnston,
+Conrad Heiney, fileformat.info,
+Philip Newton, 唐鳳, Tomaž Šolc, Mike Doherty, JT Smith and the
+MadMongers, Arden Ogg, Craig Copris,
+and I<many> other pals in Unicode's behind-the-scenes F5
+tornado underlying its code.
=head1 SEE ALSO
-Unicode Consortium: http://www.unicode.org/
+An article I wrote for I<The Perl Journal> about
+Unidecode: L<http://interglacial.com/tpj/22/>
+(B<READ IT!>)
+
+Tom Christiansen's
+I<Perl Unicode Cookbook>,
+L<http://www.perl.com/pub/2012/04/perlunicook-standard-preamble.html>
+
+Unicode Consortium: L<http://www.unicode.org/>
+
+Searchable Unihan database:
+L<http://www.unicode.org/cgi-bin/GetUnihanData.pl>
Geoffrey Sampson. 1990. I<Writing Systems: A Linguistic Introduction.>
ISBN: 0804717567
@@ -378,9 +727,12 @@ Congress.]
Rupert Snell. 2000. I<Beginner's Hindi Script (Teach Yourself
Books).> ISBN: 0658009109
-=head1 COPYRIGHT AND DISCLAIMERS
+=head1 LICENSE
+
+Copyright (c) 2001, 2014 Sean M. Burke.
-Copyright (c) 2001 Sean M. Burke. All rights reserved.
+Unidecode is distributed under the Perl Artistic License
+( L<perlartistic> ), namely:
This library is free software; you can redistribute it and/or modify
it under the same terms as Perl itself.
@@ -389,8 +741,15 @@ This program is distributed in the hope that it will be useful, but
without any warranty; without even the implied warranty of
merchantability or fitness for a particular purpose.
+=head1 DISCLAIMER
+
Much of Text::Unidecode's internal data is based on data from The
-Unicode Consortium, with which I am unafiliated.
+Unicode Consortium, with which I am unaffiliated.
+
+The views and conclusions contained in the software and documentation
+are those of the authors/contributors and should not be interpreted as
+representing official policies, either expressed or implied, of The
+Unicode Consortium.
=head1 AUTHOR
@@ -429,4 +788,3 @@ The lyrics are:
and punctuation we adore!
# End.
-
@@ -0,0 +1,18 @@
+# -*- coding:utf-8; mode:CPerl -*-
+require 5;
+ # Note that "require [ver]" is RUNTIME, but "use [ver]" is compile-time.
+BEGIN {
+print "1..2\n";
+print '# Last Modified Time-stamp: "2014-06-17 23:12:03 MDT sburke@cpan.org"', "\n";
+print "# I'm testing absolute bare minimum sanity of our test harness.\n";
+print "# Current time GMT: ", scalar( gmtime()), "\n";
+print "# Current time local: ", scalar(localtime()), "\n";
+print "# Current perl version: $]\n";
+print "# Hello:\n";
+print "ok 1\n";
+}
+
+print "# Bye:\n";
+print "ok 2\n";
+# And that's that.
+
@@ -0,0 +1,35 @@
+# -*- coding:utf-8; mode:CPerl -*-
+require 5;
+# This file is for asserting Perl version, and basic sanity.
+
+use warnings; use strict;
+use Test;
+BEGIN {plan tests => 4;}
+BEGIN {ok 1;
+ my $modtime = q[ timestamp Time-stamp: "2014-06-17 06:18:08 MDT sburke@cpan.org" ];
+ print "# So far, we're at line ", __LINE__, " of file ", __FILE__, "\n";
+ print "# with$modtime\n#\n# I'm running under Perl v$]\n";
+}
+
+# And now:
+
+print "# Requiring version:\n";
+ok 1;
+
+
+
+require 5.8.0; # <================== THE VERSION.
+
+
+
+# ("require" is done at runtime)
+
+ok 1;
+
+print "#\n# Phrased better, I'm under Perl version $^V\n";
+
+print "# And that's that.\n";
+
+ok 1;
+
+# End
@@ -0,0 +1,17 @@
+# -*- coding:utf-8; mode:CPerl -*-
+require 5;
+use warnings; use strict; use Test;
+BEGIN {plan tests => 3;}
+BEGIN {ok 1;
+ print '# Time-stamp: "2014-06-17 23:33:34 MDT sburke@cpan.org"', "\n";
+ print "# Checking that we can do: use utf8\n";
+}
+
+use utf8;
+ok 1;
+
+print "# And that's that.\n";
+
+ok 1;
+
+# End
@@ -0,0 +1,22 @@
+# -*- coding:utf-8; mode:CPerl -*-
+use strict; use warnings; use Test;
+print q[# //Time-stamp: "2014-06-17 23:16:40 MDT sburke@cpan.org"], "\n";
+
+BEGIN {plan tests => 5;}
+
+print "# Hello:\n";
+ok 1;
+
+print "# Testing: binmode(*STDOUT, ':utf8') ...\n";
+ok binmode(*STDOUT, ':utf8');
+
+print "# Testing: binmode(*STDERR, ':utf8') ...\n";
+ok binmode(*STDERR, ':utf8');
+
+print "# Testing: binmode(*STDIN, ':utf8') ...\n";
+ok binmode(*STDIN, ':utf8');
+
+
+print "# Bye:\n";
+ok 1;
+
@@ -0,0 +1,21 @@
+# -*- coding:utf-8; mode:CPerl -*-
+use 5.8.0; use strict; use warnings; use Test;
+print q[# //Time-stamp: "2014-06-17 18:27:46 MDT sburke@cpan.org"], "\n";
+
+BEGIN {plan tests => 5};
+ok 1;
+
+# We don't need "use utf8" here. We'll check that in other versions.
+
+
+print "# Checking that xXX and x{XXXX} syntaxes work in this Perl...\n";
+
+# No actual manipulation of the xXXXX or anything here.
+# This file is just so we'll error out if we're in some ancient pre-\x{XXXX} Perl.
+
+ok "\xE1";
+ok "\x{E1}";
+ok "\x{9053}";
+
+print "# Bye:\n";
+ok 1;
@@ -0,0 +1,23 @@
+# -*- coding:utf-8; mode:CPerl -*-
+use 5.8.0; use warnings; use strict;
+use Test;
+BEGIN {plan tests => 8;}
+print q[# // Time-stamp: "2014-06-23 03:42:54 MDT sburke@cpan.org"], "\n";
+
+print "# Checking basic operations with Unicode characters...\n";
+
+use utf8;
+
+ok( binmode($_, ':utf8') ) foreach ( *STDERR, *STDIN, *STDOUT );
+
+ok length( "\x{9053}\x{5fb7}\x{7d93}" ), 3;
+ok length( "道德經" ), 3;
+
+ok "道德經", "\x{9053}\x{5fb7}\x{7d93}" ;
+
+ok "道德經", join( '', chr(0x9053), chr(0x5fb7), chr(0x7d93) );
+
+print "# Bye:\n";
+ok 1;
+
+# End
@@ -0,0 +1,20 @@
+# -*- coding:utf-8; mode:CPerl -*-
+use 5.8.0; use strict; use warnings; use Test; use utf8;
+print q[# //Time-stamp: "2014-06-17 18:26:42 MDT sburke@cpan.org"], "\n";
+
+BEGIN {plan tests => 3;}
+
+
+print "# Let's just see if a bare 'use Text::Unidecode' works:\n";
+
+ok 1;
+
+use Text::Unidecode;
+
+print "# We just loaded Unidecode version: ",
+ $Text::Unidecode::VERSION || "?", "\n";
+
+ok "a", unidecode("a"); # sanity
+
+print "# Bye:\n";
+ok 1;
@@ -0,0 +1,104 @@
+# -*- coding:utf-8; mode:CPerl -*-
+use 5.8.0; use strict; use warnings; use Test;
+BEGIN {plan tests => 12};
+print q[# // Time-stamp: "2014-07-04 02:16:00 MDT sburke@cpan.org"], "\n";
+
+# Summary of, well, things.
+ok 1;
+
+use Text::Unidecode;
+print "# Text::Unidecode version $Text::Unidecode::VERSION\n";
+ok "a", unidecode("a");
+
+{
+ my($i, $n);
+ my $f = "# ^-- \"%s\" (U+%04x)\n";
+ print "# These cause loading of those modules...\n";
+ $n = unidecode( chr($i=0x0100) ); ok $n; printf $f, $n, $i;
+ $n = unidecode( chr($i=0x0400) ); ok $n; printf $f, $n, $i;
+ $n = unidecode( chr($i=0x0f00) ); ok $n; printf $f, $n, $i;
+ $n = unidecode( chr($i=0xff00) ); ok $n; printf $f, $n, $i;
+}
+
+ok 1;
+print "#\n# Now that we've loaded Text::Unidecode, let's see what else is\n",
+ "# in \%INC by now...\n";
+{
+ my @out;
+ push @out,
+ "\n\nPerl v",
+ defined($^V) ? sprintf('%vd', $^V) : $],
+ " under $^O ",
+ (defined(&Win32::BuildNumber) and defined &Win32::BuildNumber())
+ ? ("(Win32::BuildNumber ", &Win32::BuildNumber(), ")") : (),
+ (defined $MacPerl::Version)
+ ? ("(MacPerl version $MacPerl::Version)") : (),
+ "\n"
+ ;
+
+ # Ugly code to walk the symbol tables:
+ my %v;
+ my @stack = (''); # start out in %::
+ my $this;
+ my $count = 0;
+ my $pref;
+ while(@stack) {
+ $this = shift @stack;
+ die "Too many packages?" if ++$count > 1000;
+ next if exists $v{$this};
+ next if $this eq 'main'; # %main:: is %::
+
+ #print "Peeking at $this => ${$this . '::VERSION'}\n";
+ no strict 'refs';
+ if(defined ${$this . '::VERSION'} ) {
+ $v{$this} = ${$this . '::VERSION'}
+ } elsif(
+ defined *{$this . '::ISA'} or defined &{$this . '::import'}
+ or ($this ne '' and grep defined *{$_}{'CODE'}, values %{$this . "::"})
+ # If it has an ISA, an import, or any subs...
+ ) {
+ # It's a class/module with no version.
+ $v{$this} = undef;
+ } else {
+ # It's probably an unpopulated package.
+ ## $v{$this} = '...';
+ }
+
+ $pref = length($this) ? "$this\::" : '';
+ push @stack, map m/^(.+)::$/ ? "$pref$1" : (), keys %{$this . '::'};
+ #print "Stack: @stack\n";
+ }
+ push @out, " Modules in memory:\n";
+ delete @v{'', '[none]'};
+ foreach my $p (sort {lc($a) cmp lc($b)} keys %v) {
+ my $indent = ' ' x (2 + ($p =~ tr/:/:/));
+ push @out, ' ', $indent, $p, defined($v{$p}) ? " v$v{$p};\n" : ";\n";
+ }
+ push @out, sprintf "[at %s (local) / %s (GMT)]\n",
+ scalar(gmtime), scalar(localtime);
+ my $x = join '', @out;
+ $x =~ s/^/#/mg;
+ print $x;
+
+}
+ok 1;
+
+print "# Running",
+ (chr(65) eq 'A') ? " in an ASCII world.\n" : " in a non-ASCII world.\n",
+ "#\n",
+;
+ok 1;
+
+print "# \@INC:\n", map("# [$_]\n", @INC), "#\n#\n";
+ok 1;
+
+print "# \%INC:\n";
+foreach my $x (sort {lc($a) cmp lc($b)} keys %INC) {
+ print "# [$x] = [", $INC{$x} || '', "]\n";
+}
+
+ok 1;
+
+print "#\n#\n# Bye:\n";
+ok 1;
+
@@ -0,0 +1,111 @@
+# -*- coding:utf-8; mode:CPerl -*-
+use 5.8.0; use strict; use warnings;
+my $Time_Stamp = q[Time-stamp: "2014-07-09 06:02:47 MDT sburke@cpan.org"];
+
+my @Tables;
+BEGIN {
+ @Tables = (
+ 0x00 .. 0xD7,
+
+ # D8-to-DB = high surrogates
+ # DC-to-DF = low surrogates
+
+ 0xE0 .. 0xEF, # Private Use Area.
+ 0xF0 .. 0xF8, # Private Use Area.
+
+ 0xF9 .. 0xFF, # various stuff
+ );
+}
+
+use Test;
+BEGIN { plan tests => scalar(@Tables) };
+print qq[# // $Time_Stamp\n];
+
+use Text::Unidecode;
+print "# Text::Unidecode version $Text::Unidecode::VERSION\n";
+
+print "#\n# About to load these ", scalar(@Tables), " tables:\n";
+{
+ use Text::Wrap;
+ my $string_of_tables =
+ join(" ",
+ map( sprintf("%02x", $_), @Tables )
+ )
+ ;
+
+ print "#\n", wrap(
+ "# ", # first tab
+ "# ", # subsequent tab
+ $string_of_tables,
+ ), "\n#\n#\n";
+}
+
+
+my $nullmap = $Text::Unidecode::NULLMAP;
+
+print "# About to compare things to nullmap ($nullmap) and check uniqueness.\n";
+print "# Tables to consider: \n";
+
+
+
+my %Seen;
+Table:
+foreach my $table_number (@Tables) {
+ printf "# Loading table 0x%02x...\n", $table_number;
+
+ my $charnum = 0xFF + $table_number * 0x0100;
+
+ my($throw_away_value) = unidecode( chr( $charnum ) );
+ # ^--- is just to force
+ # the loading of that table.
+
+ my $table_for_this = $Text::Unidecode::Char[$table_number];
+ my $string_for_table_for_this = "" . $table_for_this;
+ my $hex_for_this_table_number =
+ sprintf( "%02x", $table_number );
+
+
+ my $seen_before_list =
+ ( $Seen{ $string_for_table_for_this } ||= [] );
+
+ push @{ $seen_before_list },
+ $hex_for_this_table_number;
+
+ if($table_for_this eq $nullmap) {
+ print "# Table 0x$hex_for_this_table_number is ",
+ "$string_for_table_for_this",
+ " which is NULLTABLE.\n";
+ ok 0; # loading that bank just got us a null table
+ print "#\n";
+ next Table;
+ }
+
+
+ if( @$seen_before_list > 1 ) {
+ print "# Table 0x$hex_for_this_table_number is ",
+ "$string_for_table_for_this",
+ " which is NOT UNIQUE--\n";
+ print "# I've already seen it for: ",
+ join(" ", @$seen_before_list), "\n",
+ ;
+ ok 0; # We've already seen this table before.
+ print "#\n";
+ next Table;
+ }
+
+ die "WHAT" unless @$seen_before_list == 1;
+
+ #So, it's just us, just now:
+ print "# Table 0x$hex_for_this_table_number is ",
+ "$string_for_table_for_this= unique.\n";
+ ok 1; # This table is unique (so far)
+ print "#\n";
+
+}
+
+print "#\n# \%INC:\n";
+foreach my $x (sort {lc($a) cmp lc($b)} keys %INC) {
+ print "# [$x] = [", $INC{$x} || '', "]\n";
+}
+
+print "# End.\n";
@@ -0,0 +1,30 @@
+# -*- coding:utf-8; mode:CPerl -*-
+use 5.8.0; use Test; use strict; use warnings; use utf8;
+print q[# //Time-stamp: "2014-07-04 02:16:17 MDT sburke@cpan.org"], "\n";
+
+BEGIN {plan tests => 7};
+
+use Text::Unidecode;
+print "# Text::Unidecode version $Text::Unidecode::VERSION\n";
+
+ok "a", unidecode("\xE1");
+ok "\x{e1}", "\xE1"; # check sanity
+ok "a", unidecode("\x{e1}");
+
+# Now, probably provoke the loading of Unidecode/01.pm...
+ok "oe", unidecode( "\x{153}" );
+
+ok "a", unidecode( "\x{FF41}" );
+
+{
+ print "#Now notes about \%INC ...\n";
+ my @incs;
+ for(sort grep m/unidec/i, keys %INC) {
+ push @incs, "# Package $_ from file: ", $INC{$_} || "?", "\n";
+ }
+ ok scalar(@incs);
+ print @incs;
+}
+
+ok 1;
+#End
@@ -0,0 +1,38 @@
+# -*- coding:utf-8; mode:CPerl -*-
+use 5.8.0;
+use Test; use strict; use warnings;
+BEGIN { plan tests => 11 };
+print q[# // Time-stamp: "2014-07-04 02:16:23 MDT sburke@cpan.org"], "\n";
+
+# Summary of, well, things.
+ok 1; print "# ^-- Hello\n";
+
+use utf8;
+
+ok binmode(*STDOUT, ":utf8"); print "# ^-- Testing binmode on STDOUT\n";
+ok binmode(*STDERR, ":utf8"); print "# ^-- Testing binmode on STDERR\n";
+
+use Text::Unidecode;
+print "# Text::Unidecode version $Text::Unidecode::VERSION\n";
+
+ok "a", unidecode("a"); print "# ^-- sanity\n";
+
+{
+ my($i, $n, $s);
+ my $f = "# ^-- Testing: \"%s\" (U+%04x) returns true. (\"%s\")\n";
+ #print "# Format: $f\n";
+
+ # Remember, when we call "ok" here, we're just testing for *truth*
+ # The test would fail iif Unidecode returns an undef or "" (or "0")
+ $n = unidecode( $s=chr($i= 0x0100 ) ); ok $n; printf $f => $s, $i, $n;
+ $n = unidecode( $s=chr($i= 0x0400 ) ); ok $n; printf $f => $s, $i, $n;
+ $n = unidecode( $s=chr($i= 0x0f00 ) ); ok $n; printf $f => $s, $i, $n;
+ $n = unidecode( $s=chr($i= 0x01ff ) ); ok $n; printf $f => $s, $i, $n;
+ $n = unidecode( $s=chr($i= 0x04ff ) ); ok $n; printf $f => $s, $i, $n;
+ $n = unidecode( $s=chr($i= 0x0fff ) ); ok $n; printf $f => $s, $i, $n;
+
+ #$n = unidecode( $s=chr($i=0x10000) ); ok $n; printf $f, $n, $s, $i;
+}
+
+print "# Bye:\n";
+ok 1;
@@ -0,0 +1,81 @@
+# -*- coding:utf-8; mode:CPerl -*-
+use 5.8.0;
+use Test; use strict; use warnings;
+my @Bank_Numbers;
+BEGIN { @Bank_Numbers = (1 .. 0xFF); }
+BEGIN {plan tests => 2 + 2 * @Bank_Numbers};
+ok 1;
+print q[# (Time-stamp: "2014-12-07 05:18:57 MST sburke@cpan.org")], "\n";
+print "# Loading all modules and checking fullness of each table.\n";
+
+use Text::Unidecode;
+print "# Text::Unidecode version $Text::Unidecode::VERSION\n";
+
+$| = 1;
+
+#
+# For each iteration:
+# 1: It's true that "defined unidecode $char"
+# 2: We've got an arrayref
+# 3: Arrayref is the right size
+
+
+# NOTE: this WILL have to change once we support surrogates
+
+
+Bank:
+foreach my $banknum ( @Bank_Numbers ) {
+ my $charnum = $banknum << 8;
+
+ # Shut up warnings about UTF-16 surrogate characters
+ # This is per https://rt.cpan.org/Ticket/Display.html?id=97456
+ my $char = do { no warnings 'utf8'; chr( $charnum ) };
+
+ print "# About to test banknum $banknum via charnum $charnum\:\n";
+
+ ok defined unidecode $char;
+ my $bank_arrayref = $Text::Unidecode::Char[$banknum];
+
+ unless(defined $bank_arrayref) {
+ # Gotta fake out the test-harness's count for this file.
+ ok 0;
+ print "# No \$Text::Unidecode::Char[$banknum] in memory?!\n";
+ next Bank;
+ }
+
+ ok( ref($bank_arrayref), 'ARRAY' ) or next Bank;
+
+ my $bank_name = "Bank_0x%02x";
+
+ # Let's work up a diagnostic describing this file.
+ my $diag = "That's $bank_name";
+
+
+ my $package_path = sprintf "Text/Unidecode/xx%02x.pm", $banknum;
+
+ my $bank_file = $INC{ $package_path } || '';
+
+ if($bank_file) {
+ $diag .= " ";
+ if(-e $bank_file) {
+ $diag .= " of size " . (-s $bank_file) . " bytes.";
+ } else {
+ $diag .= " (but can't find that file on disk!)";
+ }
+ } else {
+ $diag .= "-- But there is no $bank_file in \%INC!)";
+ }
+
+
+ #ok @{ }, 256, $diag;
+}
+
+#print map "$_ : $INC{$_}\n", sort keys %INC;
+
+sub diag {
+ my($banknum);
+}
+
+print "# Bye:\n";
+ok 1;
+#End
@@ -0,0 +1,24 @@
+# -*- coding:utf-8; mode:CPerl -*-
+use 5.8.0;
+use strict; use warnings; use Test; use utf8;
+print q[# //Time-stamp: "2014-07-04 02:17:29 MDT sburke@cpan.org"], "\n";
+
+BEGIN {plan
+ tests => 4
+;}
+
+use Text::Unidecode;
+print "# Text::Unidecode version $Text::Unidecode::VERSION\n";
+print "# Checking basic operations with Unicode characters...\n";
+
+ok 1;
+binmode($_, ":utf8") for (*STDOUT, *STDIN, *STDERR);
+
+ok( "\x{4e00}", "一" );
+ok( unidecode("\x{4e00}"), "Yi " );
+
+ok 1;
+
+# End
+
+
@@ -0,0 +1,26 @@
+# -*- coding:utf-8; mode:CPerl -*-
+use 5.8.0; use strict; use warnings; use Test; use utf8;
+print q[# //Time-stamp: "2014-07-04 02:21:51 MDT sburke@cpan.org"], "\n";
+
+BEGIN {plan tests => 4;}
+
+use Text::Unidecode;
+print "# Text::Unidecode version $Text::Unidecode::VERSION\n";
+
+print "# Checking various errors to be fixed...\n";
+
+ok 1;
+binmode($_, ":utf8") for (*STDOUT, *STDIN, *STDERR);
+
+
+
+ok( "\x{02b0}", "ʰ" );
+ok( unidecode( "ʰ" ), "h" );
+
+
+
+ok 1;
+
+# End
+
+
@@ -0,0 +1,143 @@
+# -*- coding:utf-8; mode:CPerl -*-
+use 5.8.0; use strict; use warnings; use Test; use utf8;
+print q[# //Time-stamp: "2014-07-22 05:40:49 MDT sburke@cpan.org"], "\n";
+
+BEGIN {plan tests => 75;}
+
+use Text::Unidecode;
+print "# Text::Unidecode version $Text::Unidecode::VERSION\n";
+
+print "# Checking various errors to be fixed...\n";
+
+ok 1;
+binmode($_, ":utf8") for (*STDOUT, *STDIN, *STDERR);
+
+# Thank you very much to
+
+ok( unidecode( "Ý" ), "Y" ); # https://rt.cpan.org/Ticket/Display.html?id=96889
+
+
+#From Tomaž Šolc superstar!
+
+
+ok( unidecode( "ij" ), "ij") ;
+ok( unidecode( "Dz" ), "Dz") ;
+ok( unidecode( "Ƞ" ), "N") ;
+ok( unidecode( "ȡ" ), "d") ;
+
+print "# Block 02: 'Latin; IPA; spacing accents', U+02xx\n";
+ok( unidecode( "ȴ" ), 'l') ; # U+0234
+ok( unidecode( "ȵ" ), 'n') ; # U+0235
+ok( unidecode( "ȶ" ), 't') ; # U+0236
+ok( unidecode( "ȷ" ), 'j') ; # U+0237
+ok( unidecode( "ȸ" ), 'db') ; # U+0238
+ok( unidecode( "ȹ" ), 'qp') ; # U+0239
+ok( unidecode( "Ⱥ" ), 'A') ; # U+023a
+ok( unidecode( "Ȼ" ), 'C') ; # U+023b
+ok( unidecode( "ȼ" ), 'c') ; # U+023c
+ok( unidecode( "Ƚ" ), 'L') ; # U+023d
+ok( unidecode( "Ⱦ" ), 'T') ; # U+023e
+ok( unidecode( "ȿ" ), 's') ; # U+023f
+ok( unidecode( "ɀ" ), 'z') ; # U+0240
+
+
+ok( unidecode( "Ƀ" ),'B') ; #U+0243
+ok( unidecode( "Ʉ" ),'U') ; #U+0244
+ok( unidecode( "Ʌ" ),'^') ; #U+0245
+ok( unidecode( "Ɇ" ),'E') ; #U+0246
+ok( unidecode( "ɇ" ),'e') ; #U+0247
+ok( unidecode( "Ɉ" ),'J') ; #U+0248
+ok( unidecode( "ɉ" ),'j') ; #U+0249
+ok( unidecode( "Ɋ" ),'q') ; #U+024a
+ok( unidecode( "ɋ" ),'q') ; #U+024b
+ok( unidecode( "Ɍ" ),'R') ; #U+024c
+ok( unidecode( "ɍ" ),'r') ; #U+024d
+ok( unidecode( "Ɏ" ),'Y') ; #U+024e
+ok( unidecode( "ɏ" ),'y') ; #U+024f
+
+ok( unidecode( "ɐ" ),'a') ; #U+0250
+
+
+ok( unidecode( "ɸ" ), 'F') ; # \x{0278}
+
+ok( unidecode( "ɹ" ), 'r') ; # \x{0279}
+ok( unidecode( "ɺ" ), 'r') ; # \x{027a}
+ok( unidecode( "ɻ" ), 'r') ; # \x{027b}
+ok( unidecode( "ɼ" ), 'r') ; # \x{027c}
+ok( unidecode( "ɽ" ), 'r') ; # \x{027d}
+ok( unidecode( "ɾ" ), 'r') ; # \x{027e}
+ok( unidecode( "ɿ" ), 'r') ; # \x{027f}
+ok( unidecode( "ʀ" ), 'R') ; # \x{0280}
+ok( unidecode( "ʁ" ), 'R') ; # \x{0281}
+
+
+
+ok( unidecode( "ʉ" ), 'u') ; # \x{0289}
+
+
+# ok( unidecode( "" ), '') ; # \x{02__}
+
+ok( unidecode( "ʌ" ), '^') ; # \x{028c}
+ok( unidecode( "ʍ" ), 'w') ; # \x{028d}
+ok( unidecode( "ʎ" ), 'y') ; # \x{028e}
+ok( unidecode( "ʏ" ), 'Y') ; # \x{028f}
+
+ok( unidecode( "ʮ" ), 'h') ; # \x{02ae}
+ok( unidecode( "ʯ" ), 'h') ; # \x{02af}
+ok( unidecode( "ʰ" ), 'h') ; # \x{02b0}
+
+# Rejecting TS's suggested mapping of "ʰ" to "k". I see what he
+# means, but it's too much of a stretch
+
+#======================================================================
+
+
+
+#ok( unidecode( "" ), '') ; # \x{02__}
+
+print "# Combining Latin letters, U+03xx\n";
+
+ok( unidecode( "ͣ" ), 'a');
+ok( unidecode( "ͤ" ), 'e');
+ok( unidecode( "ͥ" ), 'i');
+ok( unidecode( "ͦ" ), 'o');
+ok( unidecode( "ͧ" ), 'u');
+ok( unidecode( "ͨ" ), 'c');
+ok( unidecode( "ͩ" ), 'd');
+ok( unidecode( "ͪ" ), 'h');
+ok( unidecode( "ͫ" ), 'm');
+ok( unidecode( "ͬ" ), 'r');
+ok( unidecode( "ͭ" ), 't');
+ok( unidecode( "ͮ" ), 'v');
+ok( unidecode( "ͯ" ), 'x');
+
+print "# Russian things\n";
+ok( unidecode( "Е" ), 'E');
+ok( unidecode( "г" ), 'g');
+ok( unidecode( "е" ), 'e');
+
+
+print "# Stuff...\n";
+ok( unidecode( "։" ), '.');
+ # U+0589 | Armenian full stop
+
+ok( unidecode( "\x{05c0}" ), '|');
+ # U+05C0 | Hebrew punctuation paseq
+
+
+
+ok( unidecode( "ẛ" ), 's');
+ok( unidecode( "ẜ" ), 's');
+ok( unidecode( "ẝ" ), 's');
+ok( unidecode( "ẞ" ), 'Ss');
+ok( unidecode( "ẟ" ), 'd');
+
+# That's all of Tomaž's stuff before 0x20
+
+#======================================================================
+
+ok 1;
+
+# End
+
+
@@ -0,0 +1,65 @@
+# -*- coding:utf-8; mode:CPerl -*-
+use 5.8.0;
+use strict; use warnings; use Test;
+print q[# //Time-stamp: "2014-06-17 18:52:58 MDT sburke@cpan.org"], "\n";
+
+BEGIN { plan tests => 20; }
+use utf8;
+use Text::Unidecode;
+
+ok 1; print "# ^-- Hello\n";
+
+print "# Text::Unidecode v$Text::Unidecode::VERSION\n",
+ "# Perl $^V\n",
+ "# I am test file " . __FILE__ . "\n",
+ "# Starting tests...\n";
+
+ok binmode(*STDOUT, ":utf8"); print "# ^-- Testing binmode on STDOUT\n";
+ok binmode(*STDERR, ":utf8"); print "# ^-- Testing binmode on STDERR\n";
+
+# Be super-sure that our universe is sane
+print "# Testing string literals...\n";
+ok "ගඎ", "ගඎ", "Failure in comparing identity!?!?";
+ok "ගඎ" => 'ගඎ', "quoting operator brokenness!?";
+ok "ගඎ" => q[ගඎ], "quoting operator brokenness!?";
+ok "ගඎ" => qq[ගඎ], "quoting operator brokenness!?";
+ok "ගඎ" => "\x{0d9c}\x{0d8e}", "quoting operator brokenness!?";
+
+
+# Syntactic sugar for our calls to "ok"...
+my $from;
+my $be = sub { return unidecode($from); };
+
+print "# Latin-1...\n";
+ok $be, "AEneid", ($from= "Æneid");
+ok $be, "etude", ($from= "étude");
+
+print "# Chinese...\n";
+ok $be, "Bei Jing ", ($from="北亰");
+
+print "# Canadian syllabics...\n";
+ok $be, "shanana", ($from= "ᔕᓇᓇ");
+
+print "# Cherokee...\n";
+ok $be, "taliqua", ($from="ᏔᎵᏆ");
+
+print "# Syriac...\n";
+ok $be, "ptu'i", ($from= "ܦܛܽܐܺ");
+
+print "# Devanagari...\n";
+ok $be, "abhijiit", ($from= "अभिजीत");
+
+print "# Bengali...\n";
+ok $be, "abhijiit", ($from= "অভিজীত");
+
+print "# Malayalaam...\n";
+ok $be, "abhijiit", ($from= "അഭിജീത");
+ok $be, "mlyaalm", ($from= "മലയാലമ്");
+ # the Malayaalam word for "Malayaalam"
+ # Yes, if we were doing it right, that'd be "malayaalam", not "mlyaalm"
+
+print "# Japanese, astonishingly unmangled...\n";
+ok $be, "genmaiCha ", ($from="げんまい茶"); #
+
+print "# Bye:\n";
+ok 1;
@@ -0,0 +1,12 @@
+# -*- coding:utf-8; mode:CPerl -*-
+print "1..2\n";
+print '# Time-stamp: "2014-06-17 18:53:17 MDT sburke@cpan.org"', "\n";
+require 5;
+
+print "ok 1\n";
+
+print "# Bye from ", __FILE__, "\n";
+
+print "ok 2\n";
+
+#End
@@ -1,96 +0,0 @@
-
-# Before `make install' is performed this script should be runnable with
-# `make test'. After `make install' it should work as `perl test.pl'
-
-######################### We start with some black magic to print on failure.
- # Time-stamp: "2001-07-14 01:48:48 MDT"
-
-require 5.006;
- # This is a Perl program.
-use strict;
-use utf8;
-use Test;
-
-BEGIN { plan tests => 17 }
-
-use Text::Unidecode;
-
-ok 1;
-print "# Text::Unidecode v$Text::Unidecode::VERSION\n",
- "# Perl v$]\n",
- "# Starting tests...\n";
-
-my $in = "\x{0d9c}\x{0d8e}!\n";
-my($was, $should, $is);
-foreach my $line (
-
-#"# 7-bit purity tests: all chars 00 to 7F\n",
-#map(\ord($_), 0x00 .. 0x7f),
-
-"# Basic string tests\n",
-\(
- "",
- 1/10,
- "I like pie.",
- "\n",
- "\cm\cj",
- "I like pie.\n",
-),
-
-"#\n",
-"# COMPLEX TESTS\n",
-
-split(m/\n/, <<"EOTESTS"),
-<\x{C6}neid> <AEneid>
-<\x{E9}tude> <etude>
-<\x{5317}\x{4EB0}> <Bei Jing >
- ; Chinese
-<\x{1515}\x{14c7}\x{14c7}> <shanana>
- ; Canadian syllabics
-<\x{13d4}\x{13b5}\x{13c6}> <taliqua>
- ; Cherokee
-<\x{0726}\x{071b}\x{073d}\x{0710}\x{073a}> <ptu'i>
- ; Syriac
-<\x{0905}\x{092d}\x{093f}\x{091c}\x{0940}\x{0924}> <abhijiit>
- ; Devanagari
-<\x{0985}\x{09ad}\x{09bf}\x{099c}\x{09c0}\x{09a4}> <abhijiit>
- ; Bengali
-<\x{0d05}\x{0d2d}\x{0d3f}\x{0d1c}\x{0d40}\x{0d24}> <abhijiit>
- ; Malayalaam
-<\x{0d2e}\x{0d32}\x{0d2f}\x{0d3e}\x{0d32}\x{0d2e}\x{0d4d}> <mlyaalm>
- ; the Malayaalam word for "Malayaalam"
- ; Yes, if we were doing it right, that'd be "malayaalam", not "mlyaalm"
-
-<\x{3052}\x{3093}\x{307e}\x{3044}\x{8336}> <genmaiCha >
- ; Japanese, astonishingly unmangled.
-
-EOTESTS
-
-# TODO: more tests, I guess.
-
-"# End of test data\n",
-) {
- if(ref $line) { # it should pass thru untouched
- #print ref($line), "\n";
- $was = $should = $$line;
- } else {
- if($line =~ m/<(.*?)>\s*<(.*?)>/ or $line =~ m/\[(.*?)\]\s*\[(.*?)\]/ ){
- ($was, $should) = ($1,$2);
- } else {
- print $line if $line =~ m/^\s*#/s;
- next;
- }
- }
- $is = unidecode($was);
- if($should eq $is) {
- ok 1;
- for($should, $is, $was) { s/\n/\\n/g; s/\cm/\\cm/g; s/\cj/\\cj/g; }
- print " # <$was> -> <$is> (ok)\n";
- } else {
- ok 0;
- for($should, $is, $was) { s/\n/\\n/g }
- print " # <$was> -> <$is>, but should be <$should>\n";
- }
-}
-print "# Byebye\n";
-