# Iso-2022-jp implementation

**URL:** <https://www.ruby-forum.com/t/iso-2022-jp-implementation/246519>\
**Category:** Development\
**Created:** [August 7, 2008, 2:34pm UTC](https://www.ruby-forum.com/t/iso-2022-jp-implementation/246519 "2008-08-07T14:34:39Z")\
**Posts on this page:** 1\
**Page:** 1

<div class="post-metadata">

**Author:** ![Tanaka\_A](https://www.ruby-forum.com/letter_avatar_proxy/v4/letter/t/c67d28/32.png) [@Tanaka\_A](https://www.ruby-forum.com/u/Tanaka_A)\
**Post date:** [August 7, 2008, 2:34pm UTC](https://www.ruby-forum.com/t/iso-2022-jp-implementation/246519/1 "2008-08-07T14:34:39Z")

</div>

ISO-2022-JP ‚ÌŽÀ‘•‚Å‚·‚ªA0x1e ‚ª shift ‚¾‚Æ‚¢‚Á‚Ä‚Ý‚½‚èA

% ./ruby -ve ‘p “\x1e”.encode(“euc-jp”, “iso-2022-jp”)’  
ruby 1.9.0 (2008-08-07 revision 18416) [i686-linux]  
-e:1:in `encode': shift is not supported (RuntimeError) from -e:1:in`’

:invalid=\>:replace, :undef=\>:replace ‚Ì—¼•û‚ðŽw’肵‚Ä‚à—áŠO‚ªo‚½‚èA

% ./ruby -e ‘p “\e(X”.encode(“EUC-JP”, “ISO-2022-JP”,  
:invalid=\>:replace, :undef=\>:replace)’  
-e:1:in `encode': this mode is not supported (ESC ( X) (RuntimeError) from -e:1:in`’

ƒGƒ‰[ƒƒbƒZ[ƒW‚É NUL ‚ª“ü‚Á‚½‚èA

% ./ruby -e ‘p “\e”.encode(“EUC-JP”, “ISO-2022-JP”)’|& cat -v  
-e:1:in `encode': this mode is not supported (ESC ^@) (RuntimeError) from -e:1:in`’

-1 bytes left ‚Æ‚¢‚¤‰ö‚µ‚¢ƒƒbƒZ[ƒW‚ªo‚½‚èA

% ./ruby -e ‘p “\e$(Da”.encode(“EUC-JP”, “ISO-2022-JP”)’  
-e:1:in `encode': not fully converted, -1 bytes left (ArgumentError) from -e:1:in`’

‚·‚é‚̂ŁAì‚è‚È‚¨‚µ‚Ĉȉº‚̂悤‚É‚·‚é‚Ì‚Í‚Ç‚¤‚Å‚µ‚傤‚©B

# Index: enc/trans/iso2022.erb.c

— enc/trans/iso2022.erb.c (revision 0)  
+++ enc/trans/iso2022.erb.c (revision 0)  
@@ -0,0 +1,142 @@  
+#include “transcode\_data.h”  
+  
+\<%

- map = {}
- map[“1b2842”] = :func\_so # designate US-ASCII to G0.  
“ESC ( B”
- map[“1b284a”] = :func\_so # designate JIS X 0201 latin to G0.  
“ESC ( J”
- map[“1b2440”] = :func\_so # designate JIS X 0208 1978 to G0.  
“ESC $ @”
- map[“1b2442”] = :func\_so # designate JIS X 0208 1983 to G0.  
“ESC $ B”
- map["{00-0d,10-1a,1c-7f}"] = :func\_si
- 
- map\_jisx0208\_rest = {}
- map\_jisx0208\_rest["{21-7e}"] = :func\_so  
+%\>
- 

+\<%= transcode\_generate\_node(ActionMap.parse(map), “iso2022jp\_to\_eucjp”,  
[]) %\>  
+\<%= transcode\_generate\_node(ActionMap.parse(map\_jisx0208\_rest),  
“iso2022jp\_to\_eucjp\_jisx0208\_rest”, []) %\>  
+  
+static VALUE  
+fun\_si\_iso2022jp\_to\_eucjp(rb\_transcoding\* t, const unsigned char\* s,  
size\_t l)  
+{

- if (t-\>stateful[0] == 0)
- 

```
   return (VALUE)NOMAP;

```

- else if (0x21 \<= s[0] && s[0] \<= 0x7e)
- 

```
   return (VALUE)&iso2022jp_to_eucjp_jisx0208_rest;

```

- else
- 

```
   return (VALUE)INVALID;

```

+}  
+  
+static int  
+fun\_so\_iso2022jp\_to\_eucjp(rb\_transcoding\* t, const unsigned char\* s,  
size\_t l, unsigned char\* o)  
+{

- if (s[0] == 0x1b) {
- 

```
   if (s[1] == '(') {

```

- 

```
       switch (s[l-1]) {

```

- 

```
         case 'B':

```

- 

```
         case 'J':

```

- 

```
           t->stateful[0] = 0;

```

- 

```
           break;

```

- 

```
       }

```

- 

```
   }

```

- 

```
   else {

```

- 

```
       switch (s[l-1]) {

```

- 

```
         case '@':

```

- 

```
         case 'B':

```

- 

```
           t->stateful[0] = 1;

```

- 

```
           break;

```

- 

```
       }

```

- 

```
   }

```

- 

```
   return 0;

```

- }
- else {
- 

```
   o[0] = s[0] | 0x80;

```

- 

```
   o[1] = s[1] | 0x80;

```

- 

```
   return 2;

```

- }  
+}
- 

+static const rb\_transcoder  
+rb\_ISO\_2022\_JP\_to\_EUC\_JP = {

- “ISO-2022-JP”, “EUC-JP”, &iso2022jp\_to\_eucjp, 3, 0,
- NULL, fun\_si\_iso2022jp\_to\_eucjp, NULL, fun\_so\_iso2022jp\_to\_eucjp  
+};
- 

+\<%

- map\_eucjp = {
- “{0e,0f,1b}” =\> :undef,
- “{00-0d,10-1a,1c-7f}” =\> :func\_so,
- “{a1-fe}{a1-fe}” =\> :func\_so,
- “8e{a1-fe}” =\> :undef,
- “8f{a1-fe}{a1-fe}” =\> :undef,
- }  
+%\>
- 

+\<%= transcode\_generate\_node(ActionMap.parse(map\_eucjp),  
“eucjp\_to\_iso2022jp”, []) %\>  
+  
+static int  
+fun\_so\_eucjp\_to\_iso2022jp(rb\_transcoding \*t, const unsigned char \*s,  
size\_t l, unsigned char \*o)  
+{

- unsigned char \*output0 = o;
- 
- if (t-\>stateful[0] == 0) {
- 

```
   t->stateful[0] = 1; /* initialized flag */

```

- 

```
   t->stateful[1] = 1; /* ASCII mode */

```

- }
- 
- if (l != t-\>stateful[1]) {
- 

```
   if (l == 1) {

```

- 

```
       *o++ = 0x1b;

```

- 

```
       *o++ = '(';

```

- 

```
       *o++ = 'B';

```

- 

```
       t->stateful[1] = 1;

```

- 

```
   }

```

- 

```
   else {

```

- 

```
       *o++ = 0x1b;

```

- 

```
       *o++ = '$';

```

- 

```
       *o++ = 'B';

```

- 

```
       t->stateful[1] = 2;

```

- 

```
   }

```

- }
- 
- if (l == 1) {
- 

```
   *o++ = s[0] & 0x7f;

```

- }
- else {
- 

```
   *o++ = s[0] & 0x7f;

```

- 

```
   *o++ = s[1] & 0x7f;

```

- }
- 
- return o - output0;  
+}
- 

+static int  
+finish\_eucjp\_to\_iso2022jp(rb\_transcoding \*t, unsigned char \*o)  
+{

- unsigned char \*output0 = o;
- 
- if (t-\>stateful[0] == 0)
- 

```
   return 0;

```

- 
- if (t-\>stateful[1] != 1) {
- 

```
   *o++ = 0x1b;

```

- 

```
   *o++ = '(';

```

- 

```
   *o++ = 'B';

```

- 

```
   t->stateful[1] = 1;

```

- }
- 
- return o - output0;  
+}
- 

+static const rb\_transcoder  
+rb\_EUC\_JP\_to\_ISO\_2022\_JP = {

- “EUC-JP”, “ISO-2022-JP”, &eucjp\_to\_iso2022jp, 5, 0,
- NULL, NULL, NULL, fun\_so\_eucjp\_to\_iso2022jp,  
finish\_eucjp\_to\_iso2022jp  
+};
- 

+void  
+Init\_iso2022(void)  
+{

- rb\_register\_transcoder(&rb\_ISO\_2022\_JP\_to\_EUC\_JP);
- rb\_register\_transcoder(&rb\_EUC\_JP\_to\_ISO\_2022\_JP);  
+}
- 

# Index: enc/trans/japanese.erb.c

— enc/trans/japanese.erb.c (revision 18417)  
+++ enc/trans/japanese.erb.c (working copy)  
@@ -17,235 +17,8 @@  
\<%= transcode\_tblgen “UTF-8”, “EUC-JP”, [["{00-7f}", :nomap],  
\*UCS\_TO\_EUCJP\_TBL] %\>  
\<%= transcode\_tblgen “UTF-8”, “CP51932”, [["{00-7f}", :nomap],  
\*UCS\_TO\_EUCJP\_TBL] %\>

-#define ISO\_2022\_ENCODING(escseq, byte) ((escseq\<\<8)|byte)  
-enum ISO\_2022\_ESCSEQ {

- ISO\_2022\_CZD = ‘!’,
- ISO\_2022\_C1D = ‘"’,
- ISO\_2022\_GZD4 = ‘(’,
- ISO\_2022\_G1D4 = ‘)’,
- ISO\_2022\_G2D4 = ‘\*’,
- ISO\_2022\_G3D4 = ‘+’,
- ISO\_2022\_G1D6 = ‘-’,
- ISO\_2022\_G2D6 = ‘.’,
- ISO\_2022\_G3D6 = ‘/’,
- ISO\_2022\_GZDM4 = ISO\_2022\_ENCODING(’$’,’(’),
- ISO\_2022\_G1DM4 = ISO\_2022\_ENCODING(’$’,’)’),
- ISO\_2022\_G2DM4 = ISO\_2022\_ENCODING(’$’,’\*’),
- ISO\_2022\_G3DM4 = ISO\_2022\_ENCODING(’$’,’+’),
- ISO\_2022\_G1DM6 = ISO\_2022\_ENCODING(’$’,’-’),
- ISO\_2022\_G2DM6 = ISO\_2022\_ENCODING(’$’,’.’),
- ISO\_2022\_G3DM6 = ISO\_2022\_ENCODING(’$’,’/’),
- ISO\_2022\_DOCS = ISO\_2022\_ENCODING(’%’,‘I’),
- ISO\_2022\_IRR = ‘&’  
-};
- 
- 

## -#define ISO\_2022\_GZ\_ASCII ISO\_2022\_ENCODING(ISO\_2022\_GZD4, ‘B’) -#define ISO\_2022\_GZ\_JIS\_X\_0201\_Katakana ISO\_2022\_ENCODING(ISO\_2022\_GZD4, ‘I’) -#define ISO\_2022\_GZ\_JIS\_X\_0201\_Roman ISO\_2022\_ENCODING(ISO\_2022\_GZD4, ‘J’) -#define ISO\_2022\_GZ\_JIS\_C\_6226\_1978 ISO\_2022\_ENCODING(ISO\_2022\_GZDM4,’@’) -#define ISO\_2022\_GZ\_JIS\_X\_0208\_1983 ISO\_2022\_ENCODING(ISO\_2022\_GZDM4,‘B’) -#define ISO\_2022\_GZ\_JIS\_X\_0212\_1990 ISO\_2022\_ENCODING(ISO\_2022\_GZDM4,‘D’) -#define ISO\_2022\_GZ\_JIS\_X\_0213\_2000\_1 ISO\_2022\_ENCODING(ISO\_2022\_GZDM4,‘O’) -#define ISO\_2022\_GZ\_JIS\_X\_0213\_2000\_2 ISO\_2022\_ENCODING(ISO\_2022\_GZDM4,‘P’) -#define ISO\_2022\_GZ\_JIS\_X\_0213\_2004\_1 ISO\_2022\_ENCODING(ISO\_2022\_GZDM4,‘Q’)

## -#define UNSUPPORTED\_MODE TRANSCODE\_ERROR

-static int  
-get\_iso\_2022\_mode(const unsigned char \*\*in\_pos)  
-{

- int new\_mode;
- const unsigned char \*in\_p = \*in\_pos;
- switch (\*in\_p++) {
- 

```
 case '(':

```

- switch (\*in\_p++) {
- case ‘B’: case ‘I’: case ‘J’:
- 

```
 new_mode = ISO_2022_ENCODING(ISO_2022_GZD4, *(in_p-1));

```

- 

```
 break;

```

- default:
- 

```
 rb_raise(UNSUPPORTED_MODE, "this mode is not supported (ESC ( 

```

%c)", \*(in\_p-1));

- 

```
 break;

```

- }
- break;
- 

```
 case '$':

```

- switch (\*in\_p++) {
- case ‘@’: case ‘A’: case ‘B’:
- 

```
 new_mode = ISO_2022_ENCODING(ISO_2022_GZDM4, *(in_p-1));

```

- 

```
 break;

```

- case ‘(’:
- 

```
 switch (*in_p++) {

```

- 

```
   case 'D': case 'O': case 'P': case 'Q':

```

- new\_mode = ISO\_2022\_ENCODING(ISO\_2022\_GZDM4, \*(in\_p-1));
- break;
- 

```
   default:

```

- rb\_raise(UNSUPPORTED\_MODE, “this mode is not supported (ESC $ (  
%c)”, \*(in\_p-1));
- break;
- 

```
 }

```

- 

```
 break;

```

- default:
- 

```
 rb_raise(UNSUPPORTED_MODE, "this mode is not supported (ESC $ 

```

%c)", \*(in\_p-1));

- 

```
 break;

```

- }
- break;
- 

```
 default:

```

- rb\_raise(UNSUPPORTED\_MODE, “this mode is not supported (ESC %c)”,  
\*(in\_p-1));
- break;
- }
- \*in\_pos = in\_p;
- return new\_mode;  
-}
- 

-static void  
-from\_iso\_2022\_jp\_transcoder\_preprocessor(const unsigned char \*\*in\_pos,  
unsigned char \*\*out\_pos,

- 

```
      const unsigned char *in_stop, unsigned char *out_stop,

```

- 

```
      rb_transcoding *my_transcoding)

```

-{

- const rb\_transcoder \*my\_transcoder = my\_transcoding-\>transcoder;
- const unsigned char \*in\_p = \*in\_pos;
- unsigned char \*out\_p = \*out\_pos;
- int cur\_mode = ISO\_2022\_GZ\_ASCII;
- unsigned char c1;
- unsigned char \*out\_s = out\_stop - my\_transcoder-\>max\_output + 1;
- while (in\_p \< in\_stop) {
- if (out\_p \>= out\_s) {
- 

```
 int len = (out_p - *out_pos);

```

- 

```
 int new_len = (len + my_transcoder->max_output) * 2;

```

- 

```
 *out_pos = (*my_transcoding->flush_func)(my_transcoding, len, 

```

new\_len);

- 

```
 out_p = *out_pos + len;

```

- 

```
 out_s = *out_pos + new_len - my_transcoder->max_output;

```

- }
- c1 = \*in\_p++;
- if (c1 == 0x1B) {
- 

```
 cur_mode = get_iso_2022_mode(&in_p);

```

- }
- else if (c1 == 0x1E || c1 == 0x1F) {
- 

```
 /* SHIFT */

```

- 

```
 rb_raise(UNSUPPORTED_MODE, "shift is not supported");

```

- }
- else if (c1 \>= 0x80) {
- 

```
 rb_raise(TRANSCODE_ERROR, "invalid byte sequence");

```

- }
- else {
- 

```
 switch (cur_mode) {

```

- 

```
   case ISO_2022_GZ_ASCII:

```

- 

```
   case ISO_2022_GZ_JIS_X_0201_Roman:

```

- \*out\_p++ = c1;
- break;
- 

```
   case ISO_2022_GZ_JIS_X_0201_Katakana:

```

- \*out\_p++ = 0x8E;
- \*out\_p++ = c1 | 0x80;
- break;
- 

```
   case ISO_2022_GZ_JIS_X_0212_1990:

```

- \*out\_p++ = 0x8F;
- 

```
   case ISO_2022_GZ_JIS_C_6226_1978:

```

- 

```
   case ISO_2022_GZ_JIS_X_0208_1983:

```

- \*out\_p++ = c1 | 0x80;
- \*out\_p++ = \*in\_p++ | 0x80;
- break;
- 

```
 }

```

- }
- }
- /\* cleanup \*/
- \*in\_pos = in\_p;
- \*out\_pos = out\_p;  
-}
- 

-static int  
-select\_iso\_2022\_mode(unsigned char \*\*out\_pos, int new\_mode)  
-{

- unsigned char \*out\_p = \*out\_pos;
- \*out\_p++ = ‘\x1b’;
- switch (new\_mode\>\>8) {
- 

```
 case ISO_2022_GZD4:

```

- \*out\_p++ = new\_mode \>\> 8;
- \*out\_p++ = new\_mode & 0x7F;
- break;
- 

```
 case ISO_2022_GZDM4:

```

- \*out\_p++ = new\_mode \>\> 16;
- if ((new\_mode & 0x7F) != ‘@’ &&
- 

```
 (new_mode & 0x7F) != 'A' &&

```

- 

```
 (new_mode & 0x7F) != 'B')

```

- {
- 

```
 *out_p++ = (new_mode>>8) & 0x7F;

```

- }
- \*out\_p++ = new\_mode & 0x7F;
- break;
- 

```
 default:

```

- rb\_raise(UNSUPPORTED\_MODE, “this mode is not supported.”);
- break;
- }
- \*out\_pos = out\_p;
- return new\_mode;  
-}
- 

-static void  
-to\_iso\_2022\_jp\_transcoder\_postprocessor(const unsigned char \*\*in\_pos,  
unsigned char \*\*out\_pos,

- 

```
     const unsigned char *in_stop, unsigned char *out_stop,

```

- 

```
     rb_transcoding *my_transcoding)

```

-{

- const rb\_transcoder \*my\_transcoder = my\_transcoding-\>transcoder;
- const unsigned char \*in\_p = \*in\_pos;
- unsigned char \*out\_p = \*out\_pos;
- int cur\_mode = ISO\_2022\_GZ\_ASCII, new\_mode = 0;
- unsigned char next\_byte;
- unsigned char \*out\_s = out\_stop - my\_transcoder-\>max\_output + 1;
- while (in\_p \< in\_stop) {
- if (out\_p \>= out\_s) {
- 

```
 int len = (out_p - *out_pos);

```

- 

```
 int new_len = (len + my_transcoder->max_output) * 2;

```

- 

```
 *out_pos = (*my_transcoding->flush_func)(my_transcoding, len, 

```

new\_len);

- 

```
 out_p = *out_pos + len;

```

- 

```
 out_s = *out_pos + new_len - my_transcoder->max_output;

```

- }
- next\_byte = \*in\_p++;
- if (next\_byte \< 0x80) {
- 

```
 new_mode = ISO_2022_GZ_ASCII;

```

- }
- else if (next\_byte == 0x8E) {
- 

```
 new_mode = ISO_2022_GZ_JIS_X_0201_Katakana;

```

- 

```
 next_byte = *in_p++;

```

- }
- else if (next\_byte == 0x8F) {
- 

```
 new_mode = ISO_2022_GZ_JIS_X_0212_1990;

```

- 

```
 next_byte = *in_p++;

```

- }
- else {
- 

```
 new_mode = ISO_2022_GZ_JIS_X_0208_1983;

```

- }
- if (cur\_mode != new\_mode)
- 

```
 cur_mode = select_iso_2022_mode(&out_p, new_mode);

```

- if (cur\_mode \< 0xFFFF) {
- 

```
 *out_p++ = next_byte & 0x7F;

```

- }
- else {
- 

```
 *out_p++ = next_byte & 0x7F;

```

- 

```
 *out_p++ = *in_p++ & 0x7F;

```

- }
- }
- if (cur\_mode != ISO\_2022\_GZ\_ASCII)
- cur\_mode = select\_iso\_2022\_mode(&out\_p, ISO\_2022\_GZ\_ASCII);
- /\* cleanup \*/
- \*in\_pos = in\_p;
- \*out\_pos = out\_p;  
-}
- 

-static const rb\_transcoder  
-rb\_from\_ISO\_2022\_JP = {

- “ISO-2022-JP”, “UTF-8”, &from\_EUC\_JP, 8, 0,
- &from\_iso\_2022\_jp\_transcoder\_preprocessor, NULL,  
-};
- 

-static const rb\_transcoder  
-rb\_to\_ISO\_2022\_JP = {

- “UTF-8”, “ISO-2022-JP”, &to\_EUC\_JP, 8, 1,
- NULL, &to\_iso\_2022\_jp\_transcoder\_postprocessor,  
-};
- 

void  
Init\_japanese(void)  
{  
\<%= transcode\_register\_code %\>

- rb\_register\_transcoder(&rb\_from\_ISO\_2022\_JP);
- rb\_register\_transcoder(&rb\_to\_ISO\_2022\_JP);  
}  
Index: enc/trans/utf\_16\_32.erb.c  
===================================================================  
— enc/trans/utf\_16\_32.erb.c (revision 18417)  
+++ enc/trans/utf\_16\_32.erb.c (working copy)  
@@ -1,7 +1,7 @@  
#include “transcode\_data.h”

static int  
-fun\_so\_from\_utf\_16be(const unsigned char\* s, unsigned char\* o)  
+fun\_so\_from\_utf\_16be(rb\_transcoding\* t, const unsigned char\* s, size\_t  
l, unsigned char\* o)  
{  
if (!s[0] && s[1]\<0x80) {  
o[0] = s[1];  
@@ -29,7 +29,7 @@ fun\_so\_from\_utf\_16be(const unsigned char  
}

static int  
-fun\_so\_to\_utf\_16be(const unsigned char\* s, unsigned char\* o)  
+fun\_so\_to\_utf\_16be(rb\_transcoding\* t, const unsigned char\* s, size\_t l,  
unsigned char\* o)  
{  
if (!(s[0]&0x80)) {  
o[0] = 0x00;  
@@ -57,7 +57,7 @@ fun\_so\_to\_utf\_16be(const unsigned char\*  
}

static int  
-fun\_so\_from\_utf\_16le(const unsigned char\* s, unsigned char\* o)  
+fun\_so\_from\_utf\_16le(rb\_transcoding\* t, const unsigned char\* s, size\_t  
l, unsigned char\* o)  
{  
if (!s[1] && s[0]\<0x80) {  
o[0] = s[0];  
@@ -85,7 +85,7 @@ fun\_so\_from\_utf\_16le(const unsigned char  
}

static int  
-fun\_so\_to\_utf\_16le(const unsigned char\* s, unsigned char\* o)  
+fun\_so\_to\_utf\_16le(rb\_transcoding\* t, const unsigned char\* s, size\_t l,  
unsigned char\* o)  
{  
if (!(s[0]&0x80)) {  
o[1] = 0x00;  
@@ -113,7 +113,7 @@ fun\_so\_to\_utf\_16le(const unsigned char\*  
}

static int  
-fun\_so\_from\_utf\_32be(const unsigned char\* s, unsigned char\* o)  
+fun\_so\_from\_utf\_32be(rb\_transcoding\* t, const unsigned char\* s, size\_t  
l, unsigned char\* o)  
{  
if (!s[1]) {  
if (s[2]==0 && s[3]\<0x80) {  
@@ -142,7 +142,7 @@ fun\_so\_from\_utf\_32be(const unsigned char  
}

static int  
-fun\_so\_to\_utf\_32be(const unsigned char\* s, unsigned char\* o)  
+fun\_so\_to\_utf\_32be(rb\_transcoding\* t, const unsigned char\* s, size\_t l,  
unsigned char\* o)  
{  
o[0] = 0;  
if (!(s[0]&0x80)) {  
@@ -168,13 +168,13 @@ fun\_so\_to\_utf\_32be(const unsigned char\*  
}

static int  
-fun\_so\_from\_utf\_32le(const unsigned char\* s, unsigned char\* o)  
+fun\_so\_from\_utf\_32le(rb\_transcoding\* t, const unsigned char\* s, size\_t  
l, unsigned char\* o)  
{  
return 1;  
}

static int  
-fun\_so\_to\_utf\_32le(const unsigned char\* s, unsigned char\* o)  
+fun\_so\_to\_utf\_32le(rb\_transcoding\* t, const unsigned char\* s, size\_t l,  
unsigned char\* o)  
{  
return 4;  
}  
@@ -191,7 +191,7 @@ fun\_so\_to\_utf\_32le(const unsigned char\*  
static const rb\_transcoder  
rb\_from\_UTF\_16BE = {  
“UTF-16BE”, “UTF-8”, &from\_UTF\_16BE, 4, 0,

- NULL, NULL, NULL, NULL, NULL, &fun\_so\_from\_utf\_16be

- NULL, NULL, NULL, &fun\_so\_from\_utf\_16be  
};

\<%=  
@@ -217,7 +217,7 @@ rb\_from\_UTF\_16BE = {  
static const rb\_transcoder  
rb\_to\_UTF\_16BE = {  
“UTF-8”, “UTF-16BE”, &to\_UTF\_16BE, 4, 1,

- NULL, NULL, NULL, NULL, NULL, &fun\_so\_to\_utf\_16be

- NULL, NULL, NULL, &fun\_so\_to\_utf\_16be  
};

\<%=  
@@ -232,13 +232,13 @@ rb\_to\_UTF\_16BE = {  
static const rb\_transcoder  
rb\_from\_UTF\_16LE = {  
“UTF-16LE”, “UTF-8”, &from\_UTF\_16LE, 4, 0,

- NULL, NULL, NULL, NULL, NULL, &fun\_so\_from\_utf\_16le

- NULL, NULL, NULL, &fun\_so\_from\_utf\_16le  
};

static const rb\_transcoder  
rb\_to\_UTF\_16LE = {  
“UTF-8”, “UTF-16LE”, &to\_UTF\_16BE, 4, 1,

- NULL, NULL, NULL, NULL, NULL, &fun\_so\_to\_utf\_16le

- NULL, NULL, NULL, &fun\_so\_to\_utf\_16le  
};

\<%=  
@@ -254,13 +254,13 @@ rb\_to\_UTF\_16LE = {  
static const rb\_transcoder  
rb\_from\_UTF\_32BE = {  
“UTF-32BE”, “UTF-8”, &from\_UTF\_32BE, 4, 0,

- NULL, NULL, NULL, NULL, NULL, &fun\_so\_from\_utf\_32be

- NULL, NULL, NULL, &fun\_so\_from\_utf\_32be  
};

static const rb\_transcoder  
rb\_to\_UTF\_32BE = {  
“UTF-8”, “UTF-32BE”, &to\_UTF\_16BE, 4, 1,

- NULL, NULL, NULL, NULL, NULL, &fun\_so\_to\_utf\_32be

- NULL, NULL, NULL, &fun\_so\_to\_utf\_32be  
};

\<%=  
@@ -276,13 +276,13 @@ rb\_to\_UTF\_32BE = {  
static const rb\_transcoder  
rb\_from\_UTF\_32LE = {  
“UTF-32LE”, “UTF-8”, &from\_UTF\_32LE, 4, 0,

- NULL, NULL, NULL, NULL, NULL, &fun\_so\_from\_utf\_32le

- NULL, NULL, NULL, &fun\_so\_from\_utf\_32le  
};

static const rb\_transcoder  
rb\_to\_UTF\_32LE = {  
“UTF-8”, “UTF-32LE”, &to\_UTF\_16BE, 4, 1,

- NULL, NULL, NULL, NULL, NULL, &fun\_so\_to\_utf\_32le

- NULL, NULL, NULL, &fun\_so\_to\_utf\_32le  
};

# void Index: transcode\_data.h

— transcode\_data.h (revision 18417)  
+++ transcode\_data.h (working copy)  
@@ -63,6 +63,8 @@ typedef struct rb\_transcoding {  
VALUE ruby\_string\_dest; /\* the String used as the conversion  
destination,  
or NULL if something else is being converted \*/  
unsigned char \*(_flush\_func)(struct rb\_transcoding_, int, int);  
+

- unsigned char stateful[256]; /\* opaque data for stateful encoding  
\*/  
} rb\_transcoding;

/\* static structure, one per supported encoding pair \*/  
@@ -72,12 +74,11 @@ typedef struct rb\_transcoder {  
const BYTE\_LOOKUP \*conv\_tree\_start;  
int max\_output;  
int from\_utf8;

- void (_preprocessor)(const unsigned char_\*, unsigned char\*\*, const  
unsigned char\*, unsigned char\*, struct rb\_transcoding \*);
- void (_postprocessor)(const unsigned char_\*, unsigned char\*\*, const  
unsigned char\*, unsigned char\*, struct rb\_transcoding \*);
- VALUE (_func\_ii)(VALUE); /_ info -\> info \*/
- VALUE (\*func\_si)(const unsigned char _); /_ start -\> info \*/
- int (_func\_io)(VALUE, const unsigned char_); /\* info -\> output \*/
- int (_func\_so)(const unsigned char_, unsigned char\*); /\* start -\>  
output \*/

- VALUE (_func\_ii)(rb\_transcoding_, VALUE); /\* info -\> info \*/
- VALUE (_func\_si)(rb\_transcoding_, const unsigned char\*, size\_t); /\*  
start -\> info \*/
- int (_func\_io)(rb\_transcoding_, VALUE, const unsigned char\*); /\*  
info -\> output \*/
- int (_func\_so)(rb\_transcoding_, const unsigned char\*, size\_t,  
unsigned char\*); /\* start -\> output \*/
- int (_finish\_func)(rb\_transcoding_, unsigned char\*); /\* -\> output  
\*/  
} rb\_transcoder;

# void rb\_declare\_transcoder(const char \*enc1, const char \*enc2, const char \*lib); Index: tool/transcode-tblgen.rb

— tool/transcode-tblgen.rb (revision 18417)  
+++ tool/transcode-tblgen.rb (working copy)  
@@ -234,6 +234,12 @@ class ActionMap  
“UNDEF”  
when :invalid  
“INVALID”

- when :func\_ii
- 

```
 "FUNii"

```

- when :func\_si
- 

```
 "FUNsi"

```

- when :func\_io
- 

```
 "FUNio"

```

when :func\_so  
“FUNso”  
when /\A([0-9a-f][0-9a-f])\z/i  
Index: test/ruby/test\_transcode.rb  
===================================================================  
— test/ruby/test\_transcode.rb (revision 18417)  
+++ test/ruby/test\_transcode.rb (working copy)  
@@ -321,12 +321,13 @@ class TestTranscode \< Test::Unit::TestCa  
assert\_raise(RuntimeError) { “\x1b(A”.encode(“utf-8”,  
“iso-2022-jp”) }  
assert\_raise(RuntimeError) { “\x1b$(A”.encode(“utf-8”,  
“iso-2022-jp”) }  
assert\_raise(RuntimeError) { “\x1b$C”.encode(“utf-8”,  
“iso-2022-jp”) }

- assert\_raise(RuntimeError) { “\x1e”.encode(“utf-8”, “iso-2022-jp”)  
}

- assert\_raise(RuntimeError) { “\x0e”.encode(“utf-8”, “iso-2022-jp”)  
}  
assert\_raise(RuntimeError) { “\x80”.encode(“utf-8”, “iso-2022-jp”)  
}  
assert\_raise(RuntimeError) { “\x1b$(Dd!\x1b(B”.encode(“utf-8”,  
“iso-2022-jp”) }  
assert\_raise(RuntimeError) { “\u9299”.encode(“iso-2022-jp”) }

- #@@@@ TODO: the next test should actually fail, because iso-2022-jp  
does not include half-width kana
- check\_both\_ways("\uff71\uff72\uff73\uff74\uff75",  
“\x1b(I12345\x1b(B”, “iso-2022-jp”) # JIS X 0201 §¨©ª«

- assert\_raise(RuntimeError) { “\u9299”.encode(“iso-2022-jp”) }
- assert\_raise(RuntimeError) {  
“\uff71\uff72\uff73\uff74\uff75”.encode(“iso-2022-jp”) }
- assert\_raise(RuntimeError) { “\x1b(I12345\x1b(B”.encode(“utf-8”,  
“iso-2022-jp”) }  
end

# def test\_iso\_2022\_jp\_1 Index: transcode.c

— transcode.c (revision 18417)  
+++ transcode.c (working copy)  
@@ -25,53 +25,78 @@ static VALUE sym\_invalid, sym\_undef, sym

- Dispatch data and logic  
\*/

-static st\_table \*transcoder\_table, \*transcoder\_lib\_table;  
+typedef struct {

- const char \*from;
- const char \*to;
- const char _lib; /_ maybe null. it means that don’t load the  
library. \*/
- const rb\_transcoder \*transcoder;  
+} transcoder\_entry\_t;

-#define TRANSCODER\_INTERNAL\_SEPARATOR ‘\t’  
+static st\_table \*transcoder\_table;

-static char \*  
-transcoder\_key(const char \*from\_e, const char \*to\_e)  
+static transcoder\_entry\_t \*  
+make\_transcoder\_entry(const char \*from, const char \*to)  
{

- int to\_len = strlen(to\_e);
- int from\_len = strlen(from\_e);
- char \*const key = xmalloc(to\_len + from\_len + 2);

- st\_data\_t val;
- st\_table \*table2;

- memcpy(key, to\_e, to\_len);
- memcpy(key + to\_len + 1, from\_e, from\_len + 1);
- key[to\_len] = TRANSCODER\_INTERNAL\_SEPARATOR;
- return key;

- if (!st\_lookup(transcoder\_table, (st\_data\_t)from, &val)) {
- 

```
   val = (st_data_t)st_init_strcasetable();

```

- 

```
   st_add_direct(transcoder_table, (st_data_t)from, val);

```

- }
- table2 = (st\_table \*)val;
- if (!st\_lookup(table2, (st\_data\_t)to, &val)) {
- 

```
   transcoder_entry_t *entry = ALLOC(transcoder_entry_t);

```

- 

```
   entry->from = from;

```

- 

```
   entry->to = to;

```

- 

```
   entry->lib = NULL;

```

- 

```
   entry->transcoder = NULL;

```

- 

```
   val = (st_data_t)entry;

```

- 

```
   st_add_direct(table2, (st_data_t)to, val);

```

- }
- return (transcoder\_entry\_t \*)val;  
+}
- 

+static transcoder\_entry\_t \*  
+get\_transcoder\_entry(const char \*from, const char \*to)  
+{

- st\_data\_t val;
- st\_table \*table2;
- 
- if (!st\_lookup(transcoder\_table, (st\_data\_t)from, &val)) {
- 

```
   return NULL;

```

- }
- table2 = (st\_table \*)val;
- if (!st\_lookup(table2, (st\_data\_t)to, &val)) {
- 

```
   return NULL;

```

- }
- return (transcoder\_entry\_t \*)val;  
}

void  
rb\_register\_transcoder(const rb\_transcoder \*tr)  
{

- st\_data\_t k, val = 0;  
const char \*const from\_e = tr-\>from\_encoding;  
const char \*const to\_e = tr-\>to\_encoding;

- char \*const key = transcoder\_key(from\_e, to\_e);

- if (st\_lookup(transcoder\_table, (st\_data\_t)key, &val)) {

- xfree(key);

- transcoder\_entry\_t \*entry;
- 
- entry = make\_transcoder\_entry(from\_e, to\_e);
- if (entry-\>transcoder) {  
rb\_raise(rb\_eArgError, “transcoder from %s to %s has been already  
registered”,  
from\_e, to\_e);  
}

- k = (st\_data\_t)key;
- if (st\_delete(transcoder\_lib\_table, &k, &val)) {
- xfree((char \*)k);
- }
- st\_insert(transcoder\_table, (st\_data\_t)key, (st\_data\_t)tr);

- 
- entry-\>transcoder = tr;  
}

static void  
declare\_transcoder(const char \*to, const char \*from, const char \*lib)  
{

- const char \*const key = transcoder\_key(to, from);
- st\_data\_t k = (st\_data\_t)key, val;

- transcoder\_entry\_t \*entry;

- if (st\_delete(transcoder\_lib\_table, &k, &val)) {
- xfree((char \*)k);
- }
- st\_insert(transcoder\_lib\_table, (st\_data\_t)key, (st\_data\_t)lib);

- entry = make\_transcoder\_entry(from, to);
- entry-\>lib = lib;  
}

#define MAX\_TRANSCODER\_LIBNAME\_LEN 64  
@@ -90,38 +115,166 @@ rb\_declare\_transcoder(const char \*enc1,

#define encoding\_equal(enc1, enc2) (STRCASECMP(enc1, enc2) == 0)

+typedef struct search\_path\_queue\_tag {

- struct search\_path\_queue\_tag \*next;
- const char \*enc;  
+} search\_path\_queue\_t;
- 

+typedef struct {

- st\_table \*visited;
- search\_path\_queue\_t \*queue;
- search\_path\_queue\_t \*\*queue\_last\_ptr;
- const char \*base\_enc;  
+} search\_path\_bfs\_t;
- 

+static int  
+transcode\_search\_path\_i(st\_data\_t key, st\_data\_t val, st\_data\_t arg)  
+{

- const char \*to = (const char \*)key;
- search\_path\_bfs\_t \*bfs = (search\_path\_bfs\_t \*)arg;
- search\_path\_queue\_t \*q;
- 
- if (st\_lookup(bfs-\>visited, (st\_data\_t)to, &val)) {
- 

```
   return ST_CONTINUE;

```

- }
- 
- q = ALLOC(search\_path\_queue\_t);
- q-\>enc = to;
- q-\>next = NULL;
- \*bfs-\>queue\_last\_ptr = q;
- bfs-\>queue\_last\_ptr = &q-\>next;
- 
- st\_add\_direct(bfs-\>visited, (st\_data\_t)to,  
(st\_data\_t)bfs-\>base\_enc);
- return ST\_CONTINUE;  
+}
- 

+static int  
+transcode\_search\_path(const char \*from, const char \*to,

- void (\*callback)(const char \*from, const char \*to, int depth, void  
\*arg),
- void \*arg)  
+{
- search\_path\_bfs\_t bfs;
- search\_path\_queue\_t \*q;
- st\_data\_t val;
- st\_table \*table2;
- int found;
- 
- q = ALLOC(search\_path\_queue\_t);
- q-\>enc = from;
- q-\>next = NULL;
- bfs.queue\_last\_ptr = &q-\>next;
- bfs.queue = q;
- 
- bfs.visited = st\_init\_strcasetable();
- st\_add\_direct(bfs.visited, (st\_data\_t)from, (st\_data\_t)NULL);
- 
- while (bfs.queue) {
- 

```
   q = bfs.queue;

```

- 

```
   bfs.queue = q->next;

```

- 

```
   if (!bfs.queue)

```

- 

```
       bfs.queue_last_ptr = &bfs.queue;

```

- 
- 

```
   if (!st_lookup(transcoder_table, (st_data_t)q->enc, &val)) {

```

- 

```
       xfree(q);

```

- 

```
       continue;

```

- 

```
   }

```

- 

```
   table2 = (st_table *)val;

```

- 
- 

```
   if (st_lookup(table2, (st_data_t)to, &val)) {

```

- 

```
       st_add_direct(bfs.visited, (st_data_t)to, 

```

(st\_data\_t)q-\>enc);

- 

```
       xfree(q);

```

- 

```
       found = 1;

```

- 

```
       goto cleanup;

```

- 

```
   }

```

- 
- 

```
   bfs.base_enc = q->enc;

```

- 

```
   st_foreach(table2, transcode_search_path_i, (st_data_t)&bfs);

```

- 

```
   bfs.base_enc = NULL;

```

- 
- 

```
   xfree(q);

```

- }
- found = 0;
- 

+cleanup:

- while (bfs.queue) {
- 

```
   q = bfs.queue;

```

- 

```
   bfs.queue = q->next;

```

- 

```
   xfree(q);

```

- }
- 
- if (found) {
- 

```
   const char *enc = to;

```

- 

```
   int depth = 0;

```

- 

```
   while (1) {

```

- 

```
       st_lookup(bfs.visited, (st_data_t)enc, &val);

```

- 

```
       if (!val)

```

- 

```
           break;

```

- 

```
       depth++;

```

- 

```
       enc = (const char *)val;

```

- 

```
   }

```

- 

```
   enc = to;

```

- 

```
   while (1) {

```

- 

```
       st_lookup(bfs.visited, (st_data_t)enc, &val);

```

- 

```
       if (!val)

```

- 

```
           break;

```

- 

```
       callback((const char *)val, enc, --depth, arg);

```

- 

```
       enc = (const char *)val;

```

- 

```
   }

```

- }
- 
- st\_free\_table(bfs.visited);
- 
- return found;  
+}
- 

+static void  
+transcode\_dispatch\_cb(const char \*from, const char \*to, int depth, void  
\*arg)  
+{

- const rb\_transcoder \*\*first\_transcoder\_ptr = (const rb\_transcoder  
\*\*)arg;
- 
- transcoder\_entry\_t \*entry;
- 
- if (!\*first\_transcoder\_ptr)
- 

```
   return;

```

- 
- entry = get\_transcoder\_entry(from, to);
- if (!entry)
- 

```
   goto failed;

```

- 
- if (!entry-\>transcoder && entry-\>lib) {
- 

```
   const char *lib = entry->lib;

```

- 

```
   int len = strlen(lib);

```

- 

```
   char path[sizeof(transcoder_lib_prefix) + 

```

MAX\_TRANSCODER\_LIBNAME\_LEN];  
+

- 

```
   entry->lib = NULL;

```

- 
- 

```
   if (len > MAX_TRANSCODER_LIBNAME_LEN) goto failed;

```

- 

```
   memcpy(path, transcoder_lib_prefix, 

```

sizeof(transcoder\_lib\_prefix) - 1);

- 

```
   memcpy(path + sizeof(transcoder_lib_prefix) - 1, lib, len + 1);

```

- 

```
   if (!rb_require(path)) goto failed;

```

- }
- if (!entry-\>transcoder)
- 

```
   goto failed;

```

- 
- if (depth == 0)
- 

```
   *first_transcoder_ptr = entry->transcoder;

```

- 
- return;
- 

+failed:

- \*first\_transcoder\_ptr = NULL;
- return;  
+}
- 

static const rb\_transcoder \*  
transcode\_dispatch(const char \*from\_encoding, const char \*to\_encoding)  
{

- char \*const key = transcoder\_key(from\_encoding, to\_encoding);
- st\_data\_t k, val = 0;

- const rb\_transcoder \*first\_transcoder = (rb\_transcoder \*)1;

- while (!st\_lookup(transcoder\_table, (k = (st\_data\_t)key), &val) &&
- 

```
st_delete(transcoder_lib_table, &k, &val)) {

```

- const char \*const lib = (const char \*)val;
- int len = strlen(lib);
- char path[sizeof(transcoder\_lib\_prefix) +  
MAX\_TRANSCODER\_LIBNAME\_LEN];
- 
- xfree((char \*)k);
- if (len \> MAX\_TRANSCODER\_LIBNAME\_LEN) return NULL;
- memcpy(path, transcoder\_lib\_prefix, sizeof(transcoder\_lib\_prefix) -  
1);
- memcpy(path + sizeof(transcoder\_lib\_prefix) - 1, lib, len + 1);
- if (!rb\_require(path)) return NULL;
- }
- if (!val) {
- if (!st\_lookup(transcoder\_table, (st\_data\_t)key, &val)) {
- 

```
 xfree(key);

```

- 

```
 /* multistep logic, via UTF-8 */

```

- 

```
 if (!encoding_equal(from_encoding, "UTF-8") &&

```

- !encoding\_equal(to\_encoding, “UTF-8”) &&
- transcode\_dispatch(“UTF-8”, to\_encoding)) { /\* check that we have  
a second step \*/
- return transcode\_dispatch(from\_encoding, “UTF-8”); /\* return first  
step \*/
- 

```
 }

```

- 

```
 return NULL;

```

- }

- if (transcode\_search\_path(from\_encoding, to\_encoding,  
transcode\_dispatch\_cb, (void \*)&first\_transcoder)) {
- 

```
   return first_transcoder;

```

}

- xfree(key);
- return (rb\_transcoder \*)val;

- return NULL;  
}

static void  
@@ -245,17 +398,17 @@ transcode\_loop(const unsigned char \*\*in\_  
\*out\_p++ = getBT3(next\_info);  
continue;  
case FUNii:

- 

```
 next_info = (VALUE)(*my_transcoder->func_ii)(next_info);

```

- 

```
 next_info = (VALUE)(*my_transcoder->func_ii)(my_transcoding, 

```

next\_info);  
goto follow\_info;  
case FUNsi:

- 

```
 next_info = (VALUE)(*my_transcoder->func_si)(char_start);

```

- 

```
 next_info = (VALUE)(*my_transcoder->func_si)(my_transcoding, 

```

char\_start, (size\_t)(in\_p-char\_start));  
goto follow\_info;  
break;  
case FUNio:

- 

```
 out_p += (VALUE)(*my_transcoder->func_io)(next_info, out_p);

```

- 

```
 out_p += (VALUE)(*my_transcoder->func_io)(my_transcoding, 

```

next\_info, out\_p);  
break;  
case FUNso:

- 

```
 out_p += (VALUE)(*my_transcoder->func_so)(char_start, out_p);

```

- 

```
 out_p += (VALUE)(*my_transcoder->func_so)(my_transcoding, 

```

char\_start, (size\_t)(in\_p-char\_start), out\_p);  
break;  
case INVALID:  
goto invalid;  
@@ -290,6 +443,16 @@ transcode\_loop(const unsigned char \*_in\_  
continue;  
}  
/_ cleanup \*/

- if (my\_transcoder-\>finish\_func) {
- if (out\_p \>= out\_s) {
- 

```
 int len = (out_p - *out_pos);

```

- 

```
 int new_len = (len + my_transcoder->max_output) * 2;

```

- 

```
 *out_pos = (*my_transcoding->flush_func)(my_transcoding, len, 

```

new\_len);

- 

```
 out_p = *out_pos + len;

```

- 

```
 out_s = *out_pos + new_len - my_transcoder->max_output;

```

- }
- 

```
   out_p += my_transcoder->finish_func(my_transcoding, out_p);

```

- }  
\*in\_pos = in\_p;  
\*out\_pos = out\_p;  
}  
@@ -401,21 +564,8 @@ str\_transcode(int argc, VALUE \*argv, VAL  
}

my\_transcoding.transcoder = my\_transcoder;

- 

```
   memset(my_transcoding.stateful, 0, 

```

sizeof(my\_transcoding.stateful));

- if (my\_transcoder-\>preprocessor) {
- 

```
 fromp = sp = (unsigned char *)RSTRING_PTR(str);

```

- 

```
 slen = RSTRING_LEN(str);

```

- 

```
 blen = slen + 30; /* len + margin */

```

- 

```
 dest = rb_str_tmp_new(blen);

```

- 

```
 bp = (unsigned char *)RSTRING_PTR(dest);

```

- 

```
 my_transcoding.ruby_string_dest = dest;

```

- 

```
 (*my_transcoder->preprocessor)(&fromp, &bp, (sp+slen), (bp+blen), 

```

&my\_transcoding);

- 

```
 if (fromp != sp+slen) {

```

- rb\_raise(rb\_eArgError, “not fully converted, %“PRIdPTRDIFF” bytes  
left”, sp+slen-fromp);
- 

```
 }

```

- 

```
 rb_str_set_len(dest, (char *)bp - RSTRING_PTR(dest));

```

- 

```
 str = dest;

```

- }  
fromp = sp = (unsigned char _)RSTRING\_PTR(str);  
slen = RSTRING\_LEN(str);  
blen = slen + 30; /_ len + margin \*/  
@@ -431,21 +581,6 @@ str\_transcode(int argc, VALUE \*argv, VAL  
buf = (unsigned char \*)RSTRING\_PTR(dest);  
\*bp = ‘\0’;  
rb\_str\_set\_len(dest, bp - buf);
- if (my\_transcoder-\>postprocessor) {
- 

```
 str = dest;

```

- 

```
 fromp = sp = (unsigned char *)RSTRING_PTR(str);

```

- 

```
 slen = RSTRING_LEN(str);

```

- 

```
 blen = slen + 30; /* len + margin */

```

- 

```
 dest = rb_str_tmp_new(blen);

```

- 

```
 bp = (unsigned char *)RSTRING_PTR(dest);

```

- 

```
 my_transcoding.ruby_string_dest = dest;

```

- 

```
 (*my_transcoder->postprocessor)(&fromp, &bp, (sp+slen), 

```

(bp+blen), &my\_transcoding);

- 

```
 if (fromp != sp+slen) {

```

- rb\_raise(rb\_eArgError, “not fully converted, %“PRIdPTRDIFF” bytes  
left”, sp+slen-fromp);

- 

```
 }

```

- 

```
 buf = (unsigned char *)RSTRING_PTR(dest);

```

- 

```
 rb_str_set_len(dest, bp - buf);

```

- }

- transcoder\_lib\_table = st\_init\_strcasetable();