# Transcoder loading **URL:** \ **Category:** Development\ **Created:** [December 12, 2007, 8:07pm UTC](https://www.ruby-forum.com/t/transcoder-loading/246514 "2007-12-12T20:07:58Z")\ **Posts on this page:** 1\ **Page:** 1
**Author:** ![nobu](https://www.ruby-forum.com/letter_avatar_proxy/v4/letter/n/e9c0ed/32.png) [@nobu](https://www.ruby-forum.com/u/nobu)\ **Post date:** [December 12, 2007, 8:07pm UTC](https://www.ruby-forum.com/t/transcoder-loading/246514/1 "2007-12-12T20:07:58Z")
e$B$J$+$@$G$9!#e(B transcodere$B$r3HD%%i%$%V%i%j$K=P$7$F$\_$^$7$?!#e(Bext/e$B$H$OJL$K$7$?$[$&e(B e$B$,$$$$$+$b$7$l$^$;$s$,!#e(B include/ruby/transcode.he$B$He(Bext/enc/trans/iso\_8859/iso\_8859.ce$B$O!"e(B e$B$=$l$\>$le(Btranscode\_data.he$B$He(Btranscode\_data\_iso\_8859.ce$B$re(Bsvn mv e$B$7$?e(B e$B$b$N$G$9!#e(B # Index: common.mk — common.mk (revision 14198) +++ common.mk (working copy) @@ -25,4 +25,6 @@ ENCOBJS = ascii.$(OBJEXT) utf8.$(OBJEXT) +TRANSCODEOBJS = iso\_8859.$(OBJEXT) + COMMONOBJS = array.$(OBJEXT) bignum.$(OBJEXT) @@ -67,5 +69,4 @@ COMMONOBJS = array.$(OBJEXT) time.$(OBJEXT) transcode.$(OBJEXT) \ - transcode\_data\_iso\_8859.$(OBJEXT) util.$(OBJEXT) variable.$(OBJEXT) @@ -544,7 +545,6 @@ thread.$(OBJEXT): {$(VPATH)}thread.c {$( {$(VPATH)}node.h {$(VPATH)}util.h {$(VPATH)}signal.h {$(VPATH)}st.h {$(VPATH)}dln.h -transcode.$(OBJEXT): {$(VPATH)}transcode.c {$(VPATH)}transcode\_data.h {$(VPATH)}ruby.h {$(VPATH)}config.h +transcode.$(OBJEXT): {$(VPATH)}transcode.c {$(VPATH)}transcode.h {$(VPATH)}ruby.h {$(VPATH)}config.h {$(VPATH)}defines.h {$(VPATH)}intern.h {$(VPATH)}missing.h {$(VPATH)}encoding.h -transcode\_data\_iso\_8859.$(OBJEXT): {$(VPATH)}transcode\_data\_iso\_8859.c {$(VPATH)}transcode\_data.h cont.$(OBJEXT): {$(VPATH)}cont.c {$(VPATH)}eval\_intern.h {$(VPATH)}ruby.h {$(VPATH)}vm\_core.h {$(VPATH)}id.h {$(VPATH)}config.h Index: transcode.c =================================================================== — transcode.c (revision 14198) +++ transcode.c (working copy) @@ -13,7 +13,7 @@ #include “ruby/ruby.h” #include “ruby/encoding.h” - ## -#include “transcode\_data.h” +#define PType (int) +#include “ruby/transcode.h” +#include \ VALUE rb\_str\_tmp\_new(long); @@ -24,130 +24,62 @@ VALUE rb\_str\_shared\_replace(VALUE, VALUE \*/ ## -/\* extern declarations, should use some include file here \*/ -extern const BYTE\_LOOKUP from\_ISO\_8859\_1; -extern const BYTE\_LOOKUP from\_ISO\_8859\_2; -extern const BYTE\_LOOKUP from\_ISO\_8859\_3; -extern const BYTE\_LOOKUP from\_ISO\_8859\_4; -extern const BYTE\_LOOKUP from\_ISO\_8859\_5; -extern const BYTE\_LOOKUP from\_ISO\_8859\_6; -extern const BYTE\_LOOKUP from\_ISO\_8859\_7; -extern const BYTE\_LOOKUP from\_ISO\_8859\_8; -extern const BYTE\_LOOKUP from\_ISO\_8859\_9; -extern const BYTE\_LOOKUP from\_ISO\_8859\_10; -extern const BYTE\_LOOKUP from\_ISO\_8859\_11; -extern const BYTE\_LOOKUP from\_ISO\_8859\_13; -extern const BYTE\_LOOKUP from\_ISO\_8859\_14; -extern const BYTE\_LOOKUP from\_ISO\_8859\_15; ## -extern const BYTE\_LOOKUP to\_ISO\_8859\_1; -extern const BYTE\_LOOKUP to\_ISO\_8859\_2; -extern const BYTE\_LOOKUP to\_ISO\_8859\_3; -extern const BYTE\_LOOKUP to\_ISO\_8859\_4; -extern const BYTE\_LOOKUP to\_ISO\_8859\_5; -extern const BYTE\_LOOKUP to\_ISO\_8859\_6; -extern const BYTE\_LOOKUP to\_ISO\_8859\_7; -extern const BYTE\_LOOKUP to\_ISO\_8859\_8; -extern const BYTE\_LOOKUP to\_ISO\_8859\_9; -extern const BYTE\_LOOKUP to\_ISO\_8859\_10; -extern const BYTE\_LOOKUP to\_ISO\_8859\_11; -extern const BYTE\_LOOKUP to\_ISO\_8859\_13; -extern const BYTE\_LOOKUP to\_ISO\_8859\_14; -extern const BYTE\_LOOKUP to\_ISO\_8859\_15; - ## -/\* declarations probably need to go into separate header file, e.g. transcode.h \*/ -/\* static structure, one per supported encoding pair \*/ -typedef struct { - const char \*from\_encoding; - const char \*to\_encoding; - const BYTE\_LOOKUP \*conv\_tree\_start; - int max\_output; - int from\_utf8; -} transcoder; - ## -/\* todo: dynamic structure, one per conversion (stream) \*/ -/\* in the future, add some mechanism for dynamically adding stuff here _/ -#define MAX\_TRANSCODERS 29 /_ todo: fix: this number has to be adjusted by hand \*/ -static transcoder transcoder\_table[MAX\_TRANSCODERS]; +static st\_table \*transcoder\_table; + +#define TRANSCODER\_SEP ‘@’ /\* not sure why it’s not possible to do relocatable initializations _/ /_ maybe the code here can be removed (changed to simple initialization) _/ /_ if we move this to another file??? \*/ -static void -register\_transcoder(const char \*from\_e, const char \*to\_e, - const BYTE\_LOOKUP \*tree\_start, int max\_output, int from\_utf8) -{ - static int n = 0; - if (n \>= MAX\_TRANSCODERS) { - ``` /* we are initializing, is it okay to use rb_raise here? */ ``` - ``` rb_raise(rb_eRuntimeError /*change exception*/, "not enough ``` transcoder slots"); - } - transcoder\_table[n].from\_encoding = from\_e; - transcoder\_table[n].to\_encoding = to\_e; - transcoder\_table[n].conv\_tree\_start = tree\_start; - transcoder\_table[n].max\_output = max\_output; - transcoder\_table[n].from\_utf8 = from\_utf8; - - n++; -} - -static void -init\_transcoder\_table(void) +void +rb\_register\_transcoder(const char \*from\_e, const char \*to\_e, - ``` rb_transcode_loop_func *func, const void *arg, int ``` max\_output) { - register\_transcoder(“ISO-8859-1”, “UTF-8”, &from\_ISO\_8859\_1, 2, 0); - register\_transcoder(“ISO-8859-2”, “UTF-8”, &from\_ISO\_8859\_2, 2, 0); - register\_transcoder(“ISO-8859-3”, “UTF-8”, &from\_ISO\_8859\_3, 2, 0); - register\_transcoder(“ISO-8859-4”, “UTF-8”, &from\_ISO\_8859\_4, 2, 0); - register\_transcoder(“ISO-8859-5”, “UTF-8”, &from\_ISO\_8859\_5, 3, 0); - register\_transcoder(“ISO-8859-6”, “UTF-8”, &from\_ISO\_8859\_6, 2, 0); - register\_transcoder(“ISO-8859-7”, “UTF-8”, &from\_ISO\_8859\_7, 3, 0); - register\_transcoder(“ISO-8859-8”, “UTF-8”, &from\_ISO\_8859\_8, 3, 0); - register\_transcoder(“ISO-8859-9”, “UTF-8”, &from\_ISO\_8859\_9, 2, 0); - register\_transcoder(“ISO-8859-10”, “UTF-8”, &from\_ISO\_8859\_10, 3, 0); - register\_transcoder(“ISO-8859-11”, “UTF-8”, &from\_ISO\_8859\_11, 3, 0); - register\_transcoder(“ISO-8859-13”, “UTF-8”, &from\_ISO\_8859\_13, 3, 0); - register\_transcoder(“ISO-8859-14”, “UTF-8”, &from\_ISO\_8859\_14, 3, 0); - register\_transcoder(“ISO-8859-15”, “UTF-8”, &from\_ISO\_8859\_15, 3, 0); - register\_transcoder(“UTF-8”, “ISO-8859-1”, &to\_ISO\_8859\_1, 1, 1); - register\_transcoder(“UTF-8”, “ISO-8859-2”, &to\_ISO\_8859\_2, 1, 1); - register\_transcoder(“UTF-8”, “ISO-8859-3”, &to\_ISO\_8859\_3, 1, 1); - register\_transcoder(“UTF-8”, “ISO-8859-4”, &to\_ISO\_8859\_4, 1, 1); - register\_transcoder(“UTF-8”, “ISO-8859-5”, &to\_ISO\_8859\_5, 1, 1); - register\_transcoder(“UTF-8”, “ISO-8859-6”, &to\_ISO\_8859\_6, 1, 1); - register\_transcoder(“UTF-8”, “ISO-8859-7”, &to\_ISO\_8859\_7, 1, 1); - register\_transcoder(“UTF-8”, “ISO-8859-8”, &to\_ISO\_8859\_8, 1, 1); - register\_transcoder(“UTF-8”, “ISO-8859-9”, &to\_ISO\_8859\_9, 1, 1); - register\_transcoder(“UTF-8”, “ISO-8859-10”, &to\_ISO\_8859\_10, 1, 1); - register\_transcoder(“UTF-8”, “ISO-8859-11”, &to\_ISO\_8859\_11, 1, 1); - register\_transcoder(“UTF-8”, “ISO-8859-13”, &to\_ISO\_8859\_13, 1, 1); - register\_transcoder(“UTF-8”, “ISO-8859-14”, &to\_ISO\_8859\_14, 1, 1); - register\_transcoder(“UTF-8”, “ISO-8859-15”, &to\_ISO\_8859\_15, 1, 1); - register\_transcoder(NULL, NULL, NULL, 0, 0); - long from\_len = strlen(from\_e); - long to\_len = strlen(to\_e); - char \*const key = xmalloc(from\_len + to\_len + 2); - st\_data\_t val = 0; - rb\_transcoder \*my\_transcoder; - - memcpy(key, from\_e, from\_len); - memcpy(key + from\_len + 1, to\_e, to\_len + 1); - key[from\_len] = TRANSCODER\_SEP; - if (st\_lookup(transcoder\_table, (st\_data\_t)key, &val)) { - xfree(key); - rb\_raise(rb\_eArgError, “transcoder has been already registered - %s”, key); - } - my\_transcoder = ALLOC(rb\_transcoder); - my\_transcoder-\>transcode\_loop = func; - my\_transcoder-\>transcode\_arg = (void \*)arg; - my\_transcoder-\>max\_output = max\_output; - st\_insert(transcoder\_table, (st\_data\_t)key, (st\_data\_t)my\_transcoder); } - -static transcoder\* +static rb\_transcoder \* transcode\_dispatch(const char\* from\_encoding, const char\* to\_encoding) { - transcoder \*candidate = transcoder\_table; - - for (candidate = transcoder\_table; candidate-\>from\_encoding; candidate++) - ``` if (0==strcasecmp(from_encoding, candidate->from_encoding) ``` - ``` && 0==strcasecmp(to_encoding, candidate->to_encoding)) ``` - ``` break; ``` - /\* in the future, add multistep transcoding logic here \*/ - return candidate-\>from\_encoding ? candidate : NULL; - static const char prefix[] = “enc/trans/”; - long from\_len = strlen(from\_encoding); - long to\_len = strlen(to\_encoding); - char \*const path = alloca(from\_len + to\_len + sizeof(prefix) + 1); - char \*const key = path + sizeof(prefix) - 1; - st\_data\_t val = 0; - char \*s; - - memcpy(key, from\_encoding, from\_len); - memcpy(key + from\_len + 1, to\_encoding, to\_len + 1); - key[from\_len] = TRANSCODER\_SEP; - for (s = key; \*s; ++s) { - if (ISUPPER(\*s)) \*s = tolower(\*s); - } - if (!st\_lookup(transcoder\_table, (st\_data\_t)key, &val)) { - memcpy(path, prefix, sizeof(prefix) - 1); - if (!rb\_require(path)) return 0; - if (!st\_lookup(transcoder\_table, (st\_data\_t)key, &val)) { - ``` return 0; ``` - } - } - return (rb\_transcoder \*)val; } -/\* dynamic structure, one per conversion (similar to iconv\_t) _/ -/_ may carry conversion state (e.g. for iso-2022-jp) \*/ -typedef struct transcoding { - VALUE ruby\_string\_dest; /\* the String used as the conversion destination, - ``` or NULL if something else is being ``` converted \*/ - char \*(_flush\_func)(struct transcoding_, int, int); -} transcoding; - /\* @@ -157,14 +89,14 @@ static void transcode\_loop(char \*\*in\_pos, char \*\*out\_pos, char \*in\_stop, char \*out\_stop, - ``` transcoder *my_transcoder, ``` - ``` transcoding *my_transcoding) ``` - ``` const rb_transcoder *my_transcoder, ``` - ``` int from_utf8, ``` - ``` rb_transcoding *my_transcoding) ``` { char \*in\_p = \*in\_pos, \*out\_p = \*out\_pos; - const BYTE\_LOOKUP \*conv\_tree\_start = my\_transcoder-\>conv\_tree\_start; - const BYTE\_LOOKUP \*conv\_tree\_start = my\_transcoder-\>transcode\_arg; const BYTE\_LOOKUP \*next\_table; unsigned int next\_offset; VALUE next\_info; unsigned char next\_byte; - int from\_utf8 = my\_transcoder-\>from\_utf8; char \*out\_s = out\_stop - my\_transcoder-\>max\_output + 1; while (in\_p \< in\_stop) { @@ -238,4 +170,24 @@ transcode\_loop(char \*\*in\_pos, char \*\*out } +void +rb\_transcode\_loop\_from\_utf8(char \*\*in\_pos, char \*\*out\_pos, - ``` char *in_stop, char *out_stop, ``` - ``` const rb_transcoder *my_transcoder, ``` - ``` rb_transcoding *my_transcoding) ``` +{ - transcode\_loop(in\_pos, out\_pos, in\_stop, out\_stop, - ``` my_transcoder, 1, my_transcoding); ``` +} + +void +rb\_transcode\_loop\_simple(char \*\*in\_pos, char \*\*out\_pos, - ``` char *in_stop, char *out_stop, ``` - ``` const rb_transcoder *my_transcoder, ``` - ``` rb_transcoding *my_transcoding) ``` +{ - transcode\_loop(in\_pos, out\_pos, in\_stop, out\_stop, - ``` my_transcoder, 0, my_transcoding); ``` +} + /\* @@ -244,5 +196,5 @@ transcode\_loop(char \*\*in\_pos, char \*\*out static char \* -str\_transcoding\_resize(transcoding \*my\_transcoding, int len, int new\_len) +str\_transcoding\_resize(rb\_transcoding \*my\_transcoding, int len, int new\_len) { VALUE dest\_string = my\_transcoding-\>ruby\_string\_dest; @@ -261,6 +213,6 @@ str\_transcode(int argc, VALUE \*argv, VAL int from\_encidx, to\_encidx; VALUE from\_encval, to\_encval; - transcoder \*my\_transcoder; - transcoding my\_transcoding; - rb\_transcoder \*my\_transcoder; - rb\_transcoding my\_transcoding; - transcode\_loop(&fromp, &bp, (sp+slen), (bp+blen), my\_transcoder, &my\_transcoding); - my\_transcoder-\>transcode\_loop(&fromp, &bp, (sp+slen), (bp+blen), - ``` my_transcoder->transcode_arg, &my_transcoding); ``` if (fromp != sp+slen) { rb\_raise(rb\_eArgError, “not fully converted, %d bytes left”, sp+slen-fromp); @@ -375,5 +328,5 @@ void Init\_transcode(void) { - init\_transcoder\_table(); - transcoder\_table = st\_init\_strcasetable(); rb\_define\_method(rb\_cString, “encode”, rb\_str\_transcode, -1); rb\_define\_method(rb\_cString, “encode!”, rb\_str\_transcode\_bang, -1); Index: include/ruby/transcode.h =================================================================== — include/ruby/transcode.h (revision 14198) +++ include/ruby/transcode.h (working copy) @@ -1,2 +1,5 @@ +#ifndef RUBY\_TRANSCODE\_H +#define RUBY\_TRANSCODE\_H - typedef unsigned char base\_element; @@ -6,10 +9,15 @@ typedef struct byte\_lookup { } BYTE\_LOOKUP; -#ifdef TRANSCODE\_DATA +/\* dynamic structure, one per conversion (similar to iconv\_t) _/ +/_ may carry conversion state (e.g. for iso-2022-jp) \*/ +typedef struct rb\_transcoding { - VALUE ruby\_string\_dest; /\* the String used as the conversion destination, - ``` or NULL if something else is being ``` converted \*/ - char \*(_flush\_func)(struct rb\_transcoding_, int, int); +} rb\_transcoding; - +#ifndef PType /\* data file needs to treat this as a pointer, to remove warnings \*/ #define PType (const BYTE\_LOOKUP _) -#else -/_ in code, this is treated as just an integer \*/ -#define PType (int) #endif @@ -38,2 +46,23 @@ typedef struct byte\_lookup { #define THREETRAIL /\* legal but undefined if three more trailing UTF-8 \*/ +/\* static structure, one per supported encoding pair \*/ +typedef struct rb\_transcoder rb\_transcoder; + +typedef void rb\_transcode\_loop\_func(char \*\*, char \*\*, char \*, char \*, - ``` const struct rb_transcoder *, rb_transcoding *); ``` - +struct rb\_transcoder { - rb\_transcode\_loop\_func \*transcode\_loop; - void \*transcode\_arg; - int max\_output; +}; - +void rb\_transcode\_loop\_from\_utf8(char \*\*, char \*\*, char \*, char \*, - ``` const rb_transcoder *, rb_transcoding *); ``` +void rb\_transcode\_loop\_simple(char \*\*, char \*\*, char \*, char \*, - ``` const rb_transcoder *, rb_transcoding *); ``` - +void rb\_register\_transcoder(const char \*from\_e, const char \*to\_e, - ``` rb_transcode_loop_func *func, const void *arg, ``` - ``` int max_output); ``` # +#endif Index: ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid # — ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) +++ ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) @@ -0,0 +1 @@ +require “enc/trans/iso\_8859” Index: ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid # — ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) +++ ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) @@ -0,0 +1 @@ +require “enc/trans/iso\_8859” Index: ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid # — ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) +++ ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) @@ -0,0 +1 @@ +require “enc/trans/iso\_8859” Index: ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid # — ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) +++ ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) @@ -0,0 +1 @@ +require “enc/trans/iso\_8859” Index: ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid # — ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) +++ ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) @@ -0,0 +1 @@ +require “enc/trans/iso\_8859” Index: ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid # — ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) +++ ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) @@ -0,0 +1 @@ +require “enc/trans/iso\_8859” Index: ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid # — ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) +++ ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) @@ -0,0 +1 @@ +require “enc/trans/iso\_8859” Index: ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid # — ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) +++ ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) @@ -0,0 +1 @@ +require “enc/trans/iso\_8859” Index: ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid # — ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) +++ ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) @@ -0,0 +1 @@ +require “enc/trans/iso\_8859” Index: ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid # — ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) +++ ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) @@ -0,0 +1 @@ +require “enc/trans/iso\_8859” Index: ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid # — ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) +++ ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) @@ -0,0 +1 @@ +require “enc/trans/iso\_8859” Index: ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid # — ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) +++ ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) @@ -0,0 +1 @@ +require “enc/trans/iso\_8859” Index: ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid # — ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) +++ ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) @@ -0,0 +1 @@ +require “enc/trans/iso\_8859” Index: ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid # — ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) +++ ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) @@ -0,0 +1 @@ +require “enc/trans/iso\_8859” Index: ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid # — ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) +++ ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) @@ -0,0 +1 @@ +require “enc/trans/iso\_8859” Index: ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid # — ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) +++ ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) @@ -0,0 +1 @@ +require “enc/trans/iso\_8859” Index: ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid # — ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) +++ ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) @@ -0,0 +1 @@ +require “enc/trans/iso\_8859” Index: ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid # — ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) +++ ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) @@ -0,0 +1 @@ +require “enc/trans/iso\_8859” Index: ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid # — ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) +++ ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) @@ -0,0 +1 @@ +require “enc/trans/iso\_8859” Index: ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid # — ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) +++ ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) @@ -0,0 +1 @@ +require “enc/trans/iso\_8859” Index: ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid # — ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) +++ ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) @@ -0,0 +1 @@ +require “enc/trans/iso\_8859” Index: ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid # — ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) +++ ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) @@ -0,0 +1 @@ +require “enc/trans/iso\_8859” Index: ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid # — ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) +++ ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) @@ -0,0 +1 @@ +require “enc/trans/iso\_8859” Index: ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid # — ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) +++ ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) @@ -0,0 +1 @@ +require “enc/trans/iso\_8859” Index: ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid # — ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) +++ ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) @@ -0,0 +1 @@ +require “enc/trans/iso\_8859” Index: ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid # — ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) +++ ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) @@ -0,0 +1 @@ +require “enc/trans/iso\_8859” Index: ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid # — ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) +++ ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) @@ -0,0 +1 @@ +require “enc/trans/iso\_8859” Index: ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid # — ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) +++ ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) @@ -0,0 +1 @@ +require “enc/trans/iso\_8859” Index: ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid # — ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) +++ ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) @@ -0,0 +1 @@ +require “enc/trans/iso\_8859” Index: ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid # — ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) +++ ext/enc/trans/iso\_8859/lib/removed\_email\_address@domain.invalid (revision 0) @@ -0,0 +1 @@ +require “enc/trans/iso\_8859” Index: ext/enc/trans/iso\_8859/iso\_8859.c — ext/enc/trans/iso\_8859/iso\_8859.c (revision 0) +++ ext/enc/trans/iso\_8859/iso\_8859.c (working copy) @@ -1,4 +1,9 @@ -#define TRANSCODE\_DATA -#include “transcode\_data.h” +#include “ruby.h” +#include “ruby/transcode.h” + +typedef struct { - const BYTE\_LOOKUP \*conv\_tree\_start; - int max\_output; +} iso\_8859\_transcoder; static const unsigned char @@ -89,5 +94,5 @@ from\_ISO\_8859\_1\_infos[129] = { output2(’\xC3’,’\xBF’), }; -const BYTE\_LOOKUP +static const BYTE\_LOOKUP from\_ISO\_8859\_1 = { from\_ISO\_8859\_1\_offsets, @@ -182,5 +187,5 @@ to\_ISO\_8859\_1\_infos[3] = { NOMAP, &to\_ISO\_8859\_1\_C2, &to\_ISO\_8859\_1\_C3, }; -const BYTE\_LOOKUP +static const BYTE\_LOOKUP to\_ISO\_8859\_1 = { to\_ISO\_8859\_1\_offsets, @@ -275,5 +280,5 @@ from\_ISO\_8859\_2\_infos[129] = { output2(’\xCB’,’\x99’), }; -const BYTE\_LOOKUP +static const BYTE\_LOOKUP from\_ISO\_8859\_2 = { from\_ISO\_8859\_2\_offsets, @@ -418,5 +423,5 @@ to\_ISO\_8859\_2\_infos[6] = { &to\_ISO\_8859\_2\_C5, &to\_ISO\_8859\_2\_CB, }; -const BYTE\_LOOKUP +static const BYTE\_LOOKUP to\_ISO\_8859\_2 = { to\_ISO\_8859\_2\_offsets, @@ -507,5 +512,5 @@ from\_ISO\_8859\_3\_infos[122] = { output2(’\xC5’,’\x9D’), output2(’\xCB’,’\x99’), }; -const BYTE\_LOOKUP +static const BYTE\_LOOKUP from\_ISO\_8859\_3 = { from\_ISO\_8859\_3\_offsets, @@ -649,5 +654,5 @@ to\_ISO\_8859\_3\_infos[6] = { &to\_ISO\_8859\_3\_C5, &to\_ISO\_8859\_3\_CB, }; -const BYTE\_LOOKUP +static const BYTE\_LOOKUP to\_ISO\_8859\_3 = { to\_ISO\_8859\_3\_offsets, @@ -742,5 +747,5 @@ from\_ISO\_8859\_4\_infos[129] = { output2(’\xCB’,’\x99’), }; -const BYTE\_LOOKUP +static const BYTE\_LOOKUP from\_ISO\_8859\_4 = { from\_ISO\_8859\_4\_offsets, @@ -886,5 +891,5 @@ to\_ISO\_8859\_4\_infos[6] = { &to\_ISO\_8859\_4\_C5, &to\_ISO\_8859\_4\_CB, }; -const BYTE\_LOOKUP +static const BYTE\_LOOKUP to\_ISO\_8859\_4 = { to\_ISO\_8859\_4\_offsets, @@ -979,5 +984,5 @@ from\_ISO\_8859\_5\_infos[129] = { output2(’\xD1’,’\x9F’), }; -const BYTE\_LOOKUP +static const BYTE\_LOOKUP from\_ISO\_8859\_5 = { from\_ISO\_8859\_5\_offsets, @@ -1124,5 +1129,5 @@ to\_ISO\_8859\_5\_infos[5] = { &to\_ISO\_8859\_5\_E2, }; -const BYTE\_LOOKUP +static const BYTE\_LOOKUP to\_ISO\_8859\_5 = { to\_ISO\_8859\_5\_offsets, @@ -1194,5 +1199,5 @@ from\_ISO\_8859\_6\_infos[84] = { output2(’\xD9’,’\x91’), output2(’\xD9’,’\x92’), }; -const BYTE\_LOOKUP +static const BYTE\_LOOKUP from\_ISO\_8859\_6 = { from\_ISO\_8859\_6\_offsets, @@ -1293,5 +1298,5 @@ to\_ISO\_8859\_6\_infos[4] = { NOMAP, &to\_ISO\_8859\_6\_C2, &to\_ISO\_8859\_6\_D8, &to\_ISO\_8859\_6\_D9, }; -const BYTE\_LOOKUP +static const BYTE\_LOOKUP to\_ISO\_8859\_6 = { to\_ISO\_8859\_6\_offsets, @@ -1384,5 +1389,5 @@ from\_ISO\_8859\_7\_infos[126] = { output2(’\xCF’,’\x8D’), output2(’\xCF’,’\x8E’), }; -const BYTE\_LOOKUP +static const BYTE\_LOOKUP from\_ISO\_8859\_7 = { from\_ISO\_8859\_7\_offsets, @@ -1560,5 +1565,5 @@ to\_ISO\_8859\_7\_infos[6] = { &to\_ISO\_8859\_7\_CF, &to\_ISO\_8859\_7\_E2, }; -const BYTE\_LOOKUP +static const BYTE\_LOOKUP to\_ISO\_8859\_7 = { to\_ISO\_8859\_7\_offsets, @@ -1635,5 +1640,5 @@ from\_ISO\_8859\_8\_infos[93] = { output3(’\xE2’,’\x80’,’\x8F’), }; -const BYTE\_LOOKUP +static const BYTE\_LOOKUP from\_ISO\_8859\_8 = { from\_ISO\_8859\_8\_offsets, @@ -1770,5 +1775,5 @@ to\_ISO\_8859\_8\_infos[5] = { &to\_ISO\_8859\_8\_E2, }; -const BYTE\_LOOKUP +static const BYTE\_LOOKUP to\_ISO\_8859\_8 = { to\_ISO\_8859\_8\_offsets, @@ -1863,5 +1868,5 @@ from\_ISO\_8859\_9\_infos[129] = { output2(’\xC3’,’\xBF’), }; -const BYTE\_LOOKUP +static const BYTE\_LOOKUP from\_ISO\_8859\_9 = { from\_ISO\_8859\_9\_offsets, @@ -1990,5 +1995,5 @@ to\_ISO\_8859\_9\_infos[5] = { &to\_ISO\_8859\_9\_C5, }; -const BYTE\_LOOKUP +static const BYTE\_LOOKUP to\_ISO\_8859\_9 = { to\_ISO\_8859\_9\_offsets, @@ -2083,5 +2088,5 @@ from\_ISO\_8859\_10\_infos[129] = { output2(’\xC4’,’\xB8’), }; -const BYTE\_LOOKUP +static const BYTE\_LOOKUP from\_ISO\_8859\_10 = { from\_ISO\_8859\_10\_offsets, @@ -2245,5 +2250,5 @@ to\_ISO\_8859\_10\_infos[6] = { &to\_ISO\_8859\_10\_C5, &to\_ISO\_8859\_10\_E2, }; -const BYTE\_LOOKUP +static const BYTE\_LOOKUP to\_ISO\_8859\_10 = { to\_ISO\_8859\_10\_offsets, @@ -2334,5 +2339,5 @@ from\_ISO\_8859\_11\_infos[121] = { output3(’\xE0’,’\xB9’,’\x9B’), }; -const BYTE\_LOOKUP +static const BYTE\_LOOKUP from\_ISO\_8859\_11 = { from\_ISO\_8859\_11\_offsets, @@ -2459,5 +2464,5 @@ to\_ISO\_8859\_11\_infos[3] = { NOMAP, &to\_ISO\_8859\_11\_C2, &to\_ISO\_8859\_11\_E0, }; -const BYTE\_LOOKUP +static const BYTE\_LOOKUP to\_ISO\_8859\_11 = { to\_ISO\_8859\_11\_offsets, @@ -2552,5 +2557,5 @@ from\_ISO\_8859\_13\_infos[129] = { output3(’\xE2’,’\x80’,’\x99’), }; -const BYTE\_LOOKUP +static const BYTE\_LOOKUP from\_ISO\_8859\_13 = { from\_ISO\_8859\_13\_offsets, @@ -2712,5 +2717,5 @@ to\_ISO\_8859\_13\_infos[6] = { &to\_ISO\_8859\_13\_C5, &to\_ISO\_8859\_13\_E2, }; -const BYTE\_LOOKUP +static const BYTE\_LOOKUP to\_ISO\_8859\_13 = { to\_ISO\_8859\_13\_offsets, @@ -2805,5 +2810,5 @@ from\_ISO\_8859\_14\_infos[129] = { output2(’\xC3’,’\xBF’), }; -const BYTE\_LOOKUP +static const BYTE\_LOOKUP from\_ISO\_8859\_14 = { from\_ISO\_8859\_14\_offsets, @@ -3016,5 +3021,5 @@ to\_ISO\_8859\_14\_infos[6] = { &to\_ISO\_8859\_14\_C5, &to\_ISO\_8859\_14\_E1, }; -const BYTE\_LOOKUP +static const BYTE\_LOOKUP to\_ISO\_8859\_14 = { to\_ISO\_8859\_14\_offsets, @@ -3109,5 +3114,5 @@ from\_ISO\_8859\_15\_infos[129] = { output2(’\xC3’,’\xBF’), }; -const BYTE\_LOOKUP +static const BYTE\_LOOKUP from\_ISO\_8859\_15 = { from\_ISO\_8859\_15\_offsets, @@ -3253,5 +3258,5 @@ to\_ISO\_8859\_15\_infos[5] = { &to\_ISO\_8859\_15\_E2, }; -const BYTE\_LOOKUP +static const BYTE\_LOOKUP to\_ISO\_8859\_15 = { to\_ISO\_8859\_15\_offsets, @@ -3259,2 +3264,40 @@ to\_ISO\_8859\_15 = { }; +#define register\_transcoder(from\_e, to\_e, tree\_start, max\_output, from\_utf8) \ - rb\_register\_transcoder(from\_e, to\_e, \ - ``` (from_utf8 ? rb_transcode_loop_from_utf8 : \ ``` - ``` rb_transcode_loop_simple), \ ``` - ``` tree_start, max_output) ``` - +void +Init\_iso\_8859(void) +{ - register\_transcoder(“ISO-8859-1”, “UTF-8”, &from\_ISO\_8859\_1, 2, 0); - register\_transcoder(“ISO-8859-2”, “UTF-8”, &from\_ISO\_8859\_2, 2, 0); - register\_transcoder(“ISO-8859-3”, “UTF-8”, &from\_ISO\_8859\_3, 2, 0); - register\_transcoder(“ISO-8859-4”, “UTF-8”, &from\_ISO\_8859\_4, 2, 0); - register\_transcoder(“ISO-8859-5”, “UTF-8”, &from\_ISO\_8859\_5, 3, 0); - register\_transcoder(“ISO-8859-6”, “UTF-8”, &from\_ISO\_8859\_6, 2, 0); - register\_transcoder(“ISO-8859-7”, “UTF-8”, &from\_ISO\_8859\_7, 3, 0); - register\_transcoder(“ISO-8859-8”, “UTF-8”, &from\_ISO\_8859\_8, 3, 0); - register\_transcoder(“ISO-8859-9”, “UTF-8”, &from\_ISO\_8859\_9, 2, 0); - register\_transcoder(“ISO-8859-10”, “UTF-8”, &from\_ISO\_8859\_10, 3, 0); - register\_transcoder(“ISO-8859-11”, “UTF-8”, &from\_ISO\_8859\_11, 3, 0); - register\_transcoder(“ISO-8859-13”, “UTF-8”, &from\_ISO\_8859\_13, 3, 0); - register\_transcoder(“ISO-8859-14”, “UTF-8”, &from\_ISO\_8859\_14, 3, 0); - register\_transcoder(“ISO-8859-15”, “UTF-8”, &from\_ISO\_8859\_15, 3, 0); - register\_transcoder(“UTF-8”, “ISO-8859-1”, &to\_ISO\_8859\_1, 1, 1); - register\_transcoder(“UTF-8”, “ISO-8859-2”, &to\_ISO\_8859\_2, 1, 1); - register\_transcoder(“UTF-8”, “ISO-8859-3”, &to\_ISO\_8859\_3, 1, 1); - register\_transcoder(“UTF-8”, “ISO-8859-4”, &to\_ISO\_8859\_4, 1, 1); - register\_transcoder(“UTF-8”, “ISO-8859-5”, &to\_ISO\_8859\_5, 1, 1); - register\_transcoder(“UTF-8”, “ISO-8859-6”, &to\_ISO\_8859\_6, 1, 1); - register\_transcoder(“UTF-8”, “ISO-8859-7”, &to\_ISO\_8859\_7, 1, 1); - register\_transcoder(“UTF-8”, “ISO-8859-8”, &to\_ISO\_8859\_8, 1, 1); - register\_transcoder(“UTF-8”, “ISO-8859-9”, &to\_ISO\_8859\_9, 1, 1); - register\_transcoder(“UTF-8”, “ISO-8859-10”, &to\_ISO\_8859\_10, 1, 1); - register\_transcoder(“UTF-8”, “ISO-8859-11”, &to\_ISO\_8859\_11, 1, 1); - register\_transcoder(“UTF-8”, “ISO-8859-13”, &to\_ISO\_8859\_13, 1, 1); - register\_transcoder(“UTF-8”, “ISO-8859-14”, &to\_ISO\_8859\_14, 1, 1); - register\_transcoder(“UTF-8”, “ISO-8859-15”, &to\_ISO\_8859\_15, 1, 1); +} Index: ext/enc/trans/iso\_8859/extconf.rb =================================================================== — ext/enc/trans/iso\_8859/extconf.rb (revision 0) +++ ext/enc/trans/iso\_8859/extconf.rb (revision 0) @@ -0,0 +1 @@ +create\_makefile(“enc/trans/iso\_8859”)