[vlc-devel] [PATCH] iconv: add local ISO_6937-2 conversion

Salah-Eddin Shaban salah at videolan.org
Wed Oct 18 18:41:01 CEST 2017


And use our local conversion on all platforms.
This also fixes "Small G with Cedilla" on both 6937 and 6937-2.

Fixes #18361, #13635
---
 src/extras/libc.c | 141 ++++++++++++++++++++++++++++++++++++++----------------
 1 file changed, 100 insertions(+), 41 deletions(-)

diff --git a/src/extras/libc.c b/src/extras/libc.c
index 12e35808f3..ad9d6bc791 100644
--- a/src/extras/libc.c
+++ b/src/extras/libc.c
@@ -47,13 +47,27 @@ typedef struct os2_iconv_t
 #endif
 
 /*****************************************************************************
- * Local conversion routine from ISO_6937 to UTF-8 charset. Support for this
- * is still missing in libiconv, hence multiple operating systems lack it.
- * The conversion table adds Euro sign (0xA4) as per ETSI EN 300 468 Annex A
+ * Conversion table from ISO_6937 to UCS-4 charset. The table adds Euro sign
+ * (0xA4) as per ETSI EN 300 468 Annex A
  *****************************************************************************/
-#ifndef __linux__
-static const uint16_t to_ucs4[128] =
+static const uint16_t ISO6937toUCS4[256] =
 {
+  /* 0x00 */ 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007,
+  /* 0x08 */ 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f,
+  /* 0x10 */ 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017,
+  /* 0x18 */ 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f,
+  /* 0x20 */ 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027,
+  /* 0x28 */ 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f,
+  /* 0x30 */ 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037,
+  /* 0x38 */ 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f,
+  /* 0x40 */ 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047,
+  /* 0x48 */ 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f,
+  /* 0x50 */ 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057,
+  /* 0x58 */ 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f,
+  /* 0x60 */ 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067,
+  /* 0x68 */ 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f,
+  /* 0x70 */ 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077,
+  /* 0x78 */ 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f,
   /* 0x80 */ 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087,
   /* 0x88 */ 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f,
   /* 0x90 */ 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097,
@@ -62,8 +76,49 @@ static const uint16_t to_ucs4[128] =
   /* 0xa8 */ 0x00a4, 0x2018, 0x201c, 0x00ab, 0x2190, 0x2191, 0x2192, 0x2193,
   /* 0xb0 */ 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00d7, 0x00b5, 0x00b6, 0x00b7,
   /* 0xb8 */ 0x00f7, 0x2019, 0x201d, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00bf,
-  /* 0xc0 */ 0x0000, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
-  /* 0xc8 */ 0xffff, 0x0000, 0xffff, 0xffff, 0x0000, 0xffff, 0xffff, 0xffff,
+  /* 0xc0 */ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+  /* 0xc8 */ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+  /* 0xd0 */ 0x2014, 0x00b9, 0x00ae, 0x00a9, 0x2122, 0x266a, 0x00ac, 0x00a6,
+  /* 0xd8 */ 0x0000, 0x0000, 0x0000, 0x0000, 0x215b, 0x215c, 0x215d, 0x215e,
+  /* 0xe0 */ 0x2126, 0x00c6, 0x00d0, 0x00aa, 0x0126, 0x0000, 0x0132, 0x013f,
+  /* 0xe8 */ 0x0141, 0x00d8, 0x0152, 0x00ba, 0x00de, 0x0166, 0x014a, 0x0149,
+  /* 0xf0 */ 0x0138, 0x00e6, 0x0111, 0x00f0, 0x0127, 0x0131, 0x0133, 0x0140,
+  /* 0xf8 */ 0x0142, 0x00f8, 0x0153, 0x00df, 0x00fe, 0x0167, 0x014b, 0x00ad
+};
+
+/*****************************************************************************
+ * Conversion table from ISO_6937-2 (6937_2:1983/Add.1:1989) to UCS-4 charset.
+ * Based on glibc's iso_6937-2.c with the following additions:
+ * U+00A0 (0xa0), U+00AC (0xd6), U+00A6 (0xd7), U+00AD (0xff)
+ *****************************************************************************/
+static const uint16_t ISO6937_2toUCS4[256] =
+{
+  /* 0x00 */ 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007,
+  /* 0x08 */ 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f,
+  /* 0x10 */ 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017,
+  /* 0x18 */ 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f,
+  /* 0x20 */ 0x0020, 0x0021, 0x0022, 0x0023, 0x00a4, 0x0025, 0x0026, 0x0027,
+  /* 0x28 */ 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f,
+  /* 0x30 */ 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037,
+  /* 0x38 */ 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f,
+  /* 0x40 */ 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047,
+  /* 0x48 */ 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f,
+  /* 0x50 */ 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057,
+  /* 0x58 */ 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f,
+  /* 0x60 */ 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067,
+  /* 0x68 */ 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f,
+  /* 0x70 */ 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077,
+  /* 0x78 */ 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f,
+  /* 0x80 */ 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087,
+  /* 0x88 */ 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f,
+  /* 0x90 */ 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097,
+  /* 0x98 */ 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f,
+  /* 0xa0 */ 0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x0024, 0x00a5, 0x0023, 0x00a7,
+  /* 0xa8 */ 0x00a4, 0x2018, 0x201c, 0x00ab, 0x2190, 0x2191, 0x2192, 0x2193,
+  /* 0xb0 */ 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00d7, 0x00b5, 0x00b6, 0x00b7,
+  /* 0xb8 */ 0x00f7, 0x2019, 0x201d, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00bf,
+  /* 0xc0 */ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+  /* 0xc8 */ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
   /* 0xd0 */ 0x2014, 0x00b9, 0x00ae, 0x00a9, 0x2122, 0x266a, 0x00ac, 0x00a6,
   /* 0xd8 */ 0x0000, 0x0000, 0x0000, 0x0000, 0x215b, 0x215c, 0x215d, 0x215e,
   /* 0xe0 */ 0x2126, 0x00c6, 0x00d0, 0x00aa, 0x0126, 0x0000, 0x0132, 0x013f,
@@ -93,7 +148,7 @@ static const uint16_t to_ucs4_comb[15][64] =
     /* 0x48 */ 0x0000, 0x00cd, 0x0000, 0x0000, 0x0139, 0x0000, 0x0143, 0x00d3,
     /* 0x50 */ 0x0000, 0x0000, 0x0154, 0x015a, 0x0000, 0x00da, 0x0000, 0x0000,
     /* 0x58 */ 0x0000, 0x00dd, 0x0179, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
-    /* 0x60 */ 0x0000, 0x00e1, 0x0000, 0x0107, 0x0000, 0x00e9, 0x0000, 0x0000,
+    /* 0x60 */ 0x0000, 0x00e1, 0x0000, 0x0107, 0x0000, 0x00e9, 0x0000, 0x0123,
     /* 0x68 */ 0x0000, 0x00ed, 0x0000, 0x0000, 0x013a, 0x0000, 0x0144, 0x00f3,
     /* 0x70 */ 0x0000, 0x0000, 0x0155, 0x015b, 0x0000, 0x00fa, 0x0000, 0x0000,
     /* 0x78 */ 0x0000, 0x00fd, 0x017a, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000
@@ -229,8 +284,13 @@ static const uint16_t to_ucs4_comb[15][64] =
   }
 };
 
+/*****************************************************************************
+ * Local conversion routine from ISO_6937 and ISO_6937-2 to UTF-8 charset.
+ * Support for this is still missing in libiconv.
+ *****************************************************************************/
 static size_t ISO6937toUTF8( const char **inbuf, size_t *inbytesleft,
-                             unsigned char **outbuf, size_t *outbytesleft )
+                             unsigned char **outbuf, size_t *outbytesleft,
+                             const uint16_t *ucs4_table )
 
 
 {
@@ -246,28 +306,9 @@ static size_t ISO6937toUTF8( const char **inbuf, size_t *inbytesleft,
     while ( iptr < iend )
     {
         unsigned char c1 = *iptr;
+        uint_fast16_t ch = ucs4_table[c1];
 
-        if( c1 < 0x80 )
-        {
-            if( optr >= oend )
-            {
-                err = E2BIG;
-                break;    /* No space in outbuf */
-            }
-            *optr++ = *iptr++;
-            continue;
-        }
-
-
-        if ( optr + 2 >= oend )
-        {
-            err = E2BIG;
-            break;        /* No space in outbuf for multibyte char */
-        }
-
-        uint_fast16_t ch = to_ucs4[c1 - 0x80];
-
-        if( ch == 0xffff )
+        if( c1 >= 0xc1 && c1 <= 0xcf )
         {
             /* Composed character */
             if ( iptr + 1 >= iend )
@@ -296,14 +337,35 @@ static size_t ISO6937toUTF8( const char **inbuf, size_t *inbytesleft,
             iptr++;
         }
 
-        if ( ch < 0x800 )
+        if( ch < 0x80 )
+        {
+            if( optr >= oend )
+            {
+                err = E2BIG;
+                break;    /* No space in outbuf */
+            }
+            *optr++ = ch;
+        }
+        else if ( ch < 0x800 )
         {
+            if ( optr + 1 >= oend )
+            {
+                err = E2BIG;
+                break;
+            }
+
             optr[1] = 0x80 | (ch & 0x3f);
             optr[0] = 0xc0 | (ch >> 6);
             optr +=2;
         }
         else
         {
+            if ( optr + 2 >= oend )
+            {
+                err = E2BIG;
+                break;
+            }
+
             optr[2] = 0x80 | (ch & 0x3f);
             ch >>= 6;
             optr[1] = 0x80 | (ch & 0x3f);
@@ -326,17 +388,16 @@ static size_t ISO6937toUTF8( const char **inbuf, size_t *inbytesleft,
     return (size_t)(0);
 
 }
-#endif
 
 /*****************************************************************************
  * iconv wrapper
  *****************************************************************************/
 vlc_iconv_t vlc_iconv_open( const char *tocode, const char *fromcode )
 {
-#ifndef __linux__
     if( !strcasecmp(tocode, "UTF-8") && !strcasecmp(fromcode, "ISO_6937") )
         return (vlc_iconv_t)(-2);
-#endif
+    else if( !strcasecmp(tocode, "UTF-8") && !strcasecmp(fromcode, "ISO_6937-2") )
+        return (vlc_iconv_t)(-3);
 #if defined(HAVE_ICONV)
 # if defined(__OS2__) && defined(__INNOTEK_LIBC__)
     char tocode_ucs2[] = "UCS-2LE";
@@ -384,19 +445,19 @@ size_t vlc_iconv( vlc_iconv_t cd, const char **inbuf, size_t *inbytesleft,
 {
     size_t ret;
 
-#ifndef __linux__
-    if ( cd == (vlc_iconv_t)(-2) )
+    if ( cd == (vlc_iconv_t)(-2) || cd == (vlc_iconv_t)(-3) )
     {
         unsigned char *out = NULL;
         if (outbuf != NULL)
             out = (unsigned char *)*outbuf;
 
-        ret = ISO6937toUTF8( inbuf, inbytesleft, &out, outbytesleft );
+        const uint16_t *ucs4_table = ( cd == (vlc_iconv_t)(-2) ? ISO6937toUCS4 : ISO6937_2toUCS4 );
+
+        ret = ISO6937toUTF8( inbuf, inbytesleft, &out, outbytesleft, ucs4_table );
         if (outbuf != NULL)
             *outbuf = (char *)out;
     }
     else
-#endif
 #if defined(HAVE_ICONV)
     {
         ICONV_CONST char *cin = NULL;
@@ -415,10 +476,8 @@ size_t vlc_iconv( vlc_iconv_t cd, const char **inbuf, size_t *inbytesleft,
 
 int vlc_iconv_close( vlc_iconv_t cd )
 {
-#ifndef __linux__
-    if ( cd == (vlc_iconv_t)(-2) )
+    if ( cd == (vlc_iconv_t)(-2) || cd == (vlc_iconv_t)(-3) )
         return 0;
-#endif
 #if defined(HAVE_ICONV)
     return iconv_close( cd );
 #else
-- 
2.12.3



More information about the vlc-devel mailing list