code.vuplus.com Git - vuplus_xbmc/blob - xbmc/utils/test/TestCharsetConverter.cpp

   1 /*
   2  *      Copyright (C) 2005-2013 Team XBMC
   3  *      http://xbmc.org
   4  *
   5  *  This Program is free software; you can redistribute it and/or modify
   6  *  it under the terms of the GNU General Public License as published by
   7  *  the Free Software Foundation; either version 2, or (at your option)
   8  *  any later version.
   9  *
  10  *  This Program is distributed in the hope that it will be useful,
  11  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  *  GNU General Public License for more details.
  14  *
  15  *  You should have received a copy of the GNU General Public License
  16  *  along with XBMC; see the file COPYING.  If not, see
  17  *  <http://www.gnu.org/licenses/>.
  18  *
  19  */
  20
  21 #include "settings/Settings.h"
  22 #include "utils/CharsetConverter.h"
  23 #include "utils/StdString.h"
  24 #include "utils/Utf8Utils.h"
  25
  26 #include "gtest/gtest.h"
  27
  28 static const uint16_t refutf16LE1[] = { 0xff54, 0xff45, 0xff53, 0xff54,
  29                                         0xff3f, 0xff55, 0xff54, 0xff46,
  30                                         0xff11, 0xff16, 0xff2c, 0xff25,
  31                                         0xff54, 0xff4f, 0xff57, 0x0 };
  32
  33 static const uint16_t refutf16LE2[] = { 0xff54, 0xff45, 0xff53, 0xff54,
  34                                         0xff3f, 0xff55, 0xff54, 0xff46,
  35                                         0xff18, 0xff34, 0xff4f, 0xff1a,
  36                                         0xff3f, 0xff43, 0xff48, 0xff41,
  37                                         0xff52, 0xff53, 0xff45, 0xff54,
  38                                         0xff3f, 0xff35, 0xff34, 0xff26,
  39                                         0xff0d, 0xff11, 0xff16, 0xff2c,
  40                                         0xff25, 0xff0c, 0xff3f, 0xff23,
  41                                         0xff33, 0xff54, 0xff44, 0xff33,
  42                                         0xff54, 0xff52, 0xff49, 0xff4e,
  43                                         0xff47, 0xff11, 0xff16, 0x0 };
  44
  45 static const char refutf16LE3[] = "T\377E\377S\377T\377?\377S\377T\377"
  46                                   "R\377I\377N\377G\377#\377H\377A\377"
  47                                   "R\377S\377E\377T\377\064\377O\377\065"
  48                                   "\377T\377F\377\030\377";
  49
  50 static const uint16_t refutf16LE4[] = { 0xff54, 0xff45, 0xff53, 0xff54,
  51                                         0xff3f, 0xff55, 0xff54, 0xff46,
  52                                         0xff11, 0xff16, 0xff2c, 0xff25,
  53                                         0xff54, 0xff4f, 0xff35, 0xff34,
  54                                         0xff26, 0xff18, 0x0 };
  55
  56 static const uint32_t refutf32LE1[] = { 0xff54, 0xff45, 0xff53, 0xff54,
  57                                        0xff3f, 0xff55, 0xff54, 0xff46,
  58                                        0xff18, 0xff34, 0xff4f, 0xff1a,
  59                                        0xff3f, 0xff43, 0xff48, 0xff41,
  60                                        0xff52, 0xff53, 0xff45, 0xff54,
  61                                        0xff3f, 0xff35, 0xff34, 0xff26,
  62                                        0xff0d, 0xff13, 0xff12, 0xff2c,
  63                                        0xff25, 0xff0c, 0xff3f, 0xff23,
  64                                        0xff33, 0xff54, 0xff44, 0xff33,
  65                                        0xff54, 0xff52, 0xff49, 0xff4e,
  66                                        0xff47, 0xff13, 0xff12, 0xff3f,
  67                                        0x1f42d, 0x1f42e, 0x0 };
  68
  69 static const uint16_t refutf16BE[] = { 0x54ff, 0x45ff, 0x53ff, 0x54ff,
  70                                        0x3fff, 0x55ff, 0x54ff, 0x46ff,
  71                                        0x11ff, 0x16ff, 0x22ff, 0x25ff,
  72                                        0x54ff, 0x4fff, 0x35ff, 0x34ff,
  73                                        0x26ff, 0x18ff, 0x0};
  74
  75 static const uint16_t refucs2[] = { 0xff54, 0xff45, 0xff53, 0xff54,
  76                                     0xff3f, 0xff55, 0xff43, 0xff53,
  77                                     0xff12, 0xff54, 0xff4f, 0xff35,
  78                                     0xff34, 0xff26, 0xff18, 0x0 };
  79
  80 class TestCharsetConverter : public testing::Test
  81 {
  82 protected:
  83   TestCharsetConverter()
  84   {
  85     /* Add default settings for locale.
  86      * Settings here are taken from CGUISettings::Initialize()
  87      */
  88     /* TODO
  89     CSettingsCategory *loc = CSettings::Get().AddCategory(7, "locale", 14090);
  90     CSettings::Get().AddString(loc, "locale.language",248,"english",
  91                             SPIN_CONTROL_TEXT);
  92     CSettings::Get().AddString(loc, "locale.country", 20026, "USA",
  93                             SPIN_CONTROL_TEXT);
  94     CSettings::Get().AddString(loc, "locale.charset", 14091, "DEFAULT",
  95                             SPIN_CONTROL_TEXT); // charset is set by the
  96                                                 // language file
  97
  98     // Add default settings for subtitles
  99     CSettingsCategory *sub = CSettings::Get().AddCategory(5, "subtitles", 287);
 100     CSettings::Get().AddString(sub, "subtitles.charset", 735, "DEFAULT",
 101                             SPIN_CONTROL_TEXT);
 102     */
 103
 104     g_charsetConverter.reset();
 105     g_charsetConverter.clear();
 106   }
 107
 108   ~TestCharsetConverter()
 109   {
 110     CSettings::Get().Unload();
 111   }
 112
 113   CStdStringA refstra1, refstra2, varstra1;
 114   CStdStringW refstrw1, varstrw1;
 115   CStdString16 refstr16_1, varstr16_1;
 116   CStdString32 refstr32_1, varstr32_1;
 117   CStdString refstr1;
 118 };
 119
 120 TEST_F(TestCharsetConverter, utf8ToW)
 121 {
 122   refstra1 = "test utf8ToW";
 123   refstrw1 = L"test utf8ToW";
 124   varstrw1.clear();
 125   g_charsetConverter.utf8ToW(refstra1, varstrw1, true, false, NULL);
 126   EXPECT_STREQ(refstrw1.c_str(), varstrw1.c_str());
 127 }
 128
 129 TEST_F(TestCharsetConverter, utf16LEtoW)
 130 {
 131   refstrw1 = L"ｔｅｓｔ＿ｕｔｆ１６ＬＥｔｏｗ";
 132   /* TODO: Should be able to use '=' operator instead of assign() */
 133   refstr16_1.assign(refutf16LE1);
 134   varstrw1.clear();
 135   g_charsetConverter.utf16LEtoW(refstr16_1, varstrw1);
 136   EXPECT_STREQ(refstrw1.c_str(), varstrw1.c_str());
 137 }
 138
 139 TEST_F(TestCharsetConverter, subtitleCharsetToUtf8)
 140 {
 141   refstra1 = "test subtitleCharsetToW";
 142   varstra1.clear();
 143   g_charsetConverter.subtitleCharsetToUtf8(refstra1, varstra1);
 144
 145   /* Assign refstra1 to refstrw1 so that we can compare */
 146   EXPECT_STREQ(refstra1.c_str(), varstra1.c_str());
 147 }
 148
 149 TEST_F(TestCharsetConverter, utf8ToStringCharset_1)
 150 {
 151   refstra1 = "test utf8ToStringCharset";
 152   varstra1.clear();
 153   g_charsetConverter.utf8ToStringCharset(refstra1, varstra1);
 154   EXPECT_STREQ(refstra1.c_str(), varstra1.c_str());
 155 }
 156
 157 TEST_F(TestCharsetConverter, utf8ToStringCharset_2)
 158 {
 159   refstra1 = "test utf8ToStringCharset";
 160   varstra1 = "test utf8ToStringCharset";
 161   g_charsetConverter.utf8ToStringCharset(varstra1);
 162   EXPECT_STREQ(refstra1.c_str(), varstra1.c_str());
 163 }
 164
 165 TEST_F(TestCharsetConverter, utf8ToSystem)
 166 {
 167   refstra1 = "test utf8ToSystem";
 168   varstra1 = "test utf8ToSystem";
 169   g_charsetConverter.utf8ToSystem(varstra1);
 170   EXPECT_STREQ(refstra1.c_str(), varstra1.c_str());
 171 }
 172
 173 TEST_F(TestCharsetConverter, utf8To_ASCII)
 174 {
 175   refstra1 = "test utf8To: charset ASCII, CStdStringA";
 176   varstra1.clear();
 177   g_charsetConverter.utf8To("ASCII", refstra1, varstra1);
 178   EXPECT_STREQ(refstra1.c_str(), varstra1.c_str());
 179 }
 180
 181 TEST_F(TestCharsetConverter, utf8To_UTF16LE)
 182 {
 183   refstra1 = "ｔｅｓｔ＿ｕｔｆ８Ｔｏ：＿ｃｈａｒｓｅｔ＿ＵＴＦ－１６ＬＥ，＿"
 184              "ＣＳｔｄＳｔｒｉｎｇ１６";
 185   refstr16_1.assign(refutf16LE2);
 186   varstr16_1.clear();
 187   g_charsetConverter.utf8To("UTF-16LE", refstra1, varstr16_1);
 188   EXPECT_TRUE(!memcmp(refstr16_1.c_str(), varstr16_1.c_str(),
 189                       refstr16_1.length() * sizeof(uint16_t)));
 190 }
 191
 192 TEST_F(TestCharsetConverter, utf8To_UTF32LE)
 193 {
 194   refstra1 = "ｔｅｓｔ＿ｕｔｆ８Ｔｏ：＿ｃｈａｒｓｅｔ＿ＵＴＦ－３２ＬＥ，＿"
 195              "ＣＳｔｄＳｔｒｉｎｇ３２＿🐭🐮";
 196   refstr32_1.assign(refutf32LE1);
 197   varstr32_1.clear();
 198   g_charsetConverter.utf8To("UTF-32LE", refstra1, varstr32_1);
 199   EXPECT_TRUE(!memcmp(refstr32_1.c_str(), varstr32_1.c_str(),
 200                       sizeof(refutf32LE1)));
 201 }
 202
 203 TEST_F(TestCharsetConverter, stringCharsetToUtf8)
 204 {
 205   refstra1 = "ｔｅｓｔ＿ｓｔｒｉｎｇＣｈａｒｓｅｔＴｏＵｔｆ８";
 206   varstra1.clear();
 207   g_charsetConverter.ToUtf8("UTF-16LE", refutf16LE3, varstra1);
 208   EXPECT_STREQ(refstra1.c_str(), varstra1.c_str());
 209 }
 210
 211 TEST_F(TestCharsetConverter, isValidUtf8_1)
 212 {
 213   varstra1.clear();
 214   g_charsetConverter.ToUtf8("UTF-16LE", refutf16LE3, varstra1);
 215   EXPECT_TRUE(CUtf8Utils::isValidUtf8(varstra1.c_str()));
 216 }
 217
 218 TEST_F(TestCharsetConverter, isValidUtf8_2)
 219 {
 220   refstr1 = refutf16LE3;
 221   EXPECT_FALSE(CUtf8Utils::isValidUtf8(refstr1));
 222 }
 223
 224 TEST_F(TestCharsetConverter, isValidUtf8_3)
 225 {
 226   varstra1.clear();
 227   g_charsetConverter.ToUtf8("UTF-16LE", refutf16LE3, varstra1);
 228   EXPECT_TRUE(CUtf8Utils::isValidUtf8(varstra1.c_str()));
 229 }
 230
 231 TEST_F(TestCharsetConverter, isValidUtf8_4)
 232 {
 233   EXPECT_FALSE(CUtf8Utils::isValidUtf8(refutf16LE3));
 234 }
 235
 236 /* TODO: Resolve correct input/output for this function */
 237 // TEST_F(TestCharsetConverter, ucs2CharsetToStringCharset)
 238 // {
 239 //   void ucs2CharsetToStringCharset(const CStdStringW& strSource,
 240 //                                   CStdStringA& strDest, bool swap = false);
 241 // }
 242
 243 TEST_F(TestCharsetConverter, wToUTF8)
 244 {
 245   refstrw1 = L"ｔｅｓｔ＿ｗＴｏＵＴＦ８";
 246   refstra1 = "ｔｅｓｔ＿ｗＴｏＵＴＦ８";
 247   varstra1.clear();
 248   g_charsetConverter.wToUTF8(refstrw1, varstra1);
 249   EXPECT_STREQ(refstra1.c_str(), varstra1.c_str());
 250 }
 251
 252 TEST_F(TestCharsetConverter, utf16BEtoUTF8)
 253 {
 254   refstr16_1.assign(refutf16BE);
 255   refstra1 = "ｔｅｓｔ＿ｕｔｆ１６ＢＥｔｏＵＴＦ８";
 256   varstra1.clear();
 257   g_charsetConverter.utf16BEtoUTF8(refstr16_1, varstra1);
 258   EXPECT_STREQ(refstra1.c_str(), varstra1.c_str());
 259 }
 260
 261 TEST_F(TestCharsetConverter, utf16LEtoUTF8)
 262 {
 263   refstr16_1.assign(refutf16LE4);
 264   refstra1 = "ｔｅｓｔ＿ｕｔｆ１６ＬＥｔｏＵＴＦ８";
 265   varstra1.clear();
 266   g_charsetConverter.utf16LEtoUTF8(refstr16_1, varstra1);
 267   EXPECT_STREQ(refstra1.c_str(), varstra1.c_str());
 268 }
 269
 270 TEST_F(TestCharsetConverter, ucs2ToUTF8)
 271 {
 272   refstr16_1.assign(refucs2);
 273   refstra1 = "ｔｅｓｔ＿ｕｃｓ２ｔｏＵＴＦ８";
 274   varstra1.clear();
 275   g_charsetConverter.ucs2ToUTF8(refstr16_1, varstra1);
 276   EXPECT_STREQ(refstra1.c_str(), varstra1.c_str());
 277 }
 278
 279 TEST_F(TestCharsetConverter, utf8logicalToVisualBiDi)
 280 {
 281   refstra1 = "ｔｅｓｔ＿ｕｔｆ８ｌｏｇｉｃａｌＴｏＶｉｓｕａｌＢｉＤｉ";
 282   refstra2 = "ｔｅｓｔ＿ｕｔｆ８ｌｏｇｉｃａｌＴｏＶｉｓｕａｌＢｉＤｉ";
 283   varstra1.clear();
 284   g_charsetConverter.utf8logicalToVisualBiDi(refstra1, varstra1);
 285   EXPECT_STREQ(refstra2.c_str(), varstra1.c_str());
 286 }
 287
 288 /* TODO: Resolve correct input/output for this function */
 289 // TEST_F(TestCharsetConverter, utf32ToStringCharset)
 290 // {
 291 //   void utf32ToStringCharset(const unsigned long* strSource, CStdStringA& strDest);
 292 // }
 293
 294 TEST_F(TestCharsetConverter, getCharsetLabels)
 295 {
 296   std::vector<CStdString> reflabels;
 297   reflabels.push_back("Western Europe (ISO)");
 298   reflabels.push_back("Central Europe (ISO)");
 299   reflabels.push_back("South Europe (ISO)");
 300   reflabels.push_back("Baltic (ISO)");
 301   reflabels.push_back("Cyrillic (ISO)");
 302   reflabels.push_back("Arabic (ISO)");
 303   reflabels.push_back("Greek (ISO)");
 304   reflabels.push_back("Hebrew (ISO)");
 305   reflabels.push_back("Turkish (ISO)");
 306   reflabels.push_back("Central Europe (Windows)");
 307   reflabels.push_back("Cyrillic (Windows)");
 308   reflabels.push_back("Western Europe (Windows)");
 309   reflabels.push_back("Greek (Windows)");
 310   reflabels.push_back("Turkish (Windows)");
 311   reflabels.push_back("Hebrew (Windows)");
 312   reflabels.push_back("Arabic (Windows)");
 313   reflabels.push_back("Baltic (Windows)");
 314   reflabels.push_back("Vietnamesse (Windows)");
 315   reflabels.push_back("Thai (Windows)");
 316   reflabels.push_back("Chinese Traditional (Big5)");
 317   reflabels.push_back("Chinese Simplified (GBK)");
 318   reflabels.push_back("Japanese (Shift-JIS)");
 319   reflabels.push_back("Korean");
 320   reflabels.push_back("Hong Kong (Big5-HKSCS)");
 321
 322   std::vector<std::string> varlabels = g_charsetConverter.getCharsetLabels();
 323   ASSERT_EQ(reflabels.size(), varlabels.size());
 324
 325   std::vector<std::string>::iterator it;
 326   for (it = varlabels.begin(); it < varlabels.end(); it++)
 327   {
 328     EXPECT_STREQ((reflabels.at(it - varlabels.begin())).c_str(), (*it).c_str());
 329   }
 330 }
 331
 332 TEST_F(TestCharsetConverter, getCharsetLabelByName)
 333 {
 334   CStdString varstr =
 335     g_charsetConverter.getCharsetLabelByName("ISO-8859-1");
 336   EXPECT_STREQ("Western Europe (ISO)", varstr.c_str());
 337   varstr.clear();
 338   varstr = g_charsetConverter.getCharsetLabelByName("Bogus");
 339   EXPECT_STREQ("", varstr.c_str());
 340 }
 341
 342 TEST_F(TestCharsetConverter, getCharsetNameByLabel)
 343 {
 344   CStdString varstr =
 345     g_charsetConverter.getCharsetNameByLabel("Western Europe (ISO)");
 346   EXPECT_STREQ("ISO-8859-1", varstr.c_str());
 347   varstr.clear();
 348   varstr = g_charsetConverter.getCharsetNameByLabel("Bogus");
 349   EXPECT_STREQ("", varstr.c_str());
 350 }
 351
 352 TEST_F(TestCharsetConverter, unknownToUTF8_1)
 353 {
 354   refstra1 = "ｔｅｓｔ＿ｕｎｋｎｏｗｎＴｏＵＴＦ８";
 355   varstra1 = "ｔｅｓｔ＿ｕｎｋｎｏｗｎＴｏＵＴＦ８";
 356   g_charsetConverter.unknownToUTF8(varstra1);
 357   EXPECT_STREQ(refstra1.c_str(), varstra1.c_str());
 358 }
 359
 360 TEST_F(TestCharsetConverter, unknownToUTF8_2)
 361 {
 362   refstra1 = "ｔｅｓｔ＿ｕｎｋｎｏｗｎＴｏＵＴＦ８";
 363   varstra1.clear();
 364   g_charsetConverter.unknownToUTF8(refstra1, varstra1);
 365   EXPECT_STREQ(refstra1.c_str(), varstra1.c_str());
 366 }
 367
 368 TEST_F(TestCharsetConverter, toW)
 369 {
 370   refstra1 = "ｔｅｓｔ＿ｔｏＷ：＿ｃｈａｒｓｅｔ＿ＵＴＦ－１６ＬＥ";
 371   refstrw1 = L"\xBDEF\xEF94\x85BD\xBDEF\xEF93\x94BD\xBCEF\xEFBF"
 372              L"\x94BD\xBDEF\xEF8F\xB7BC\xBCEF\xEF9A\xBFBC\xBDEF"
 373              L"\xEF83\x88BD\xBDEF\xEF81\x92BD\xBDEF\xEF93\x85BD"
 374              L"\xBDEF\xEF94\xBFBC\xBCEF\xEFB5\xB4BC\xBCEF\xEFA6"
 375              L"\x8DBC\xBCEF\xEF91\x96BC\xBCEF\xEFAC\xA5BC";
 376   varstrw1.clear();
 377   g_charsetConverter.toW(refstra1, varstrw1, "UTF-16LE");
 378   EXPECT_STREQ(refstrw1.c_str(), varstrw1.c_str());
 379 }
 380
 381 TEST_F(TestCharsetConverter, fromW)
 382 {
 383   refstrw1 = L"\xBDEF\xEF94\x85BD\xBDEF\xEF93\x94BD\xBCEF\xEFBF"
 384              L"\x86BD\xBDEF\xEF92\x8FBD\xBDEF\xEF8D\xB7BC\xBCEF"
 385              L"\xEF9A\xBFBC\xBDEF\xEF83\x88BD\xBDEF\xEF81\x92BD"
 386              L"\xBDEF\xEF93\x85BD\xBDEF\xEF94\xBFBC\xBCEF\xEFB5"
 387              L"\xB4BC\xBCEF\xEFA6\x8DBC\xBCEF\xEF91\x96BC\xBCEF"
 388              L"\xEFAC\xA5BC";
 389   refstra1 = "ｔｅｓｔ＿ｆｒｏｍＷ：＿ｃｈａｒｓｅｔ＿ＵＴＦ－１６ＬＥ";
 390   varstra1.clear();
 391   g_charsetConverter.fromW(refstrw1, varstra1, "UTF-16LE");
 392   EXPECT_STREQ(refstra1.c_str(), varstra1.c_str());
 393 }