#include "CharsetDetection.h"
#include "utils/CharsetConverter.h"
#include "utils/StringUtils.h"
+#include "utils/Utf8Utils.h"
+#include "LangInfo.h"
+#include "utils/log.h"
/* XML declaration can be virtually any size (with many-many whitespaces)
* but for in real world we don't need to process megabytes of data
* so limit search for XML declaration to reasonable value */
const size_t CCharsetDetection::m_XmlDeclarationMaxLength = 250;
+/* According to http://www.w3.org/TR/2013/CR-html5-20130806/single-page.html#charset
+ * encoding must be placed in first 1024 bytes of document */
+const size_t CCharsetDetection::m_HtmlCharsetEndSearchPos = 1024;
+
+/* According to http://www.w3.org/TR/2013/CR-html5-20130806/single-page.html#space-character
+ * tab, LF, FF, CR or space can be used as whitespace */
+const std::string CCharsetDetection::m_HtmlWhitespaceChars("\x09\x0A\x0C\x0D\x20"); // tab, LF, FF, CR and space
std::string CCharsetDetection::GetBomEncoding(const char* const content, const size_t contentLength)
{
return true;
}
+bool CCharsetDetection::ConvertHtmlToUtf8(const std::string& htmlContent, std::string& converted, const std::string& serverReportedCharset, std::string& usedHtmlCharset)
+{
+ converted.clear();
+ usedHtmlCharset.clear();
+ if (htmlContent.empty())
+ {
+ usedHtmlCharset = "UTF-8"; // any charset can be used for empty content, use UTF-8 as default
+ return false;
+ }
+
+ // this is relaxed implementation of http://www.w3.org/TR/2013/CR-html5-20130806/single-page.html#determining-the-character-encoding
+
+ // try to get charset from Byte Order Mark
+ std::string bomCharset(GetBomEncoding(htmlContent));
+ if (checkConversion(bomCharset, htmlContent, converted))
+ {
+ usedHtmlCharset = bomCharset;
+ return true;
+ }
+
+ // try charset from HTTP header (or from other out-of-band source)
+ if (checkConversion(serverReportedCharset, htmlContent, converted))
+ {
+ usedHtmlCharset = serverReportedCharset;
+ return true;
+ }
+
+ // try to find charset in HTML
+ std::string declaredCharset(GetHtmlEncodingFromHead(htmlContent));
+ if (!declaredCharset.empty())
+ {
+ if (declaredCharset.compare(0, 3, "UTF", 3) == 0)
+ declaredCharset = "UTF-8"; // charset string was found in singlebyte mode, charset can't be multibyte encoding
+ if (checkConversion(declaredCharset, htmlContent, converted))
+ {
+ usedHtmlCharset = declaredCharset;
+ return true;
+ }
+ }
+
+ // try UTF-8 if not tried before
+ if (bomCharset != "UTF-8" && serverReportedCharset != "UTF-8" && declaredCharset != "UTF-8" && checkConversion("UTF-8", htmlContent, converted))
+ {
+ usedHtmlCharset = "UTF-8";
+ return false; // only guessed value
+ }
+
+ // try user charset
+ std::string userCharset(g_langInfo.GetGuiCharSet());
+ if (checkConversion(userCharset, htmlContent, converted))
+ {
+ usedHtmlCharset = userCharset;
+ return false; // only guessed value
+ }
+
+ // try WINDOWS-1252
+ if (checkConversion("WINDOWS-1252", htmlContent, converted))
+ {
+ usedHtmlCharset = "WINDOWS-1252";
+ return false; // only guessed value
+ }
+
+ // can't find exact charset
+ // use one of detected as fallback
+ if (!bomCharset.empty())
+ usedHtmlCharset = bomCharset;
+ else if (!serverReportedCharset.empty())
+ usedHtmlCharset = serverReportedCharset;
+ else if (!declaredCharset.empty())
+ usedHtmlCharset = declaredCharset;
+ else if (!userCharset.empty())
+ usedHtmlCharset = userCharset;
+ else
+ usedHtmlCharset = "WINDOWS-1252";
+
+ CLog::Log(LOGWARNING, "%s: Can't correctly convert to UTF-8 charset, converting as \"%s\"", __FUNCTION__, usedHtmlCharset.c_str());
+ g_charsetConverter.ToUtf8(usedHtmlCharset, htmlContent, converted, false);
+
+ return false;
+}
+
+bool CCharsetDetection::checkConversion(const std::string& srcCharset, const std::string& src, std::string& dst)
+{
+ if (srcCharset.empty())
+ return false;
+
+ if (srcCharset != "UTF-8")
+ {
+ if (g_charsetConverter.ToUtf8(srcCharset, src, dst, true))
+ return true;
+ }
+ else if (CUtf8Utils::isValidUtf8(src))
+ {
+ dst = src;
+ return true;
+ }
+
+ return false;
+}
+
+std::string CCharsetDetection::GetHtmlEncodingFromHead(const std::string& htmlContent)
+{
+ std::string smallerHtmlContent;
+ if (htmlContent.length() > 2 * m_HtmlCharsetEndSearchPos)
+ smallerHtmlContent.assign(htmlContent, 0, 2 * m_HtmlCharsetEndSearchPos); // use twice more bytes to search for charset for safety
+
+ const std::string& html = smallerHtmlContent.empty() ? htmlContent : smallerHtmlContent; // limit search
+ const char* const htmlC = html.c_str(); // for null-termination
+ const size_t len = html.length();
+
+ // this is an implementation of http://www.w3.org/TR/2013/CR-html5-20130806/single-page.html#prescan-a-byte-stream-to-determine-its-encoding
+ // labels in comments correspond to the labels in HTML5 standard
+ // note: opposite to standard, everything is converted to uppercase instead of lower case
+ size_t pos = 0;
+ while (pos < len) // "loop" label
+ {
+ if (html.compare(pos, 4, "<!--", 4) == 0)
+ {
+ pos = html.find("-->", pos + 2);
+ if (pos == std::string::npos)
+ return "";
+ pos += 2;
+ }
+ else if (htmlC[pos] == '<' && (htmlC[pos + 1] == 'm' || htmlC[pos + 1] == 'M') && (htmlC[pos + 2] == 'e' || htmlC[pos + 2] == 'E')
+ && (htmlC[pos + 3] == 't' || htmlC[pos + 3] == 'T') && (htmlC[pos + 4] == 'a' || htmlC[pos + 4] == 'A')
+ && (htmlC[pos + 5] == 0x09 || htmlC[pos + 5] == 0x0A || htmlC[pos + 5] == 0x0C || htmlC[pos + 5] == 0x0D || htmlC[pos + 5] == 0x20 || htmlC[pos + 5] == 0x2F))
+ { // this is case insensitive "<meta" and one of tab, LF, FF, CR, space or slash
+ pos += 5; // "pos" points to symbol after "<meta"
+ std::string attrName, attrValue;
+ bool gotPragma = false;
+ std::string contentCharset;
+ do // "attributes" label
+ {
+ pos = GetHtmlAttribute(html, pos, attrName, attrValue);
+ if (attrName == "HTTP-EQUIV" && attrValue == "CONTENT-TYPE")
+ gotPragma = true;
+ else if (attrName == "CONTENT")
+ contentCharset = ExtractEncodingFromHtmlMeta(attrValue);
+ else if (attrName == "CHARSET")
+ {
+ StringUtils::Trim(attrValue, m_HtmlWhitespaceChars.c_str()); // tab, LF, FF, CR, space
+ if (!attrValue.empty())
+ return attrValue;
+ }
+ } while (!attrName.empty() && pos < len);
+
+ // "processing" label
+ if (gotPragma && !contentCharset.empty())
+ return contentCharset;
+ }
+ else if (htmlC[pos] == '<' && ((htmlC[pos + 1] >= 'A' && htmlC[pos + 1] <= 'Z') || (htmlC[pos + 1] >= 'a' && htmlC[pos + 1] <= 'z')))
+ {
+ pos = html.find_first_of("\x09\x0A\x0C\x0D >", pos); // tab, LF, FF, CR, space or '>'
+ std::string attrName, attrValue;
+ do
+ {
+ pos = GetHtmlAttribute(html, pos, attrName, attrValue);
+ } while (pos < len && !attrName.empty());
+ }
+ else if (html.compare(pos, 2, "<!", 2) == 0 || html.compare(pos, 2, "</", 2) == 0 || html.compare(pos, 2, "<?", 2) == 0)
+ pos = html.find('>', pos);
+
+ if (pos == std::string::npos)
+ return "";
+
+ // "next byte" label
+ pos++;
+ }
+
+ return ""; // no charset was found
+}
+
+size_t CCharsetDetection::GetHtmlAttribute(const std::string& htmlContent, size_t pos, std::string& attrName, std::string& attrValue)
+{
+ attrName.clear();
+ attrValue.clear();
+ static const char* const htmlWhitespaceSlash = "\x09\x0A\x0C\x0D\x20\x2F"; // tab, LF, FF, CR, space or slash
+ const char* const htmlC = htmlContent.c_str();
+ const size_t len = htmlContent.length();
+
+ // this is an implementation of http://www.w3.org/TR/2013/CR-html5-20130806/single-page.html#concept-get-attributes-when-sniffing
+ // labels in comments correspond to the labels in HTML5 standard
+ // note: opposite to standard, everything is converted to uppercase instead of lower case
+ pos = htmlContent.find_first_not_of(htmlWhitespaceSlash, pos);
+ if (pos == std::string::npos || htmlC[pos] == '>')
+ return pos; // only white spaces or slashes up to the end of the htmlContent or no more attributes
+
+ while (pos < len && htmlC[pos] != '=')
+ {
+ const char chr = htmlC[pos];
+ if (chr == '/' || chr == '>')
+ return pos; // no attributes or empty attribute value
+ else if (m_HtmlWhitespaceChars.find(chr) != std::string::npos) // chr is one of whitespaces
+ {
+ pos = htmlContent.find_first_not_of(m_HtmlWhitespaceChars, pos); // "spaces" label
+ if (pos == std::string::npos || htmlC[pos] != '=')
+ return pos; // only white spaces up to the end or no attribute value
+ break;
+ }
+ else
+ appendCharAsAsciiUpperCase(attrName, chr);
+
+ pos++;
+ }
+
+ if (pos >= len)
+ return std::string::npos; // no '=', '/' or '>' were found up to the end of htmlContent
+
+ pos++; // advance pos to character after '='
+
+ pos = htmlContent.find_first_not_of(m_HtmlWhitespaceChars, pos); // "value" label
+ if (pos == std::string::npos)
+ return pos; // only white spaces remain in htmlContent
+
+ if (htmlC[pos] == '>')
+ return pos; // empty attribute value
+ else if (htmlC[pos] == '"' || htmlC[pos] == '\'')
+ {
+ const char qChr = htmlC[pos];
+ // "quote loop" label
+ while (++pos < len)
+ {
+ const char chr = htmlC[pos];
+ if (chr == qChr)
+ return pos + 1;
+ else
+ appendCharAsAsciiUpperCase(attrValue, chr);
+ }
+ return std::string::npos; // no closing quote is found
+ }
+
+ appendCharAsAsciiUpperCase(attrValue, htmlC[pos]);
+ pos++;
+
+ while (pos < len)
+ {
+ const char chr = htmlC[pos];
+ if (m_HtmlWhitespaceChars.find(chr) != std::string::npos || chr == '>')
+ return pos;
+ else
+ appendCharAsAsciiUpperCase(attrValue, chr);
+
+ pos++;
+ }
+
+ return std::string::npos; // rest of htmlContent was attribute value
+}
+
+std::string CCharsetDetection::ExtractEncodingFromHtmlMeta(std::string metaContent, size_t pos /*= 0*/)
+{
+ size_t len = metaContent.length();
+ if (pos >= len)
+ return "";
+
+ const char* const metaContentC = metaContent.c_str();
+
+ // this is an implementation of http://www.w3.org/TR/2013/CR-html5-20130806/single-page.html#algorithm-for-extracting-a-character-encoding-from-a-meta-element
+ // labels in comments correspond to the labels in HTML5 standard
+ // note: opposite to standard, case sensitive match is used as argument is always in uppercase
+ std::string charset;
+ do
+ {
+ // "loop" label
+ pos = metaContent.find("CHARSET", pos);
+ if (pos == std::string::npos)
+ return "";
+
+ pos = metaContent.find_first_not_of(m_HtmlWhitespaceChars, pos + 7); // '7' is the length of 'CHARSET'
+ if (pos != std::string::npos && metaContentC[pos] == '=')
+ {
+ pos = metaContent.find_first_not_of(m_HtmlWhitespaceChars, pos + 1);
+ if (pos != std::string::npos)
+ {
+ if (metaContentC[pos] == '\'' || metaContentC[pos] == '"')
+ {
+ const char qChr = metaContentC[pos];
+ pos++;
+ const size_t closeQpos = metaContent.find(qChr, pos);
+ if (closeQpos != std::string::npos)
+ charset.assign(metaContent, pos, closeQpos - pos);
+ }
+ else
+ charset.assign(metaContent, pos, metaContent.find("\x09\x0A\x0C\x0D ;", pos) - pos); // assign content up to the next tab, LF, FF, CR, space, semicolon or end of string
+ }
+ break;
+ }
+ } while (pos < len);
+
+ static const char* const htmlWhitespaceCharsC = m_HtmlWhitespaceChars.c_str();
+ StringUtils::Trim(charset, htmlWhitespaceCharsC);
+
+ return charset;
+}
+
+inline void CCharsetDetection::appendCharAsAsciiUpperCase(std::string& str, const char chr)
+{
+ if (chr >= 'a' && chr <= 'z')
+ str.push_back(chr - ('a' - 'A')); // convert to upper case
+ else
+ str.push_back(chr);
+}
projects / vuplus_xbmc / commitdiff