Merge pull request #3434 from Karlson2k/regexp_rework_02

RegExp rework - part 02

diff --git a/xbmc/Application.cpp b/xbmc/Application.cpp
index cd05639..6a3fbdb 100644 (file)
--- a/xbmc/Application.cpp
+++ b/xbmc/Application.cpp
@@ -706,6 +706,7 @@ bool CApplication::Create()
   CLog::Log(LOGNOTICE, "The executable running is: %s", executable.c_str());
   CLog::Log(LOGNOTICE, "Local hostname: %s", m_network->GetHostName().c_str());
   CLog::Log(LOGNOTICE, "Log File is located: %sxbmc.log", g_advancedSettings.m_logFolder.c_str());
+  CRegExp::LogCheckUtf8Support();
   CLog::Log(LOGNOTICE, "-----------------------------------------------------------------------");
 
   CStdString strExecutablePath;

diff --git a/xbmc/FileItem.cpp b/xbmc/FileItem.cpp
index 78ce04f..a30c705 100644 (file)
--- a/xbmc/FileItem.cpp
+++ b/xbmc/FileItem.cpp
@@ -2288,7 +2288,7 @@ void CFileItemList::StackFolders()
 {
   // Precompile our REs
   VECCREGEXP folderRegExps;
-  CRegExp folderRegExp(true);
+  CRegExp folderRegExp(true, true);
   const CStdStringArray& strFolderRegExps = g_advancedSettings.m_folderStackRegExps;
 
   CStdStringArray::const_iterator strExpression = strFolderRegExps.begin();
@@ -2407,7 +2407,7 @@ void CFileItemList::StackFiles()
 {
   // Precompile our REs
   VECCREGEXP stackRegExps;
-  CRegExp tmpRegExp(true);
+  CRegExp tmpRegExp(true, true);
   const CStdStringArray& strStackRegExps = g_advancedSettings.m_videoStackRegExps;
   CStdStringArray::const_iterator strRegExp = strStackRegExps.begin();
   while (strRegExp != strStackRegExps.end())
@@ -3235,7 +3235,7 @@ CStdString CFileItem::FindTrailer() const
 
   // Precompile our REs
   VECCREGEXP matchRegExps;
-  CRegExp tmpRegExp(true);
+  CRegExp tmpRegExp(true, true);
   const CStdStringArray& strMatchRegExps = g_advancedSettings.m_trailerMatchRegExps;
 
   CStdStringArray::const_iterator strRegExp = strMatchRegExps.begin();

diff --git a/xbmc/Util.cpp b/xbmc/Util.cpp
index e8e5bd3..260f3c0 100644 (file)
--- a/xbmc/Util.cpp
+++ b/xbmc/Util.cpp
@@ -243,8 +243,8 @@ void CUtil::CleanString(const CStdString& strFileName, CStdString& strTitle, CSt
 
   const CStdStringArray &regexps = g_advancedSettings.m_videoCleanStringRegExps;
 
-  CRegExp reTags(true);
-  CRegExp reYear;
+  CRegExp reTags(true, true);
+  CRegExp reYear(false, true);
 
   if (!reYear.RegComp(g_advancedSettings.m_videoCleanDateTimeRegExp))
   {
@@ -518,7 +518,7 @@ bool CUtil::ExcludeFileOrFolder(const CStdString& strFileOrFolder, const CStdStr
   if (strFileOrFolder.IsEmpty())
     return false;
 
-  CRegExp regExExcludes(true);  // case insensitive regex
+  CRegExp regExExcludes(true, true);  // case insensitive regex
 
   for (unsigned int i = 0; i < regexps.size(); i++)
   {

diff --git a/xbmc/cores/ExternalPlayer/ExternalPlayer.cpp b/xbmc/cores/ExternalPlayer/ExternalPlayer.cpp
index d497aee..b709a7b 100644 (file)
--- a/xbmc/cores/ExternalPlayer/ExternalPlayer.cpp
+++ b/xbmc/cores/ExternalPlayer/ExternalPlayer.cpp
@@ -185,7 +185,7 @@ void CExternalPlayer::Process()
       CStdString strMatch = vecSplit[0];
       strMatch.Replace(",,",",");
       bool bCaseless = vecSplit[3].Find('i') > -1;
-      CRegExp regExp(bCaseless);
+      CRegExp regExp(bCaseless, true);
 
       if (!regExp.RegComp(strMatch.c_str()))
       { // invalid regexp - complain in logs

diff --git a/xbmc/cores/playercorefactory/PlayerSelectionRule.cpp b/xbmc/cores/playercorefactory/PlayerSelectionRule.cpp
index f3ba675..4402a9d 100644 (file)
--- a/xbmc/cores/playercorefactory/PlayerSelectionRule.cpp
+++ b/xbmc/cores/playercorefactory/PlayerSelectionRule.cpp
@@ -118,7 +118,7 @@ void CPlayerSelectionRule::GetPlayers(const CFileItem& item, VECPLAYERCORES &vec
   if (m_tDVDFile >= 0 && (m_tDVDFile > 0) != item.IsDVDFile()) return;
   if (m_tDVDImage >= 0 && (m_tDVDImage > 0) != item.IsDVDImage()) return;
 
-  CRegExp regExp;
+  CRegExp regExp(false, true);
 
   if (m_bStreamDetails)
   {

diff --git a/xbmc/filesystem/StackDirectory.cpp b/xbmc/filesystem/StackDirectory.cpp
index 295eb55..7cc8a66 100644 (file)
--- a/xbmc/filesystem/StackDirectory.cpp
+++ b/xbmc/filesystem/StackDirectory.cpp
@@ -59,7 +59,7 @@ namespace XFILE
   {
     // Load up our REs
     VECCREGEXP  RegExps;
-    CRegExp     tempRE(true);
+    CRegExp     tempRE(true, true);
     const CStdStringArray& strRegExps = g_advancedSettings.m_videoStackRegExps;
     CStdStringArray::const_iterator itRegExp = strRegExps.begin();
     vector<pair<int, CStdString> > badStacks;

diff --git a/xbmc/utils/RegExp.cpp b/xbmc/utils/RegExp.cpp
index 51993f0..0c2e4c1 100644 (file)
--- a/xbmc/utils/RegExp.cpp
+++ b/xbmc/utils/RegExp.cpp
@@ -27,12 +27,27 @@
 
 using namespace PCRE;
 
-CRegExp::CRegExp(bool caseless)
+#ifndef PCRE_UCP
+#define PCRE_UCP 0
+#endif // PCRE_UCP
+
+int CRegExp::m_Utf8Supported = -1;
+int CRegExp::m_UcpSupported  = -1;
+
+
+CRegExp::CRegExp(bool caseless /*= false*/, bool utf8 /*= false*/)
 {
   m_re          = NULL;
   m_iOptions    = PCRE_DOTALL | PCRE_NEWLINE_ANY;
   if(caseless)
     m_iOptions |= PCRE_CASELESS;
+  if (utf8)
+  {
+    if (IsUtf8Supported())
+      m_iOptions |= PCRE_UTF8;
+    if (AreUnicodePropertiesSupported())
+      m_iOptions |= PCRE_UCP;
+  }
 
   m_offset      = 0;
   m_bMatched    = false;
@@ -152,6 +167,17 @@ int CRegExp::PrivateRegFind(size_t bufferLen, const char *str, unsigned int star
       CLog::Log(LOGERROR, "PCRE: Match limit reached");
       return -1;
 
+#ifdef PCRE_ERROR_SHORTUTF8 
+    case PCRE_ERROR_SHORTUTF8:
+#endif
+    case PCRE_ERROR_BADUTF8:
+      CLog::Log(LOGERROR, "PCRE: Bad UTF-8 character");
+      return -1;
+
+    case PCRE_ERROR_BADUTF8_OFFSET:
+      CLog::Log(LOGERROR, "PCRE: Offset (%d) is pointing to the middle of UTF-8 character", startoffset);
+      return -1;
+
     default:
       CLog::Log(LOGERROR, "PCRE: Unknown error: %d", rc);
       return -1;
@@ -304,3 +330,54 @@ inline bool CRegExp::IsValidSubNumber(int iSub) const
   return iSub >= 0 && iSub <= m_iMatchCount && iSub <= m_MaxNumOfBackrefrences;
 }
 
+
+bool CRegExp::IsUtf8Supported(void)
+{
+  if (m_Utf8Supported == -1)
+  {
+    if (pcre_config(PCRE_CONFIG_UTF8, &m_Utf8Supported) != 0)
+      m_Utf8Supported = 0;
+  }
+
+  return m_Utf8Supported == 1;
+}
+
+bool CRegExp::AreUnicodePropertiesSupported(void)
+{
+#if defined(PCRE_CONFIG_UNICODE_PROPERTIES) && PCRE_UCP != 0
+  if (m_UcpSupported == -1)
+  {
+    if (pcre_config(PCRE_CONFIG_UNICODE_PROPERTIES, &m_UcpSupported) != 0)
+      m_UcpSupported = 0;
+  }
+#endif
+
+  return m_UcpSupported == 1;
+}
+
+bool CRegExp::LogCheckUtf8Support(void)
+{
+  bool utf8FullSupport = true;
+
+  if (!CRegExp::IsUtf8Supported())
+  {
+    utf8FullSupport = false;
+    CLog::Log(LOGWARNING, "UTF-8 is not supported in PCRE lib, support for national symbols is limited!");
+  }
+
+  if (!CRegExp::AreUnicodePropertiesSupported())
+  {
+    utf8FullSupport = false;
+    CLog::Log(LOGWARNING, "Unicode properties are not enabled in PCRE lib, support for national symbols may be limited!");
+  }
+
+  if (!utf8FullSupport)
+  {
+    CLog::Log(LOGNOTICE, "Consider installing PCRE lib version 8.10 or later with enabled Unicode properties and UTF-8 support. Your PCRE lib version: %s", PCRE::pcre_version());
+#if PCRE_UCP == 0
+    CLog::Log(LOGNOTICE, "You will need to rebuild XBMC after PCRE lib update.", PCRE::pcre_version());
+#endif
+  }
+
+  return utf8FullSupport;
+}

diff --git a/xbmc/utils/RegExp.h b/xbmc/utils/RegExp.h
index c48319f..a9c62c5 100644 (file)
--- a/xbmc/utils/RegExp.h
+++ b/xbmc/utils/RegExp.h
@@ -41,7 +41,7 @@ class CRegExp
 {
 public:
   static const int m_MaxNumOfBackrefrences = 20;
-  CRegExp(bool caseless = false);
+  CRegExp(bool caseless = false, bool utf8 = false);
   CRegExp(const CRegExp& re);
   ~CRegExp();
 
@@ -71,6 +71,9 @@ public:
   int GetNamedSubPatternNumber(const char* strName) const;
   void DumpOvector(int iLog);
   const CRegExp& operator= (const CRegExp& re);
+  static bool IsUtf8Supported(void);
+  static bool AreUnicodePropertiesSupported(void);
+  static bool LogCheckUtf8Support(void);
 
 private:
   int PrivateRegFind(size_t bufferLen, const char *str, unsigned int startoffset = 0, int maxNumberOfCharsToTest = -1);
@@ -87,6 +90,8 @@ private:
   bool        m_bMatched;
   std::string m_subject;
   std::string m_pattern;
+  static int  m_Utf8Supported;     
+  static int  m_UcpSupported;
 };
 
 typedef std::vector<CRegExp> VECCREGEXP;

diff --git a/xbmc/utils/ScraperParser.cpp b/xbmc/utils/ScraperParser.cpp
index 700ebfa..2a5268c 100644 (file)
--- a/xbmc/utils/ScraperParser.cpp
+++ b/xbmc/utils/ScraperParser.cpp
@@ -204,7 +204,7 @@ void CScraperParser::ParseExpression(const CStdString& input, CStdString& dest,
       if (stricmp(sensitive,"yes") == 0)
         bInsensitive=false; // match case sensitive
 
-    CRegExp reg(bInsensitive);
+    CRegExp reg(bInsensitive, true);
     CStdString strExpression;
     if (pExpression->FirstChild())
       strExpression = pExpression->FirstChild()->Value();

diff --git a/xbmc/video/VideoInfoScanner.cpp b/xbmc/video/VideoInfoScanner.cpp
index e89412c..cef6aa4 100644 (file)
--- a/xbmc/video/VideoInfoScanner.cpp
+++ b/xbmc/video/VideoInfoScanner.cpp
@@ -869,7 +869,7 @@ namespace VIDEO
 
     for (unsigned int i=0;i<expression.size();++i)
     {
-      CRegExp reg(true);
+      CRegExp reg(true, true);
       if (!reg.RegComp(expression[i].regexp))
         continue;
 
@@ -938,7 +938,7 @@ namespace VIDEO
       // add what we found by now
       episodeList.push_back(episode);
 
-      CRegExp reg2(true);
+      CRegExp reg2(true, true);
       // check the remainder of the string for any further episodes.
       if (!byDate && reg2.RegComp(g_advancedSettings.m_tvshowMultiPartEnumRegExp))
       {