add ability to select default encoding for dvb texts in many ways.. ( take a look...

diff --git a/data/Makefile.am b/data/Makefile.am
index 6f69842..f3bc5ae 100644 (file)
--- a/data/Makefile.am
+++ b/data/Makefile.am
@@ -3,4 +3,4 @@ AUTOMAKE_OPTIONS = gnu
 installdir = $(DATADIR)/enigma2
 
 install_DATA = \
 installdir = $(DATADIR)/enigma2
 
 install_DATA = \

-       *.xml *.png
+       *.xml *.png encoding.conf

diff --git a/data/encoding.conf b/data/encoding.conf
new file mode 100644 (file)
index 0000000..345d9de
--- /dev/null
+++ b/data/encoding.conf
@@ -0,0 +1,30 @@
+#Fallback encoding when in dvb-text no encoding table is given
+#Countycode ISO8859-X
+tur ISO8859-9
+gre ISO8859-7
+pol ISO8859-2
+rus ISO8859-5
+bul ISO8859-5
+#Table of transponder using two byte char mappings (VideoTexSuppl)
+#TSID ONID
+0x447 0x1 #ASTRA 19.2° UPC Direct
+0x427 0x1
+0x44b 0x1
+0x4ff 0x1
+11000 318 #Hotbird 13.0° Cyfra+
+11900 318
+12200 318
+1500 318
+400 318
+0x5dc 0x13e
+#Fallback encoding table for following transponders
+#TSID ONID ISO8859-X
+#0x447 0x1 ISO8859-9
+50200 126 ISO8859-9 # Digiturk 7°E 11.492 V 30.000 3/4
+50300 126 ISO8859-9 # Digiturk 7°E 11.639 H 30.000 3/4
+50400 126 ISO8859-9 # Digiturk 7°E 11.534 V 30.000 3/4
+50600 126 ISO8859-9 # Digiturk 7°E 11.575 V and 42°E 11.729 V 15.555 5/6
+50700 126 ISO8859-9 # Digiturk 7°E 11.596 H 30.000 3/4
+50800 126 ISO8859-9 # Digiturk 7°E 11.678 H 30.000 3/4
+50900 126 ISO8859-9 # Skyturk  7°E 11.513 H 27.500 3/4
+51000 126 ISO8859-9 # Digiturk 7°E 11.617 V 30.000 3/4

diff --git a/lib/base/Makefile.am b/lib/base/Makefile.am
index 7c66c4d..f7176e2 100644 (file)
--- a/lib/base/Makefile.am
+++ b/lib/base/Makefile.am
@@ -7,4 +7,4 @@ libenigma_base_a_SOURCES = \
        buffer.cpp ebase.cpp econfig.cpp eerror.cpp elock.cpp \
        init.cpp message.cpp thread.cpp \
        smartptr.cpp estring.cpp connection.cpp \
        buffer.cpp ebase.cpp econfig.cpp eerror.cpp elock.cpp \
        init.cpp message.cpp thread.cpp \
        smartptr.cpp estring.cpp connection.cpp \

-       filepush.cpp
+       filepush.cpp encoding.cpp

diff --git a/lib/base/encoding.cpp b/lib/base/encoding.cpp
new file mode 100644 (file)
index 0000000..e865169
--- /dev/null
+++ b/lib/base/encoding.cpp
@@ -0,0 +1,59 @@
+#include <lib/base/encoding.h>
+#include <lib/base/eerror.h>
+#include <config.h>
+
+eDVBTextEncodingHandler encodingHandler;  // the one and only instance
+
+eDVBTextEncodingHandler::eDVBTextEncodingHandler()
+{
+       const char * file=DATADIR "/enigma2/encoding.conf";
+       FILE *f = fopen(file, "rt");
+       if (f)
+       {
+               char *line = (char*) malloc(256);
+               size_t bufsize=256;
+               char countrycode[256];
+               while( getline(&line, &bufsize, f) != -1 )
+               {
+                       if ( line[0] == '#' )
+                               continue;
+                       int tsid, onid, encoding;
+                       if ( sscanf( line, "%s ISO8859-%d", countrycode, &encoding ) == 2 )
+                               m_CountryCodeDefaultMapping[countrycode]=encoding;
+                       else if ( (sscanf( line, "0x%x 0x%x ISO8859-%d", &tsid, &onid, &encoding ) == 3 )
+                                       ||(sscanf( line, "%d %d ISO8859-%d", &tsid, &onid, &encoding ) == 3 ) )
+                               m_TransponderDefaultMapping[(tsid<<16)|onid]=encoding;
+                       else if ( (sscanf( line, "0x%x 0x%x", &tsid, &onid ) == 2 )
+                                       ||(sscanf( line, "%d %d", &tsid, &onid ) == 2 ) )
+                               m_TransponderUseTwoCharMapping.insert((tsid<<16)|onid);
+                       else
+                               eDebug("couldn't parse %s", line);
+               }
+               fclose(f);
+               free(line);
+       }
+       else
+               eDebug("[eDVBTextEncodingHandler] couldn't open %s !", file);
+}
+
+void eDVBTextEncodingHandler::getTransponderDefaultMapping(int tsidonid, int &table)
+{
+       std::map<int, int>::iterator it =
+               m_TransponderDefaultMapping.find(tsidonid);
+       if ( it != m_TransponderDefaultMapping.end() )
+               table = it->second;
+}
+
+bool eDVBTextEncodingHandler::getTransponderUseTwoCharMapping(int tsidonid)
+{
+       return m_TransponderUseTwoCharMapping.find(tsidonid) != m_TransponderUseTwoCharMapping.end();
+}
+
+int eDVBTextEncodingHandler::getCountryCodeDefaultMapping( const std::string &country_code )
+{
+       std::map<std::string, int>::iterator it =
+               m_CountryCodeDefaultMapping.find(country_code);
+       if ( it != m_CountryCodeDefaultMapping.end() )
+               return it->second;
+       return 0;  // ISO8859-1 / Latin1
+}

diff --git a/lib/base/encoding.h b/lib/base/encoding.h
new file mode 100644 (file)
index 0000000..3f107d6
--- /dev/null
+++ b/lib/base/encoding.h
@@ -0,0 +1,22 @@
+#ifndef __lib_base_encoding_h__
+#define __lib_base_encoding_h__
+
+#include <string>
+#include <set>
+#include <map>
+
+class eDVBTextEncodingHandler
+{
+       std::map<std::string, int> m_CountryCodeDefaultMapping;
+       std::map<int, int> m_TransponderDefaultMapping;
+       std::set<int> m_TransponderUseTwoCharMapping;
+public:
+       eDVBTextEncodingHandler();
+       void getTransponderDefaultMapping(int tsidonid, int &table);
+       bool getTransponderUseTwoCharMapping(int tsidonid);
+       int getCountryCodeDefaultMapping( const std::string &country_code );
+};
+
+extern eDVBTextEncodingHandler encodingHandler;
+
+#endif // __lib_base_encoding_h__
\ No newline at end of file

diff --git a/lib/base/estring.cpp b/lib/base/estring.cpp
index 4259c58..10bfbef 100644 (file)
--- a/lib/base/estring.cpp
+++ b/lib/base/estring.cpp
@@ -1,12 +1,10 @@
 #include <string>
 #include <ctype.h>
 #include <limits.h>
 #include <string>
 #include <ctype.h>
 #include <limits.h>

-#include <lib/base/elock.h>

 #include <lib/base/eerror.h>
 #include <lib/base/eerror.h>

+#include <lib/base/encoding.h>

 #include <lib/base/estring.h>
 
 #include <lib/base/estring.h>
 

-static pthread_mutex_t lock=PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP;
-

 std::string getNum(int val, int sys)
 {
 //     Returns a string that contain the value val as string
 std::string getNum(int val, int sys)
 {
 //     Returns a string that contain the value val as string


@@ -331,15 +329,8 @@ std::string convertDVBUTF8(const unsigned char *data, int len, int table, int ts
 
        int i=0, t=0;
 
 
        int i=0, t=0;
 

-#if 0  // FIXME

        if ( tsidonid )
        if ( tsidonid )

-       {
-               std::map<int, int>::iterator it =
-                       std::string::TransponderDefaultMapping.find(tsidonid);
-               if ( it != std::string::TransponderDefaultMapping.end() )
-                       table = it->second;
-       }
-#endif
+               encodingHandler.getTransponderDefaultMapping(tsidonid, table);

 
        switch(data[0])
        {
 
        switch(data[0])
        {


@@ -386,17 +377,18 @@ std::string convertDVBUTF8(const unsigned char *data, int len, int table, int ts
                        break;
        }
 
                        break;
        }
 

+       bool useTwoCharMapping =
+               tsidonid && encodingHandler.getTransponderUseTwoCharMapping(tsidonid);
+

        unsigned char res[2048];
        while (i < len)
        {
                unsigned long code=0;
 
        unsigned char res[2048];
        while (i < len)
        {
                unsigned long code=0;
 

-#if 0  // FIXME
-               if ( i+1 < len && tsidonid &&
-                       std::string::TransponderUseTwoCharMapping.find(tsidonid) != std::string::TransponderUseTwoCharMapping.end() &&
+               if ( useTwoCharMapping && i+1 < len &&

                        (code=doVideoTexSuppl(data[i], data[i+1])) )
                        i+=2;
                        (code=doVideoTexSuppl(data[i], data[i+1])) )
                        i+=2;

-#endif
+

                if (!code)
                        code=recode(data[i++], table);
                if (!code)
                if (!code)
                        code=recode(data[i++], table);
                if (!code)

diff --git a/lib/dvb/eit.cpp b/lib/dvb/eit.cpp
index b78b227..cc2c29b 100644 (file)
--- a/lib/dvb/eit.cpp
+++ b/lib/dvb/eit.cpp
@@ -14,10 +14,11 @@ void eDVBServiceEITHandler::EITready(int error)
                        for (std::vector<EventInformationSection*>::const_iterator i = ptr->getSections().begin();
                                i != ptr->getSections().end(); ++i)
                        {
                        for (std::vector<EventInformationSection*>::const_iterator i = ptr->getSections().begin();
                                i != ptr->getSections().end(); ++i)
                        {

-                               for (EventConstIterator ev = (*i)->getEvents()->begin(); ev != (*i)->getEvents()->end(); ++ev)
+                               const EventInformationSection *eit = *i;
+                               for (EventConstIterator ev = eit->getEvents()->begin(); ev != eit->getEvents()->end(); ++ev)

                                {
                                        ePtr<eServiceEvent> evt = new eServiceEvent();
                                {
                                        ePtr<eServiceEvent> evt = new eServiceEvent();

-                                       evt->parseFrom(*ev);
+                                       evt->parseFrom(*ev,(eit->getTransportStreamId()<<16)|eit->getOriginalNetworkId());

                                        if (!a)
                                                m_event_now = evt;
                                        else
                                        if (!a)
                                                m_event_now = evt;
                                        else

diff --git a/lib/dvb/epgcache.cpp b/lib/dvb/epgcache.cpp
index 587d680..6dea73b 100644 (file)
--- a/lib/dvb/epgcache.cpp
+++ b/lib/dvb/epgcache.cpp
@@ -1066,7 +1066,8 @@ RESULT eEPGCache::lookupEventTime(const eServiceReference &service, time_t t, eP
        {
                Event ev((uint8_t*)data->get());
                result = new eServiceEvent();
        {
                Event ev((uint8_t*)data->get());
                result = new eServiceEvent();

-               ret = result->parseFrom(&ev);
+               const eServiceReferenceDVB &ref = (const eServiceReferenceDVB&)service;
+               ret = result->parseFrom(&ev, (ref.getTransportStreamID().get()<<16)|ref.getOriginalNetworkID().get());

        }
        return ret;
 }
        }
        return ret;
 }


@@ -1123,7 +1124,8 @@ RESULT eEPGCache::lookupEventId(const eServiceReference &service, int event_id,
        {
                Event ev((uint8_t*)data->get());
                result = new eServiceEvent();
        {
                Event ev((uint8_t*)data->get());
                result = new eServiceEvent();

-               ret = result->parseFrom(&ev);
+               const eServiceReferenceDVB &ref = (const eServiceReferenceDVB&)service;
+               ret = result->parseFrom(&ev, (ref.getTransportStreamID().get()<<16)|ref.getOriginalNetworkID().get());

        }
        return ret;
 }
        }
        return ret;
 }


@@ -1147,6 +1149,8 @@ RESULT eEPGCache::startTimeQuery(const eServiceReference &service, time_t begin,
                }
                else
                        m_timemap_cursor = It->second.second.begin();
                }
                else
                        m_timemap_cursor = It->second.second.begin();

+               const eServiceReferenceDVB &ref = (const eServiceReferenceDVB&)service;
+               currentQueryTsidOnid = (ref.getTransportStreamID().get()<<16) | ref.getOriginalNetworkID().get();

                return 0;
        }
        return -1;
                return 0;
        }
        return -1;


@@ -1188,7 +1192,7 @@ RESULT eEPGCache::getNextTimeEntry(ePtr<eServiceEvent> &result)
        {
                Event ev((uint8_t*)m_timemap_cursor++->second->get());
                result = new eServiceEvent();
        {
                Event ev((uint8_t*)m_timemap_cursor++->second->get());
                result = new eServiceEvent();

-               return result->parseFrom(&ev);
+               return result->parseFrom(&ev, currentQueryTsidOnid);

        }
        return -1;
 }
        }
        return -1;
 }

diff --git a/lib/dvb/epgcache.h b/lib/dvb/epgcache.h
index 3c2b612..fe461d9 100644 (file)
--- a/lib/dvb/epgcache.h
+++ b/lib/dvb/epgcache.h
@@ -221,6 +221,7 @@ private:
        void DVBChannelRunning(iDVBChannel *);
 
        timeMap::iterator m_timemap_cursor, m_timemap_end;
        void DVBChannelRunning(iDVBChannel *);
 
        timeMap::iterator m_timemap_cursor, m_timemap_end;

+       int currentQueryTsidOnid; // needed for getNextTimeEntry.. only valid until next startTimeQuery call

 #endif // SWIG
 public:
        static eEPGCache *getInstance() { return instance; }
 #endif // SWIG
 public:
        static eEPGCache *getInstance() { return instance; }

diff --git a/lib/service/event.cpp b/lib/service/event.cpp
index 684c9e5..cfd2ac5 100644 (file)
--- a/lib/service/event.cpp
+++ b/lib/service/event.cpp
@@ -1,5 +1,6 @@
 #include <lib/service/event.h>
 #include <lib/base/estring.h>
 #include <lib/service/event.h>
 #include <lib/base/estring.h>

+#include <lib/base/encoding.h>

 #include <lib/dvb/dvbtime.h>
 #include <dvbsi++/event_information_section.h>
 #include <dvbsi++/short_event_descriptor.h>
 #include <lib/dvb/dvbtime.h>
 #include <dvbsi++/event_information_section.h>
 #include <dvbsi++/short_event_descriptor.h>


@@ -52,7 +53,7 @@ std::string ISOtbl[MAX_LANG][2] =
 };
 
 /* search for the presence of language from given EIT event descriptors*/
 };
 
 /* search for the presence of language from given EIT event descriptors*/

-bool eServiceEvent::loadLanguage(Event *evt, std::string lang)
+bool eServiceEvent::loadLanguage(Event *evt, std::string lang, int tsidonid)

 {
        bool retval=0;
        for (DescriptorConstIterator desc = evt->getDescriptors()->begin(); desc != evt->getDescriptors()->end(); ++desc)
 {
        bool retval=0;
        for (DescriptorConstIterator desc = evt->getDescriptors()->begin(); desc != evt->getDescriptors()->end(); ++desc)


@@ -62,10 +63,12 @@ bool eServiceEvent::loadLanguage(Event *evt, std::string lang)
                        case SHORT_EVENT_DESCRIPTOR:
                        {
                                const ShortEventDescriptor *sed = (ShortEventDescriptor*)*desc;
                        case SHORT_EVENT_DESCRIPTOR:
                        {
                                const ShortEventDescriptor *sed = (ShortEventDescriptor*)*desc;

-                               if (lang.empty() || sed->getIso639LanguageCode() == lang)
+                               const std::string &cc = sed->getIso639LanguageCode();
+                               int table=encodingHandler.getCountryCodeDefaultMapping(cc);
+                               if (lang.empty() || cc == lang)

                                {
                                {

-                                       m_event_name = convertDVBUTF8(sed->getEventName());
-                                       m_short_description = convertDVBUTF8(sed->getText());
+                                       m_event_name = convertDVBUTF8(sed->getEventName(), table, tsidonid);
+                                       m_short_description = convertDVBUTF8(sed->getText(), table, tsidonid);

                                        retval=1;
                                }
                                break;
                                        retval=1;
                                }
                                break;


@@ -73,9 +76,11 @@ bool eServiceEvent::loadLanguage(Event *evt, std::string lang)
                        case EXTENDED_EVENT_DESCRIPTOR:
                        {
                                const ExtendedEventDescriptor *eed = (ExtendedEventDescriptor*)*desc;
                        case EXTENDED_EVENT_DESCRIPTOR:
                        {
                                const ExtendedEventDescriptor *eed = (ExtendedEventDescriptor*)*desc;

-                               if (lang.empty() || eed->getIso639LanguageCode() == lang)
+                               const std::string &cc = eed->getIso639LanguageCode();
+                               int table=encodingHandler.getCountryCodeDefaultMapping(cc);
+                               if (lang.empty() || cc == lang)

                                {
                                {

-                                       m_extended_description += convertDVBUTF8(eed->getText());
+                                       m_extended_description += convertDVBUTF8(eed->getText(), table, tsidonid);

                                        retval=1;
                                }
 #if 0
                                        retval=1;
                                }
 #if 0


@@ -99,7 +104,7 @@ bool eServiceEvent::loadLanguage(Event *evt, std::string lang)
        return retval;
 }
 
        return retval;
 }
 

-RESULT eServiceEvent::parseFrom(Event *evt)
+RESULT eServiceEvent::parseFrom(Event *evt, int tsidonid)

 {
        uint16_t stime_mjd = evt->getStartTimeMjd();
        uint32_t stime_bcd = evt->getStartTimeBcd();
 {
        uint16_t stime_mjd = evt->getStartTimeMjd();
        uint32_t stime_bcd = evt->getStartTimeBcd();


@@ -115,11 +120,11 @@ RESULT eServiceEvent::parseFrom(Event *evt)
        std::string country="de_DE";  // TODO use local data here
        for (int i=0; i < MAX_LANG; i++)
                if (country==ISOtbl[i][0])
        std::string country="de_DE";  // TODO use local data here
        for (int i=0; i < MAX_LANG; i++)
                if (country==ISOtbl[i][0])

-                       if (loadLanguage(evt,ISOtbl[i][1]))
+                       if (loadLanguage(evt, ISOtbl[i][1], tsidonid))

                                return 0;
                                return 0;

-       if (loadLanguage(evt,"eng"))
+       if (loadLanguage(evt, "eng", tsidonid))

                return 0;
                return 0;

-       if (loadLanguage(evt,std::string()))
+       if (loadLanguage(evt, std::string(), tsidonid))

                return 0;
        return 0;
 }
                return 0;
        return 0;
 }

diff --git a/lib/service/event.h b/lib/service/event.h
index 7e6cf5d..07106e4 100644 (file)
--- a/lib/service/event.h
+++ b/lib/service/event.h
@@ -1,7 +1,7 @@
 #ifndef __lib_service_event_h
 #define __lib_service_event_h
 
 #ifndef __lib_service_event_h
 #define __lib_service_event_h
 

-#ifndef PYTHON
+#ifndef SWIG

 #include <time.h>
 #include <lib/base/object.h>
 #include <string>
 #include <time.h>
 #include <lib/base/object.h>
 #include <string>


@@ -11,14 +11,16 @@ class Event;
 class eServiceEvent: public iObject
 {
 DECLARE_REF(eServiceEvent);
 class eServiceEvent: public iObject
 {
 DECLARE_REF(eServiceEvent);

+#ifndef SWIG
+       bool loadLanguage(Event *event, std::string lang, int tsidonid);
+#endif

 public:
 public:

-#ifndef PYTHON
+#ifndef SWIG

        time_t m_begin;
        int m_duration;
        std::string m_event_name, m_short_description, m_extended_description;
        // .. additional info
        time_t m_begin;
        int m_duration;
        std::string m_event_name, m_short_description, m_extended_description;
        // .. additional info

-       bool loadLanguage(Event *event, std::string lang);
-       RESULT parseFrom(Event *evt);
+       RESULT parseFrom(Event *evt, int tsidonid=0);

 #endif
        time_t getBeginTime() { return m_begin; }
        int getDuration() { return m_duration; }
 #endif
        time_t getBeginTime() { return m_begin; }
        int getDuration() { return m_duration; }


@@ -28,8 +30,8 @@ public:
        std::string getBeginTimeString();
 };
 
        std::string getBeginTimeString();
 };
 

-#ifndef PYTHON

 TEMPLATE_TYPEDEF(ePtr<eServiceEvent>, eServiceEventPtr);
 TEMPLATE_TYPEDEF(ePtr<eServiceEvent>, eServiceEventPtr);

+#ifndef SWIG

 
 class eDebugClass: public iObject
 {
 
 class eDebugClass: public iObject
 {