tagstrip should handle all entities now using htmlentitydefs

diff --git a/simplerss/CONTROL/control b/simplerss/CONTROL/control
index 076c247..ab88d76 100644 (file)
--- a/simplerss/CONTROL/control
+++ b/simplerss/CONTROL/control
@@ -5,5 +5,5 @@ Architecture: mipsel
 Section: extra
 Priority: optional
 Maintainer: Moritz Venn <moritz.venn@freaque.net>
-Depends: enigma2(>=2.6git20090615), twisted-web, python-codecs, python-xml
+Depends: enigma2(>=2.6git20090615), twisted-web, python-codecs, python-xml, python-html
 Source: http://schwerkraft.elitedvb.net/scm/?group_id=11

diff --git a/simplerss/src/TagStrip.py b/simplerss/src/TagStrip.py
index 858b897..d24d7ef 100644 (file)
--- a/simplerss/src/TagStrip.py
+++ b/simplerss/src/TagStrip.py
@@ -1,42 +1,7 @@
 # -*- coding: utf-8 -*-
-from re import sub
+from re import sub, finditer
 
-# Entities to be converted
-entities = (
-       # ISO-8895-1 (most common)
-       ("ä", u"ä"),
-       ("ä", u"ä"),
-       ("ü", u"ü"),
-       ("ü", u"ü"),
-       ("ö", u"ö"),
-       ("ö", u"ö"),
-       ("Ä", u"Ä"),
-       ("Ä", u"Ä"),
-       ("Ü", u"Ü"),
-       ("Ü", u"Ü"),
-       ("Ö", u"Ö"),
-       ("Ö", u"Ö"),
-       ("ß", u"ß"),
-       ("ß", u"ß"),
-
-       # Rarely used entities
-       ("…", u"..."),
-       ("–", u"-"),
-       (" ", u" "),
-       (""", u"\""),
-       ("&", u"&"),
-       ("'", u"'"),
-       ("&#60;", u"<"),
-       ("&#62;", u">"),
-
-       # Common entities
-       ("&lt;", u"<"),
-       ("&gt;", u">"),
-       ("&nbsp;", u" "),
-       ("&amp;", u"&"),
-       ("&quot;", u"\""),
-       ("&apos;", u"'"),
-)
+import htmlentitydefs
 
 def strip_readable(html):
        # Newlines are rendered as whitespace in html
@@ -60,11 +25,30 @@ def strip_readable(html):
 
 def strip(html):
        # Strip enclosed tags
-       html = sub('<(.*?)>', '', html)
-
-       # Convert html entities
-       for escaped, unescaped in entities:
-               html = html.replace(escaped, unescaped)
+       html = sub('<.*?>', '', html)
+
+       entitydict = {}
+
+       entities = finditer('&([^#]\D{1,5}?);', html)
+       for x in entities:
+               key = x.group(0)
+               if key not in entitydict:
+                       entitydict[key] = htmlentitydefs.name2codepoint[x.group(1)]
+
+       entities = finditer('&#x([0-9A-Fa-f]{2,2}?);', html)
+       for x in entities:
+               key = x.group(0)
+               if key not in entitydict:
+                       entitydict[key] = "%d" % int(key[3:5], 16)
+
+       entities = finditer('&#(\d{1,5}?);', html)
+       for x in entities:
+               key = x.group(0)
+               if key not in entitydict:
+                       entitydict[key] = x.group(1)
+
+       for key, codepoint in entitydict.items():
+               html = html.replace(key, unichr(int(codepoint)))
 
        # Return result with leading/trailing whitespaces removed
        return html.strip()