Commit: 108d48fa43fc49064b94b1d5ed7236fa89bd9781 - newsfour (git)

news4 - RSS aggrigation system

Commit MetaInfo

• Révision: 108d48fa43fc49064b94b1d5ed7236fa89bd9781 (tree)
• l'heure: 2012-10-03 05:24:10
• Auteur: hylom <hylom@hylo...>
• Commiter: hylom

Message de Log

implement some filters

Change Summary

• modified: fetcher.py
• add: filters/cleanup.py
• add: filters/gizmodo.py
• add: filters/pr_block.py
• delete: filters/{extractimg.py => img_block.py}
• modified: templates/index.tmpl.html

Modification

--- a/fetcher.py
+++ b/fetcher.py
@@ -38,19 +38,30 @@ class FeedFetcher(object):
             return None
         return entry
 
+    def _apply_filters(self, filters, entries):
+        for f in filters:
+            entry_filter = self._get_filter(f)
+            entries = [entry_filter(x) for x in entries]
+            # remove entry which is None
+            entries = [x for x in entries if x]
+        return entries
+
+    def _apply_pre_filters(self, entries):
+        return self._apply_filters(config['pre_filters'], entries)
+
+    def _apply_post_filters(self, entries):
+        return self._apply_filters(config['post_filters'], entries)
+
     def get_entries(self):
         'get entries'
         entries = self._fetch()
-        entries = [self._embeded_filter(x) for x in entries]
-        entries = [x for x in entries if x]
+        entries = self._apply_pre_filters(entries)
 
         if 'filter' in self._feed:
             filters = self._feed.get('filter', None)
-            for filter in filters:
-                entry_filter = self._get_filter(filter)
-                entries = [entry_filter(x) for x in entries]
-            # remove entry which is None
-            entries = [x for x in entries if x]
+            entries = self._apply_filters(filters, entries)
+
+        entries = self._apply_post_filters(entries)
         return entries
 
     def _get_filter(self, filter_name):
@@ -65,8 +76,16 @@ class FeedFetcher(object):
                           globals(),
                           locals(),
                           [filter_name,])
-        mod = mods.__getattribute__(filter_name)
+        try:
+            mod = mods.__getattribute__(filter_name)
+        except AttributeError:
+            raise FilterError(filter_name)
 
         # return module's entry_filter function
         return mod.entry_filter
         
+class FilterError(Exception):
+    def __init__(self, value):
+        self.value = value
+    def __str__(self):
+        return 'filter "' + self.value + '" is not found.'

--- /dev/null
+++ b/filters/cleanup.py
@@ -0,0 +1,27 @@
+# filter for Image extraction
+# -*- coding: utf-8 -*-
+
+import re
+
+re_blank = re.compile(r'<\s*(\w+)[^>]*>\s*</\s*\1\s*>')
+re_br = re.compile(r'<\s*br\s*/?>')
+
+def _replace_all(rex, text):
+    m = rex.search(text)
+    while(m):
+        text = rex.sub('', text)
+        m = rex.search(text)
+    return text
+
+def entry_filter(entry):
+    body = entry['body']
+
+    # 空のタグを削除
+    body = _replace_all(re_blank, body)
+
+    # brタグを削除
+    body = _replace_all(re_br, body)
+
+    entry['body'] = body
+    return entry
+

--- /dev/null
+++ b/filters/gizmodo.py
@@ -0,0 +1,30 @@
+# filter for slashdot.jp
+# -*- coding: utf-8 -*-
+
+import re
+
+re_read_all = re.compile(ur'''<p>\s*<a href=['"][^'"]+['"]>\s*すべて読む\s*</a>.*?</p>''')
+re_related = re.compile(ur'''<p>\s*関連ストーリー：.*?</p>''')
+re_topics = re.compile(ur'''<a href="http://slashdot.jp/stories/\w+">(.*?)</a>''')
+
+def entry_filter(entry):
+    # すべて読む、関連ストーリーを削除
+    body = entry['body']
+    topics = []
+    m = re_read_all.search(body)
+    if m:
+        s = m.group(0)
+        itr = re_topics.findall(s)
+        for items in itr:
+            topics.append(items)
+        
+    body = re_read_all.sub('', body)
+    body = re_related.sub('', body)
+    if 'tags' in entry:
+        entry['tags'].extend(topics)
+    else:
+        entry['tags'] = topics
+    entry['body'] = body
+
+    return entry
+

--- /dev/null
+++ b/filters/pr_block.py
@@ -0,0 +1,12 @@
+# PR filter
+# -*- coding: utf-8 -*-
+
+import re
+
+def entry_filter(entry):
+    if re.search(u'^(PR|AD)(:|：)', entry['title']):
+        print 'delete PR entry - %s' % entry['title']
+        return None
+    return entry
+
+

--- a/templates/index.tmpl.html
+++ b/templates/index.tmpl.html
@@ -64,13 +64,13 @@
                 <span>タグ：</span>
                 % for tag in entry.tags:
                 <span>${tag} </span>
+                % endfor
                 % if 'images' in entry:
                 <span>画像：</span>
                 % for imgurl in entry.images:
                 <span><a href="${imgurl}">*</a></span>
                 % endfor
                 % endif
-                % endfor
               </div>
             </div>
           </div>