Merge pull request #82 from Androbin/mutant

Some changes from the apertium fork

modules/archwiki.py

@@ -10,36 +10,33 @@ modified from Wikipedia module
 author: mutantmonkey <mutantmonkey@mutantmonkey.in>
 """
 
-import re
-import web
 import wiki
 
-wikiapi = 'https://wiki.archlinux.org/api.php?action=query&list=search&srsearch={0}&limit=1&prop=snippet&format=json'
+endpoints = {
-wikiuri = 'https://wiki.archlinux.org/index.php/{0}'
+    'api': 'https://wiki.archlinux.org/api.php?action=query&list=search&srsearch={0}&limit=1&format=json',
-wikisearch = 'https://wiki.archlinux.org/index.php/Special:Search?' \
+    'url': 'https://wiki.archlinux.org/index.php/{0}',
-                          + 'search={0}&fulltext=Search'
+    'search': 'https://wiki.archlinux.org/index.php/Special:Search?search={0}&fulltext=Search',
+}
 
 def awik(phenny, input): 
-    origterm = input.groups()[1]
+    """.awik <term> - Look up something on the ArchWiki."""
+
+    origterm = input.group(1)
     if not origterm:
         return phenny.say('Perhaps you meant ".awik dwm"?')
 
-    term = web.unquote(origterm)
+    term, section = wiki.parse_term(origterm)
-    term = term[0].upper() + term[1:]
-    term = term.replace(' ', '_')
 
-    w = wiki.Wiki(wikiapi, wikiuri, wikisearch)
+    w = wiki.Wiki(endpoints)
+    match = w.search(term)
 
-    try:
+    if not match:
-        result = w.search(term)
+        phenny.say('Can\'t find anything in the ArchWiki for "{0}".'.format(term))
-    except web.ConnectionError:
+        return
-        error = "Can't connect to wiki.archlinux.org ({0})".format(wikiuri.format(term))
-        return phenny.say(error)
 
-    if result is not None: 
+    snippet, url = wiki.extract_snippet(match, section)
-        phenny.say(result)
+
-    else:
+    phenny.say('"{0}" - {1}'.format(snippet, url))
-        phenny.say('Can\'t find anything in the ArchWiki for "{0}".'.format(origterm))
 
 awik.commands = ['awik']
 awik.priority = 'high'

modules/test/test_archwiki.py

@@ -2,38 +2,76 @@
 test_archwiki.py - tests for the arch wiki module
 author: mutantmonkey <mutantmonkey@mutantmonkey.in>
 """
-
-import re
 import unittest
-from mock import MagicMock, Mock
+from mock import MagicMock
 from modules import archwiki
+import wiki
 
 
 class TestArchwiki(unittest.TestCase):
     def setUp(self):
         self.phenny = MagicMock()
+        self.input = MagicMock()
+
+        self.term = None
+        self.section = None
+
+    def prepare(self):
+        if self.section:
+            self.text = self.term + '#' + self.section
+            url_text = wiki.format_term(self.term) +\
+                '#' + wiki.format_section(self.section)
+        else:
+            self.text = self.term
+            url_text = wiki.format_term(self.term)
+
+        self.input.group = lambda x: [None, self.text][x]
+        self.url = 'https://wiki.archlinux.org/index.php/{0}'.format(url_text)
+
+    def check_snippet(self, output):
+        self.assertIn(self.url, output)
+
+        for keyword in self.keywords:
+            self.assertIn(keyword, output)
 
     def test_awik(self):
-        input = Mock(groups=lambda: ['', "KVM"])
+        self.term = "OpenDMARC"
-        archwiki.awik(self.phenny, input)
+        self.prepare()
 
+        archwiki.awik(self.phenny, self.input)
         out = self.phenny.say.call_args[0][0]
-        m = re.match('^.* - https:\/\/wiki\.archlinux\.org\/index\.php\/KVM$',
+
-                out, flags=re.UNICODE)
+        self.keywords = ['policy', 'mail', 'transfer', 'providers']
-        self.assertTrue(m)
+        self.check_snippet(out)
+
+    def test_awik_fragment(self):
+        self.term = "KVM"
+        self.section = "Kernel support"
+        self.prepare()
+
+        archwiki.awik(self.phenny, self.input)
+        out = self.phenny.say.call_args[0][0]
+
+        self.keywords = ['kernel', 'modules', 'KVM', 'VIRTIO']
+        self.check_snippet(out)
 
     def test_awik_invalid(self):
-        term = "KVM#Enabling_KSM"
+        self.term = "KVM"
-        input = Mock(groups=lambda: ['', term])
+        self.section = "Enabling KSM"
-        archwiki.awik(self.phenny, input)
+        self.prepare()
 
-        self.phenny.say.assert_called_once_with( "Can't find anything in "\
+        archwiki.awik(self.phenny, self.input)
-                "the ArchWiki for \"{0}\".".format(term))
+        out = self.phenny.say.call_args[0][0]
+
+        message = "No '{0}' section found.".format(self.section)
+        self.assertEqual(out, '"{0}" - {1}'.format(message, self.url))
 
     def test_awik_none(self):
-        term = "Ajgoajh"
+        self.term = "Ajgoajh"
-        input = Mock(groups=lambda: ['', term])
+        self.prepare()
-        archwiki.awik(self.phenny, input)
 
-        self.phenny.say.assert_called_once_with( "Can't find anything in "\
+        archwiki.awik(self.phenny, self.input)
-                "the ArchWiki for \"{0}\".".format(term))
+        out = self.phenny.say.call_args[0][0]
+
+        expected = "Can't find anything in the ArchWiki for \"{0}\"."
+        self.assertEqual(out, expected.format(self.text))

modules/test/test_vtluugwiki.py

@@ -2,38 +2,77 @@
 test_vtluugwiki.py - tests for the VTLUUG wiki module
 author: mutantmonkey <mutantmonkey@mutantmonkey.in>
 """
-
-import re
 import unittest
-from mock import MagicMock, Mock
+from mock import MagicMock
 from modules import vtluugwiki
+import wiki
 
 
 class TestVtluugwiki(unittest.TestCase):
     def setUp(self):
         self.phenny = MagicMock()
+        self.input = MagicMock()
+
+        self.term = None
+        self.section = None
+
+    def prepare(self):
+        if self.section:
+            self.text = self.term + '#' + self.section
+            url_text = wiki.format_term(self.term) +\
+                '#' + wiki.format_section(self.section)
+        else:
+            self.text = self.term
+            url_text = wiki.format_term(self.term)
+
+        self.input.groups.return_value = [None, self.text]
+        self.url = 'https://vtluug.org/wiki/{0}'.format(url_text)
+
+    def check_snippet(self, output):
+        self.assertIn(self.url, output)
+
+        for keyword in self.keywords:
+            self.assertIn(keyword, output)
 
     def test_vtluug(self):
-        input = Mock(groups=lambda: ['', "VT-Wireless"])
+        self.term = "VT-Wireless"
-        vtluugwiki.vtluug(self.phenny, input)
+        self.prepare()
 
+        vtluugwiki.vtluug(self.phenny, self.input)
         out = self.phenny.say.call_args[0][0]
-        m = re.match('^.* - https:\/\/vtluug\.org\/wiki\/VT-Wireless$',
+
-                out, flags=re.UNICODE)
+        self.keywords = ['campus', 'wireless', 'networks']
-        self.assertTrue(m)
+        self.check_snippet(out)
+
+    def test_vtluug_fragment(self):
+        self.term = "EAP-TLS"
+        self.section = "netctl"
+        self.prepare()
+
+        vtluugwiki.vtluug(self.phenny, self.input)
+        out = self.phenny.say.call_args[0][0]
+
+        self.keywords = ['Arch', 'Linux', 'netctl']
+        self.check_snippet(out)
 
     def test_vtluug_invalid(self):
-        term = "EAP-TLS#netcfg"
+        self.term = "EAP-TLS"
-        input = Mock(groups=lambda: ['', term])
+        self.section = "netcfg"
-        vtluugwiki.vtluug(self.phenny, input)
+        self.prepare()
 
-        self.phenny.say.assert_called_once_with( "Can't find anything in "\
+        vtluugwiki.vtluug(self.phenny, self.input)
-                "the VTLUUG Wiki for \"{0}\".".format(term))
+        out = self.phenny.say.call_args[0][0]
+
+        message = "No '{0}' section found.".format(self.section)
+        self.assertEqual(out, '"{0}" - {1}'.format(message, self.url))
 
     def test_vtluug_none(self):
-        term = "Ajgoajh"
+        self.term = "Ajgoajh"
-        input = Mock(groups=lambda: ['', term])
+        self.prepare()
-        vtluugwiki.vtluug(self.phenny, input)
+
+        vtluugwiki.vtluug(self.phenny, self.input)
+        out = self.phenny.say.call_args[0][0]
+
+        expected = "Can't find anything in the VTLUUG Wiki for \"{0}\"."
+        self.assertEqual(out, expected.format(self.text))
 
-        self.phenny.say.assert_called_once_with( "Can't find anything in "\
-                "the VTLUUG Wiki for \"{0}\".".format(term))

modules/test/test_wikipedia.py

@@ -2,38 +2,76 @@
 test_wikipedia.py - tests for the wikipedia module
 author: mutantmonkey <mutantmonkey@mutantmonkey.in>
 """
-
-import re
 import unittest
-from mock import MagicMock, Mock
+from mock import MagicMock
 from modules import wikipedia
+import wiki
 
 
 class TestWikipedia(unittest.TestCase):
     def setUp(self):
         self.phenny = MagicMock()
+        self.input = MagicMock()
+
+        self.term = None
+        self.section = None
+
+    def prepare(self):
+        if self.section:
+            self.text = self.term + '#' + self.section
+            url_text = wiki.format_term(self.term) +\
+                '#' + wiki.format_section(self.section)
+        else:
+            self.text = self.term
+            url_text = wiki.format_term(self.term)
+
+        self.input.groups.return_value = [None, self.text]
+        self.url = 'https://en.wikipedia.org/wiki/{0}'.format(url_text)
+
+    def check_snippet(self, output):
+        self.assertIn(self.url, output)
+
+        for keyword in self.keywords:
+            self.assertIn(keyword, output)
 
     def test_wik(self):
-        input = Mock(groups=lambda: ['', "Human back"])
+        self.term = "Human back"
-        wikipedia.wik(self.phenny, input)
+        self.prepare()
 
+        wikipedia.wik(self.phenny, self.input)
         out = self.phenny.say.call_args[0][0]
-        m = re.match('^.* - https:\/\/en\.wikipedia\.org\/wiki\/Human_back$',
+
-                out, flags=re.UNICODE)
+        self.keywords = ['human', 'back', 'body', 'buttocks', 'neck']
-        self.assertTrue(m)
+        self.check_snippet(out)
+
+    def test_wik_fragment(self):
+        self.term = "New York City"
+        self.section = "Climate"
+        self.prepare()
+
+        wikipedia.wik(self.phenny, self.input)
+        out = self.phenny.say.call_args[0][0]
+
+        self.keywords = ['New York', 'climate', 'humid', 'subtropical']
+        self.check_snippet(out)
 
     def test_wik_invalid(self):
-        term = "New York City#Climate"
+        self.term = "New York City"
-        input = Mock(groups=lambda: ['', term])
+        self.section = "Physics"
-        wikipedia.wik(self.phenny, input)
+        self.prepare()
 
-        self.phenny.say.assert_called_once_with( "Can't find anything in "\
+        wikipedia.wik(self.phenny, self.input)
-                "Wikipedia for \"{0}\".".format(term))
+        out = self.phenny.say.call_args[0][0]
+
+        message = "No '{0}' section found.".format(self.section)
+        self.assertEqual(out, '"{0}" - {1}'.format(message, self.url))
 
     def test_wik_none(self):
-        term = "Ajgoajh"
+        self.term = "Ajgoajh"
-        input = Mock(groups=lambda: ['', term])
+        self.prepare()
-        wikipedia.wik(self.phenny, input)
 
-        self.phenny.say.assert_called_once_with( "Can't find anything in "\
+        wikipedia.wik(self.phenny, self.input)
-                "Wikipedia for \"{0}\".".format(term))
+        out = self.phenny.say.call_args[0][0]
+
+        expected = "Can't find anything in Wikipedia for \"{0}\"."
+        self.assertEqual(out, expected.format(self.text))

modules/vtluugwiki.py

@@ -10,14 +10,13 @@ modified from Wikipedia module
 author: mutantmonkey <mutantmonkey@mutantmonkey.in>
 """
 
-import re
-import web
 import wiki
 
-wikiapi = 'https://vtluug.org/w/api.php?action=query&list=search&srsearch={0}&limit=1&prop=snippet&format=json'
+endpoints = {
-wikiuri = 'https://vtluug.org/wiki/{0}'
+    'api': 'https://vtluug.org/w/api.php?action=query&list=search&srsearch={0}&limit=1&prop=snippet&format=json',
-wikisearch = 'https://vtluug.org/wiki/Special:Search?' \
+    'url': 'https://vtluug.org/wiki/{0}',
-                          + 'search={0}&fulltext=Search'
+    'search': 'https://vtluug.org/wiki/Special:Search?search={0}&fulltext=Search',
+}
 
 def vtluug(phenny, input): 
     """.vtluug <term> - Look up something on the VTLUUG wiki."""
@@ -26,22 +25,19 @@ def vtluug(phenny, input):
     if not origterm: 
         return phenny.say('Perhaps you meant ".vtluug VT-Wireless"?')
 
-    term = web.unquote(origterm)
+    term, section = wiki.parse_term(origterm)
-    term = term[0].upper() + term[1:]
-    term = term.replace(' ', '_')
 
-    w = wiki.Wiki(wikiapi, wikiuri, wikisearch)
+    w = wiki.Wiki(endpoints)
+    match = w.search(term)
 
-    try:
+    if not match:
-        result = w.search(term)
+        phenny.say('Can\'t find anything in the VTLUUG Wiki for "{0}".'.format(term))
-    except web.ConnectionError:
+        return
-        error = "Can't connect to vtluug.org ({0})".format(wikiuri.format(term))
+
-        return phenny.say(error)
+    snippet, url = wiki.extract_snippet(match, section)
+
+    phenny.say('"{0}" - {1}'.format(snippet, url))
 
-    if result is not None: 
-        phenny.say(result)
-    else:
-        phenny.say('Can\'t find anything in the VTLUUG Wiki for "{0}".'.format(origterm))
 vtluug.commands = ['vtluug']
 vtluug.priority = 'high'
 

modules/wikipedia.py

@@ -7,14 +7,13 @@ Licensed under the Eiffel Forum License 2.
 http://inamidst.com/phenny/
 """
 
-import re
-import web
 import wiki
 
-wikiapi = 'https://en.wikipedia.org/w/api.php?action=query&list=search&srsearch={0}&limit=1&prop=snippet&format=json'
+endpoints = {
-wikiuri = 'https://en.wikipedia.org/wiki/{0}'
+    'api': 'https://en.wikipedia.org/w/api.php?format=json&action=query&list=search&srsearch={0}&prop=snippet&limit=1',
-wikisearch = 'https://en.wikipedia.org/wiki/Special:Search?' \
+    'url': 'https://en.wikipedia.org/wiki/{0}',
-                          + 'search={0}&fulltext=Search'
+    'search': 'https://en.wikipedia.org/wiki/Special:Search?search={0}&fulltext=Search',
+}
 
 def wik(phenny, input): 
     """.wik <term> - Look up something on Wikipedia."""
@@ -23,22 +22,19 @@ def wik(phenny, input):
     if not origterm: 
         return phenny.say('Perhaps you meant ".wik Zen"?')
 
-    term = web.unquote(origterm)
+    origterm = origterm.strip()
-    term = term[0].upper() + term[1:]
+    term, section = wiki.parse_term(origterm)
-    term = term.replace(' ', '_')
 
-    w = wiki.Wiki(wikiapi, wikiuri, wikisearch)
+    w = wiki.Wiki(endpoints)
+    match = w.search(term)
 
-    try:
+    if not match:
-        result = w.search(term)
-    except web.ConnectionError:
-        error = "Can't connect to en.wikipedia.org ({0})".format(wikiuri.format(term))
-        return phenny.say(error)
-
-    if result is not None: 
-        phenny.say(result)
-    else:
         phenny.say('Can\'t find anything in Wikipedia for "{0}".'.format(origterm))
+        return
+
+    snippet, url = wiki.extract_snippet(match, section)
+
+    phenny.say('"{0}" - {1}'.format(snippet, url))
 
 wik.commands = ['wik']
 wik.priority = 'high'

phenny

@@ -155,32 +155,21 @@ def main(argv=None):
         module = imp.load_source(name, config_name)
         module.filename = config_name
 
-        if not hasattr(module, 'prefix'):
+        defaults = {
-            module.prefix = r'\.'
+            'prefix': r'\.',
+            'name': 'Phenny Palmersbot, http://inamidst.com/phenny/',
+            'port': 6667,
+            'ssl': False,
+            'ca_certs': None,
+            'ssl_cert': None,
+            'ssl_key': None,
+            'ipv6': False,
+            'password': None,
+        }
 
-        if not hasattr(module, 'name'):
+        for key, value in defaults.items():
-            module.name = 'Phenny Palmersbot, http://inamidst.com/phenny/'
+            if not hasattr(module, key):
-
+                setattr(module, key, value)
-        if not hasattr(module, 'port'):
-            module.port = 6667
-
-        if not hasattr(module, 'ssl'):
-            module.ssl = False
-
-        if not hasattr(module, 'ca_certs'):
-            module.ca_certs = None
-
-        if not hasattr(module, 'ssl_cert'):
-            module.ssl_cert = None
-
-        if not hasattr(module, 'ssl_key'):
-            module.ssl_key = None
-
-        if not hasattr(module, 'ipv6'):
-            module.ipv6 = False
-
-        if not hasattr(module, 'password'):
-            module.password = None
 
         if module.host == 'irc.example.net':
             error = ('Error: you must edit the config file first!\n' +

wiki.py

@@ -1,5 +1,8 @@
 import json
+import lxml.html
 import re
+from requests.exceptions import HTTPError
+from urllib.parse import quote, unquote
 import web
 
 
@@ -16,15 +19,104 @@ abbrs = ['etc', 'ca', 'cf', 'Co', 'Ltd', 'Inc', 'Mt', 'Mr', 'Mrs',
          'syn', 'transl', 'sess', 'fl', 'Op', 'Dec', 'Brig', 'Gen'] \
    + list('ABCDEFGHIJKLMNOPQRSTUVWXYZ') \
    + list('abcdefghijklmnopqrstuvwxyz')
-t_sentence = r'^.{5,}?(?<!\b%s)(?:\.(?=[\[ ][A-Z0-9]|\Z)|\Z)'
+no_abbr = ''.join('(?<! ' + abbr + ')' for abbr in abbrs)
-r_sentence = re.compile(t_sentence % r')(?<!\b'.join(abbrs))
+breaks = re.compile('({})+'.format('|'.join([
+    no_abbr + '[.!?](?:[ \n]|\[[0-9]+\]|$)',
+    '。', '｡', '．', '！', '？',
+])))
 
+def format_term(term):
+    term = term.replace(' ', '_')
+    term = term[0].upper() + term[1:]
+    return term
+
+def deformat_term(term):
+    term = term.replace('_', ' ')
+    return term
+
+def format_section(section):
+    section = section.replace(' ', '_')
+    section = quote(section)
+    section = section.replace('%', '.')
+    section = section.replace(".3A", ":")
+    return section
+
+def parse_term(origterm):
+    if "#" in origterm:
+        term, section = origterm.split("#")[:2]
+        term, section = term.strip(), section.strip()
+    else:
+        term = origterm.strip()
+        section = None
+
+    return (term, section)
+
+def good_content(text, content):
+    if text.tag not in ['p', 'ul', 'ol']:
+        return False
+
+    if not content.strip():
+        return False
+
+    if not breaks.search(content):
+        return False
+
+    if text.find(".//span[@id='coordinates']") is not None:
+        return False
+
+    return True
+
+def search_content(text):
+    if text is None:
+        return None
+
+    content = text.text_content()
+
+    while not good_content(text, content):
+        text = text.getnext()
+
+        if text is None:
+            return None
+
+        content = text.text_content()
+
+    return content
+
+def extract_snippet(match, origsection=None):
+    html, url = match
+    page = lxml.html.fromstring(html)
+    article = page.get_element_by_id('mw-content-text')
+
+    if origsection:
+        section = format_section(origsection)
+        text = article.find(".//span[@id='{0}']".format(section))
+        url += "#" + unquote(section)
+
+        if text is None:
+            return ("No '{0}' section found.".format(origsection), url)
+
+        text = text.getparent().getnext()
+        content = search_content(text)
+
+        if text is None:
+            return ("No section text found.", url)
+    else:
+        text = article.find('./p')
+
+        if text is None:
+            text = article.find('./div/p')
+
+        content = search_content(text)
+
+        if text is None:
+            return ("No introduction text found.", url)
+
+    sentences = [x.strip() for x in breaks.split(content)]
+    return (sentences[0], url)
 
 class Wiki(object):
-    def __init__(self, api, url, searchurl=""):
+    def __init__(self, endpoints):
-        self.api = api
+        self.endpoints = endpoints
-        self.url = url
-        self.searchurl = searchurl
 
     @staticmethod
     def unescape(s): 
@@ -41,18 +133,34 @@ class Wiki(object):
         html = r_whitespace.sub(' ', html)
         return Wiki.unescape(html).strip()
 
-    def search(self, term, last=False):
+    def search(self, term):
-        url = self.api.format(term)
-        bytes = web.get(url)
         try:
-            result = json.loads(bytes)
+            exactterm = format_term(term)
-            result = result['query']['search']
+            exactterm = quote(exactterm)
-            if len(result) <= 0:
+            exacturl = self.endpoints['url'].format(exactterm)
-                return None
+            html = web.get(exacturl)
+            return (html, exacturl)
+        except HTTPError:
+            pass
+
+        term = deformat_term(term)
+        term = quote(term)
+        apiurl = self.endpoints['api'].format(term)
+
+        try:
+            result = json.loads(web.get(apiurl))
         except ValueError:
             return None
-        term = result[0]['title']
-        term = term.replace(' ', '_')
-        snippet = self.text(result[0]['snippet'])
-        return "{0} - {1}".format(snippet, self.url.format(term))
 
+        result = result['query']['search']
+
+        if not result:
+            return None
+
+        term = result[0]['title']
+        term = format_term(term)
+        term = quote(term)
+
+        url = self.endpoints['url'].format(term)
+        html = web.get(url)
+        return (html, url)