mutantmonkey
GitHub
commit
8e984128a6
|
|
@ -10,36 +10,33 @@ modified from Wikipedia module
|
|||
author: mutantmonkey <mutantmonkey@mutantmonkey.in>
|
||||
"""
|
||||
|
||||
import re
|
||||
import web
|
||||
import wiki
|
||||
|
||||
wikiapi = 'https://wiki.archlinux.org/api.php?action=query&list=search&srsearch={0}&limit=1&prop=snippet&format=json'
|
||||
wikiuri = 'https://wiki.archlinux.org/index.php/{0}'
|
||||
wikisearch = 'https://wiki.archlinux.org/index.php/Special:Search?' \
|
||||
+ 'search={0}&fulltext=Search'
|
||||
endpoints = {
|
||||
'api': 'https://wiki.archlinux.org/api.php?action=query&list=search&srsearch={0}&limit=1&format=json',
|
||||
'url': 'https://wiki.archlinux.org/index.php/{0}',
|
||||
'search': 'https://wiki.archlinux.org/index.php/Special:Search?search={0}&fulltext=Search',
|
||||
}
|
||||
|
||||
def awik(phenny, input):
|
||||
origterm = input.groups()[1]
|
||||
if not origterm:
|
||||
""".awik <term> - Look up something on the ArchWiki."""
|
||||
|
||||
origterm = input.group(1)
|
||||
if not origterm:
|
||||
return phenny.say('Perhaps you meant ".awik dwm"?')
|
||||
|
||||
term = web.unquote(origterm)
|
||||
term = term[0].upper() + term[1:]
|
||||
term = term.replace(' ', '_')
|
||||
term, section = wiki.parse_term(origterm)
|
||||
|
||||
w = wiki.Wiki(wikiapi, wikiuri, wikisearch)
|
||||
w = wiki.Wiki(endpoints)
|
||||
match = w.search(term)
|
||||
|
||||
try:
|
||||
result = w.search(term)
|
||||
except web.ConnectionError:
|
||||
error = "Can't connect to wiki.archlinux.org ({0})".format(wikiuri.format(term))
|
||||
return phenny.say(error)
|
||||
if not match:
|
||||
phenny.say('Can\'t find anything in the ArchWiki for "{0}".'.format(term))
|
||||
return
|
||||
|
||||
if result is not None:
|
||||
phenny.say(result)
|
||||
else:
|
||||
phenny.say('Can\'t find anything in the ArchWiki for "{0}".'.format(origterm))
|
||||
snippet, url = wiki.extract_snippet(match, section)
|
||||
|
||||
phenny.say('"{0}" - {1}'.format(snippet, url))
|
||||
|
||||
awik.commands = ['awik']
|
||||
awik.priority = 'high'
|
||||
|
|
|
|||
|
|
@ -2,38 +2,76 @@
|
|||
test_archwiki.py - tests for the arch wiki module
|
||||
author: mutantmonkey <mutantmonkey@mutantmonkey.in>
|
||||
"""
|
||||
|
||||
import re
|
||||
import unittest
|
||||
from mock import MagicMock, Mock
|
||||
from mock import MagicMock
|
||||
from modules import archwiki
|
||||
import wiki
|
||||
|
||||
|
||||
class TestArchwiki(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.phenny = MagicMock()
|
||||
self.input = MagicMock()
|
||||
|
||||
self.term = None
|
||||
self.section = None
|
||||
|
||||
def prepare(self):
|
||||
if self.section:
|
||||
self.text = self.term + '#' + self.section
|
||||
url_text = wiki.format_term(self.term) +\
|
||||
'#' + wiki.format_section(self.section)
|
||||
else:
|
||||
self.text = self.term
|
||||
url_text = wiki.format_term(self.term)
|
||||
|
||||
self.input.group = lambda x: [None, self.text][x]
|
||||
self.url = 'https://wiki.archlinux.org/index.php/{0}'.format(url_text)
|
||||
|
||||
def check_snippet(self, output):
|
||||
self.assertIn(self.url, output)
|
||||
|
||||
for keyword in self.keywords:
|
||||
self.assertIn(keyword, output)
|
||||
|
||||
def test_awik(self):
|
||||
input = Mock(groups=lambda: ['', "KVM"])
|
||||
archwiki.awik(self.phenny, input)
|
||||
self.term = "OpenDMARC"
|
||||
self.prepare()
|
||||
|
||||
archwiki.awik(self.phenny, self.input)
|
||||
out = self.phenny.say.call_args[0][0]
|
||||
m = re.match('^.* - https:\/\/wiki\.archlinux\.org\/index\.php\/KVM$',
|
||||
out, flags=re.UNICODE)
|
||||
self.assertTrue(m)
|
||||
|
||||
self.keywords = ['policy', 'mail', 'transfer', 'providers']
|
||||
self.check_snippet(out)
|
||||
|
||||
def test_awik_fragment(self):
|
||||
self.term = "KVM"
|
||||
self.section = "Kernel support"
|
||||
self.prepare()
|
||||
|
||||
archwiki.awik(self.phenny, self.input)
|
||||
out = self.phenny.say.call_args[0][0]
|
||||
|
||||
self.keywords = ['kernel', 'modules', 'KVM', 'VIRTIO']
|
||||
self.check_snippet(out)
|
||||
|
||||
def test_awik_invalid(self):
|
||||
term = "KVM#Enabling_KSM"
|
||||
input = Mock(groups=lambda: ['', term])
|
||||
archwiki.awik(self.phenny, input)
|
||||
self.term = "KVM"
|
||||
self.section = "Enabling KSM"
|
||||
self.prepare()
|
||||
|
||||
self.phenny.say.assert_called_once_with( "Can't find anything in "\
|
||||
"the ArchWiki for \"{0}\".".format(term))
|
||||
archwiki.awik(self.phenny, self.input)
|
||||
out = self.phenny.say.call_args[0][0]
|
||||
|
||||
message = "No '{0}' section found.".format(self.section)
|
||||
self.assertEqual(out, '"{0}" - {1}'.format(message, self.url))
|
||||
|
||||
def test_awik_none(self):
|
||||
term = "Ajgoajh"
|
||||
input = Mock(groups=lambda: ['', term])
|
||||
archwiki.awik(self.phenny, input)
|
||||
self.term = "Ajgoajh"
|
||||
self.prepare()
|
||||
|
||||
self.phenny.say.assert_called_once_with( "Can't find anything in "\
|
||||
"the ArchWiki for \"{0}\".".format(term))
|
||||
archwiki.awik(self.phenny, self.input)
|
||||
out = self.phenny.say.call_args[0][0]
|
||||
|
||||
expected = "Can't find anything in the ArchWiki for \"{0}\"."
|
||||
self.assertEqual(out, expected.format(self.text))
|
||||
|
|
|
|||
|
|
@ -2,38 +2,77 @@
|
|||
test_vtluugwiki.py - tests for the VTLUUG wiki module
|
||||
author: mutantmonkey <mutantmonkey@mutantmonkey.in>
|
||||
"""
|
||||
|
||||
import re
|
||||
import unittest
|
||||
from mock import MagicMock, Mock
|
||||
from mock import MagicMock
|
||||
from modules import vtluugwiki
|
||||
import wiki
|
||||
|
||||
|
||||
class TestVtluugwiki(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.phenny = MagicMock()
|
||||
self.input = MagicMock()
|
||||
|
||||
self.term = None
|
||||
self.section = None
|
||||
|
||||
def prepare(self):
|
||||
if self.section:
|
||||
self.text = self.term + '#' + self.section
|
||||
url_text = wiki.format_term(self.term) +\
|
||||
'#' + wiki.format_section(self.section)
|
||||
else:
|
||||
self.text = self.term
|
||||
url_text = wiki.format_term(self.term)
|
||||
|
||||
self.input.groups.return_value = [None, self.text]
|
||||
self.url = 'https://vtluug.org/wiki/{0}'.format(url_text)
|
||||
|
||||
def check_snippet(self, output):
|
||||
self.assertIn(self.url, output)
|
||||
|
||||
for keyword in self.keywords:
|
||||
self.assertIn(keyword, output)
|
||||
|
||||
def test_vtluug(self):
|
||||
input = Mock(groups=lambda: ['', "VT-Wireless"])
|
||||
vtluugwiki.vtluug(self.phenny, input)
|
||||
self.term = "VT-Wireless"
|
||||
self.prepare()
|
||||
|
||||
vtluugwiki.vtluug(self.phenny, self.input)
|
||||
out = self.phenny.say.call_args[0][0]
|
||||
m = re.match('^.* - https:\/\/vtluug\.org\/wiki\/VT-Wireless$',
|
||||
out, flags=re.UNICODE)
|
||||
self.assertTrue(m)
|
||||
|
||||
self.keywords = ['campus', 'wireless', 'networks']
|
||||
self.check_snippet(out)
|
||||
|
||||
def test_vtluug_fragment(self):
|
||||
self.term = "EAP-TLS"
|
||||
self.section = "netctl"
|
||||
self.prepare()
|
||||
|
||||
vtluugwiki.vtluug(self.phenny, self.input)
|
||||
out = self.phenny.say.call_args[0][0]
|
||||
|
||||
self.keywords = ['Arch', 'Linux', 'netctl']
|
||||
self.check_snippet(out)
|
||||
|
||||
def test_vtluug_invalid(self):
|
||||
term = "EAP-TLS#netcfg"
|
||||
input = Mock(groups=lambda: ['', term])
|
||||
vtluugwiki.vtluug(self.phenny, input)
|
||||
self.term = "EAP-TLS"
|
||||
self.section = "netcfg"
|
||||
self.prepare()
|
||||
|
||||
self.phenny.say.assert_called_once_with( "Can't find anything in "\
|
||||
"the VTLUUG Wiki for \"{0}\".".format(term))
|
||||
vtluugwiki.vtluug(self.phenny, self.input)
|
||||
out = self.phenny.say.call_args[0][0]
|
||||
|
||||
message = "No '{0}' section found.".format(self.section)
|
||||
self.assertEqual(out, '"{0}" - {1}'.format(message, self.url))
|
||||
|
||||
def test_vtluug_none(self):
|
||||
term = "Ajgoajh"
|
||||
input = Mock(groups=lambda: ['', term])
|
||||
vtluugwiki.vtluug(self.phenny, input)
|
||||
self.term = "Ajgoajh"
|
||||
self.prepare()
|
||||
|
||||
vtluugwiki.vtluug(self.phenny, self.input)
|
||||
out = self.phenny.say.call_args[0][0]
|
||||
|
||||
expected = "Can't find anything in the VTLUUG Wiki for \"{0}\"."
|
||||
self.assertEqual(out, expected.format(self.text))
|
||||
|
||||
self.phenny.say.assert_called_once_with( "Can't find anything in "\
|
||||
"the VTLUUG Wiki for \"{0}\".".format(term))
|
||||
|
|
|
|||
|
|
@ -2,38 +2,76 @@
|
|||
test_wikipedia.py - tests for the wikipedia module
|
||||
author: mutantmonkey <mutantmonkey@mutantmonkey.in>
|
||||
"""
|
||||
|
||||
import re
|
||||
import unittest
|
||||
from mock import MagicMock, Mock
|
||||
from mock import MagicMock
|
||||
from modules import wikipedia
|
||||
import wiki
|
||||
|
||||
|
||||
class TestWikipedia(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.phenny = MagicMock()
|
||||
self.input = MagicMock()
|
||||
|
||||
self.term = None
|
||||
self.section = None
|
||||
|
||||
def prepare(self):
|
||||
if self.section:
|
||||
self.text = self.term + '#' + self.section
|
||||
url_text = wiki.format_term(self.term) +\
|
||||
'#' + wiki.format_section(self.section)
|
||||
else:
|
||||
self.text = self.term
|
||||
url_text = wiki.format_term(self.term)
|
||||
|
||||
self.input.groups.return_value = [None, self.text]
|
||||
self.url = 'https://en.wikipedia.org/wiki/{0}'.format(url_text)
|
||||
|
||||
def check_snippet(self, output):
|
||||
self.assertIn(self.url, output)
|
||||
|
||||
for keyword in self.keywords:
|
||||
self.assertIn(keyword, output)
|
||||
|
||||
def test_wik(self):
|
||||
input = Mock(groups=lambda: ['', "Human back"])
|
||||
wikipedia.wik(self.phenny, input)
|
||||
self.term = "Human back"
|
||||
self.prepare()
|
||||
|
||||
wikipedia.wik(self.phenny, self.input)
|
||||
out = self.phenny.say.call_args[0][0]
|
||||
m = re.match('^.* - https:\/\/en\.wikipedia\.org\/wiki\/Human_back$',
|
||||
out, flags=re.UNICODE)
|
||||
self.assertTrue(m)
|
||||
|
||||
self.keywords = ['human', 'back', 'body', 'buttocks', 'neck']
|
||||
self.check_snippet(out)
|
||||
|
||||
def test_wik_fragment(self):
|
||||
self.term = "New York City"
|
||||
self.section = "Climate"
|
||||
self.prepare()
|
||||
|
||||
wikipedia.wik(self.phenny, self.input)
|
||||
out = self.phenny.say.call_args[0][0]
|
||||
|
||||
self.keywords = ['New York', 'climate', 'humid', 'subtropical']
|
||||
self.check_snippet(out)
|
||||
|
||||
def test_wik_invalid(self):
|
||||
term = "New York City#Climate"
|
||||
input = Mock(groups=lambda: ['', term])
|
||||
wikipedia.wik(self.phenny, input)
|
||||
self.term = "New York City"
|
||||
self.section = "Physics"
|
||||
self.prepare()
|
||||
|
||||
self.phenny.say.assert_called_once_with( "Can't find anything in "\
|
||||
"Wikipedia for \"{0}\".".format(term))
|
||||
wikipedia.wik(self.phenny, self.input)
|
||||
out = self.phenny.say.call_args[0][0]
|
||||
|
||||
message = "No '{0}' section found.".format(self.section)
|
||||
self.assertEqual(out, '"{0}" - {1}'.format(message, self.url))
|
||||
|
||||
def test_wik_none(self):
|
||||
term = "Ajgoajh"
|
||||
input = Mock(groups=lambda: ['', term])
|
||||
wikipedia.wik(self.phenny, input)
|
||||
self.term = "Ajgoajh"
|
||||
self.prepare()
|
||||
|
||||
self.phenny.say.assert_called_once_with( "Can't find anything in "\
|
||||
"Wikipedia for \"{0}\".".format(term))
|
||||
wikipedia.wik(self.phenny, self.input)
|
||||
out = self.phenny.say.call_args[0][0]
|
||||
|
||||
expected = "Can't find anything in Wikipedia for \"{0}\"."
|
||||
self.assertEqual(out, expected.format(self.text))
|
||||
|
|
|
|||
|
|
@ -10,14 +10,13 @@ modified from Wikipedia module
|
|||
author: mutantmonkey <mutantmonkey@mutantmonkey.in>
|
||||
"""
|
||||
|
||||
import re
|
||||
import web
|
||||
import wiki
|
||||
|
||||
wikiapi = 'https://vtluug.org/w/api.php?action=query&list=search&srsearch={0}&limit=1&prop=snippet&format=json'
|
||||
wikiuri = 'https://vtluug.org/wiki/{0}'
|
||||
wikisearch = 'https://vtluug.org/wiki/Special:Search?' \
|
||||
+ 'search={0}&fulltext=Search'
|
||||
endpoints = {
|
||||
'api': 'https://vtluug.org/w/api.php?action=query&list=search&srsearch={0}&limit=1&prop=snippet&format=json',
|
||||
'url': 'https://vtluug.org/wiki/{0}',
|
||||
'search': 'https://vtluug.org/wiki/Special:Search?search={0}&fulltext=Search',
|
||||
}
|
||||
|
||||
def vtluug(phenny, input):
|
||||
""".vtluug <term> - Look up something on the VTLUUG wiki."""
|
||||
|
|
@ -26,22 +25,19 @@ def vtluug(phenny, input):
|
|||
if not origterm:
|
||||
return phenny.say('Perhaps you meant ".vtluug VT-Wireless"?')
|
||||
|
||||
term = web.unquote(origterm)
|
||||
term = term[0].upper() + term[1:]
|
||||
term = term.replace(' ', '_')
|
||||
term, section = wiki.parse_term(origterm)
|
||||
|
||||
w = wiki.Wiki(wikiapi, wikiuri, wikisearch)
|
||||
w = wiki.Wiki(endpoints)
|
||||
match = w.search(term)
|
||||
|
||||
try:
|
||||
result = w.search(term)
|
||||
except web.ConnectionError:
|
||||
error = "Can't connect to vtluug.org ({0})".format(wikiuri.format(term))
|
||||
return phenny.say(error)
|
||||
if not match:
|
||||
phenny.say('Can\'t find anything in the VTLUUG Wiki for "{0}".'.format(term))
|
||||
return
|
||||
|
||||
snippet, url = wiki.extract_snippet(match, section)
|
||||
|
||||
phenny.say('"{0}" - {1}'.format(snippet, url))
|
||||
|
||||
if result is not None:
|
||||
phenny.say(result)
|
||||
else:
|
||||
phenny.say('Can\'t find anything in the VTLUUG Wiki for "{0}".'.format(origterm))
|
||||
vtluug.commands = ['vtluug']
|
||||
vtluug.priority = 'high'
|
||||
|
||||
|
|
|
|||
|
|
@ -7,14 +7,13 @@ Licensed under the Eiffel Forum License 2.
|
|||
http://inamidst.com/phenny/
|
||||
"""
|
||||
|
||||
import re
|
||||
import web
|
||||
import wiki
|
||||
|
||||
wikiapi = 'https://en.wikipedia.org/w/api.php?action=query&list=search&srsearch={0}&limit=1&prop=snippet&format=json'
|
||||
wikiuri = 'https://en.wikipedia.org/wiki/{0}'
|
||||
wikisearch = 'https://en.wikipedia.org/wiki/Special:Search?' \
|
||||
+ 'search={0}&fulltext=Search'
|
||||
endpoints = {
|
||||
'api': 'https://en.wikipedia.org/w/api.php?format=json&action=query&list=search&srsearch={0}&prop=snippet&limit=1',
|
||||
'url': 'https://en.wikipedia.org/wiki/{0}',
|
||||
'search': 'https://en.wikipedia.org/wiki/Special:Search?search={0}&fulltext=Search',
|
||||
}
|
||||
|
||||
def wik(phenny, input):
|
||||
""".wik <term> - Look up something on Wikipedia."""
|
||||
|
|
@ -23,22 +22,19 @@ def wik(phenny, input):
|
|||
if not origterm:
|
||||
return phenny.say('Perhaps you meant ".wik Zen"?')
|
||||
|
||||
term = web.unquote(origterm)
|
||||
term = term[0].upper() + term[1:]
|
||||
term = term.replace(' ', '_')
|
||||
origterm = origterm.strip()
|
||||
term, section = wiki.parse_term(origterm)
|
||||
|
||||
w = wiki.Wiki(wikiapi, wikiuri, wikisearch)
|
||||
w = wiki.Wiki(endpoints)
|
||||
match = w.search(term)
|
||||
|
||||
try:
|
||||
result = w.search(term)
|
||||
except web.ConnectionError:
|
||||
error = "Can't connect to en.wikipedia.org ({0})".format(wikiuri.format(term))
|
||||
return phenny.say(error)
|
||||
|
||||
if result is not None:
|
||||
phenny.say(result)
|
||||
else:
|
||||
if not match:
|
||||
phenny.say('Can\'t find anything in Wikipedia for "{0}".'.format(origterm))
|
||||
return
|
||||
|
||||
snippet, url = wiki.extract_snippet(match, section)
|
||||
|
||||
phenny.say('"{0}" - {1}'.format(snippet, url))
|
||||
|
||||
wik.commands = ['wik']
|
||||
wik.priority = 'high'
|
||||
|
|
|
|||
39
phenny
39
phenny
|
|
@ -155,32 +155,21 @@ def main(argv=None):
|
|||
module = imp.load_source(name, config_name)
|
||||
module.filename = config_name
|
||||
|
||||
if not hasattr(module, 'prefix'):
|
||||
module.prefix = r'\.'
|
||||
defaults = {
|
||||
'prefix': r'\.',
|
||||
'name': 'Phenny Palmersbot, http://inamidst.com/phenny/',
|
||||
'port': 6667,
|
||||
'ssl': False,
|
||||
'ca_certs': None,
|
||||
'ssl_cert': None,
|
||||
'ssl_key': None,
|
||||
'ipv6': False,
|
||||
'password': None,
|
||||
}
|
||||
|
||||
if not hasattr(module, 'name'):
|
||||
module.name = 'Phenny Palmersbot, http://inamidst.com/phenny/'
|
||||
|
||||
if not hasattr(module, 'port'):
|
||||
module.port = 6667
|
||||
|
||||
if not hasattr(module, 'ssl'):
|
||||
module.ssl = False
|
||||
|
||||
if not hasattr(module, 'ca_certs'):
|
||||
module.ca_certs = None
|
||||
|
||||
if not hasattr(module, 'ssl_cert'):
|
||||
module.ssl_cert = None
|
||||
|
||||
if not hasattr(module, 'ssl_key'):
|
||||
module.ssl_key = None
|
||||
|
||||
if not hasattr(module, 'ipv6'):
|
||||
module.ipv6 = False
|
||||
|
||||
if not hasattr(module, 'password'):
|
||||
module.password = None
|
||||
for key, value in defaults.items():
|
||||
if not hasattr(module, key):
|
||||
setattr(module, key, value)
|
||||
|
||||
if module.host == 'irc.example.net':
|
||||
error = ('Error: you must edit the config file first!\n' +
|
||||
|
|
|
|||
142
wiki.py
142
wiki.py
|
|
@ -1,5 +1,8 @@
|
|||
import json
|
||||
import lxml.html
|
||||
import re
|
||||
from requests.exceptions import HTTPError
|
||||
from urllib.parse import quote, unquote
|
||||
import web
|
||||
|
||||
|
||||
|
|
@ -16,15 +19,104 @@ abbrs = ['etc', 'ca', 'cf', 'Co', 'Ltd', 'Inc', 'Mt', 'Mr', 'Mrs',
|
|||
'syn', 'transl', 'sess', 'fl', 'Op', 'Dec', 'Brig', 'Gen'] \
|
||||
+ list('ABCDEFGHIJKLMNOPQRSTUVWXYZ') \
|
||||
+ list('abcdefghijklmnopqrstuvwxyz')
|
||||
t_sentence = r'^.{5,}?(?<!\b%s)(?:\.(?=[\[ ][A-Z0-9]|\Z)|\Z)'
|
||||
r_sentence = re.compile(t_sentence % r')(?<!\b'.join(abbrs))
|
||||
no_abbr = ''.join('(?<! ' + abbr + ')' for abbr in abbrs)
|
||||
breaks = re.compile('({})+'.format('|'.join([
|
||||
no_abbr + '[.!?](?:[ \n]|\[[0-9]+\]|$)',
|
||||
'。', '｡', '．', '！', '？',
|
||||
])))
|
||||
|
||||
def format_term(term):
|
||||
term = term.replace(' ', '_')
|
||||
term = term[0].upper() + term[1:]
|
||||
return term
|
||||
|
||||
def deformat_term(term):
|
||||
term = term.replace('_', ' ')
|
||||
return term
|
||||
|
||||
def format_section(section):
|
||||
section = section.replace(' ', '_')
|
||||
section = quote(section)
|
||||
section = section.replace('%', '.')
|
||||
section = section.replace(".3A", ":")
|
||||
return section
|
||||
|
||||
def parse_term(origterm):
|
||||
if "#" in origterm:
|
||||
term, section = origterm.split("#")[:2]
|
||||
term, section = term.strip(), section.strip()
|
||||
else:
|
||||
term = origterm.strip()
|
||||
section = None
|
||||
|
||||
return (term, section)
|
||||
|
||||
def good_content(text, content):
|
||||
if text.tag not in ['p', 'ul', 'ol']:
|
||||
return False
|
||||
|
||||
if not content.strip():
|
||||
return False
|
||||
|
||||
if not breaks.search(content):
|
||||
return False
|
||||
|
||||
if text.find(".//span[@id='coordinates']") is not None:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def search_content(text):
|
||||
if text is None:
|
||||
return None
|
||||
|
||||
content = text.text_content()
|
||||
|
||||
while not good_content(text, content):
|
||||
text = text.getnext()
|
||||
|
||||
if text is None:
|
||||
return None
|
||||
|
||||
content = text.text_content()
|
||||
|
||||
return content
|
||||
|
||||
def extract_snippet(match, origsection=None):
|
||||
html, url = match
|
||||
page = lxml.html.fromstring(html)
|
||||
article = page.get_element_by_id('mw-content-text')
|
||||
|
||||
if origsection:
|
||||
section = format_section(origsection)
|
||||
text = article.find(".//span[@id='{0}']".format(section))
|
||||
url += "#" + unquote(section)
|
||||
|
||||
if text is None:
|
||||
return ("No '{0}' section found.".format(origsection), url)
|
||||
|
||||
text = text.getparent().getnext()
|
||||
content = search_content(text)
|
||||
|
||||
if text is None:
|
||||
return ("No section text found.", url)
|
||||
else:
|
||||
text = article.find('./p')
|
||||
|
||||
if text is None:
|
||||
text = article.find('./div/p')
|
||||
|
||||
content = search_content(text)
|
||||
|
||||
if text is None:
|
||||
return ("No introduction text found.", url)
|
||||
|
||||
sentences = [x.strip() for x in breaks.split(content)]
|
||||
return (sentences[0], url)
|
||||
|
||||
class Wiki(object):
|
||||
def __init__(self, api, url, searchurl=""):
|
||||
self.api = api
|
||||
self.url = url
|
||||
self.searchurl = searchurl
|
||||
def __init__(self, endpoints):
|
||||
self.endpoints = endpoints
|
||||
|
||||
@staticmethod
|
||||
def unescape(s):
|
||||
|
|
@ -41,18 +133,34 @@ class Wiki(object):
|
|||
html = r_whitespace.sub(' ', html)
|
||||
return Wiki.unescape(html).strip()
|
||||
|
||||
def search(self, term, last=False):
|
||||
url = self.api.format(term)
|
||||
bytes = web.get(url)
|
||||
def search(self, term):
|
||||
try:
|
||||
result = json.loads(bytes)
|
||||
result = result['query']['search']
|
||||
if len(result) <= 0:
|
||||
return None
|
||||
exactterm = format_term(term)
|
||||
exactterm = quote(exactterm)
|
||||
exacturl = self.endpoints['url'].format(exactterm)
|
||||
html = web.get(exacturl)
|
||||
return (html, exacturl)
|
||||
except HTTPError:
|
||||
pass
|
||||
|
||||
term = deformat_term(term)
|
||||
term = quote(term)
|
||||
apiurl = self.endpoints['api'].format(term)
|
||||
|
||||
try:
|
||||
result = json.loads(web.get(apiurl))
|
||||
except ValueError:
|
||||
return None
|
||||
term = result[0]['title']
|
||||
term = term.replace(' ', '_')
|
||||
snippet = self.text(result[0]['snippet'])
|
||||
return "{0} - {1}".format(snippet, self.url.format(term))
|
||||
|
||||
result = result['query']['search']
|
||||
|
||||
if not result:
|
||||
return None
|
||||
|
||||
term = result[0]['title']
|
||||
term = format_term(term)
|
||||
term = quote(term)
|
||||
|
||||
url = self.endpoints['url'].format(term)
|
||||
html = web.get(url)
|
||||
return (html, url)
|
||||
|
|
|
|||
Loading…
Reference in New Issue