From 0e0cc52f0c2320a341c755b89056de29ab3c6d18 Mon Sep 17 00:00:00 2001 From: "St.Huang" Date: Mon, 30 Jul 2018 17:42:14 +0800 Subject: [PATCH] Adapt to anki 2.1 --- addons/fastwq/service/base.py | 46 ++++++++++++++++--- addons/fastwq/service/dict/LDOCE6.py | 17 +------ addons/fastwq/service/dict/baicizhan.py | 13 ++---- addons/fastwq/service/dict/baidu_chinese.py | 28 ++++------- addons/fastwq/service/dict/bing.py | 15 +++--- addons/fastwq/service/dict/bing3tp.py | 15 +++--- addons/fastwq/service/dict/cambridge.py | 33 ++++--------- addons/fastwq/service/dict/esdict.py | 39 ++++++---------- addons/fastwq/service/dict/frdic.py | 45 +++++++----------- addons/fastwq/service/dict/iciba.py | 22 ++++----- addons/fastwq/service/dict/longman.py | 28 +++-------- addons/fastwq/service/dict/minidict.py | 9 ++-- addons/fastwq/service/dict/oxford.py | 9 ++-- addons/fastwq/service/dict/oxford_learning.py | 11 ++--- addons/fastwq/service/dict/remotemdx.py | 12 ++--- addons/fastwq/service/dict/yahoo.py | 25 ++-------- addons/fastwq/service/dict/youdao.py | 29 +++++------- addons/fastwq/service/dict/youdaofr.py | 15 +++--- addons/fastwq/service/dict/youdaoko.py | 17 +++---- addons21/fastwq/service/base.py | 6 ++- addons21/fastwq/service/dict/bing.py | 3 +- addons21/fastwq/service/dict/bing3tp.py | 3 +- addons21/fastwq/service/dict/cambridge.py | 3 +- addons21/fastwq/service/dict/esdict.py | 3 +- addons21/fastwq/service/dict/longman.py | 3 +- addons21/fastwq/service/dict/yahoo.py | 8 +--- addons21/fastwq/service/dict/youdao.py | 4 +- 27 files changed, 176 insertions(+), 285 deletions(-) diff --git a/addons/fastwq/service/base.py b/addons/fastwq/service/base.py index 96b74a0..3bc13fa 100644 --- a/addons/fastwq/service/base.py +++ b/addons/fastwq/service/base.py @@ -33,6 +33,7 @@ import random from collections import defaultdict from functools import wraps from hashlib import md5 +from hashlib import sha1 import cookielib from aqt import mw @@ -50,11 +51,29 @@ except ImportError: __all__ = [ - 'register', 'export', 'copy_static_file', 'with_styles', 'parse_html', 'service_wrap', + 'register', 'export', 'copy_static_file', 'with_styles', 'parse_html', 'service_wrap', 'get_hex_name', 'Service', 'WebService', 'LocalService', 'MdxService', 'StardictService', 'QueryResult' ] +def get_hex_name(prefix, val, suffix): + ''' get sha1 hax name ''' + hex_digest = sha1( + val.encode('utf-8') if isinstance(val, unicode) + else val + ).hexdigest().lower() + name = '.'.join([ + '-'.join([ + prefix, hex_digest[:8], hex_digest[8:16], + hex_digest[16:24], hex_digest[24:32], hex_digest[32:], + ]), + suffix, + ]) + return name + +def _is_method_or_func(object): + return inspect.isfunction(object) or inspect.ismethod(object) + def register(labels): """ register the dict service with a labels, which will be shown in the dicts list. @@ -62,7 +81,7 @@ def register(labels): def _deco(cls): cls.__register_label__ = _cl(labels) - methods = inspect.getmembers(cls, predicate=inspect.ismethod) + methods = inspect.getmembers(cls, predicate=_is_method_or_func) exports = [] for method in methods: attrs = getattr(method[1], '__export_attrs__', None) @@ -198,10 +217,16 @@ class Service(object): return result def cached(self, key): - return (self.word in self.cache) and self.cache[self.word].has_key(key) + return (self.word in self.cache) and (key in self.cache[self.word]) def cache_result(self, key): return self.cache[self.word].get(key, u'') + + def _get_from_api(self): + return {} + + def _get_field(self, key, default=u''): + return self.cache_result(key) if self.cached(key) else self._get_from_api().get(key, default) @property def unique(self): @@ -210,6 +235,13 @@ class Service(object): @unique.setter def unique(self, value): self._unique = value + + @property + def quote_word(self): + return urllib2.quote( + self.word.encode('utf-8') if isinstance(self.word, unicode) + else self.word + ) @property def support(self): @@ -269,8 +301,10 @@ class WebService(Service): return getattr(self, '__register_label__', self.unique) def get_response(self, url, data=None, headers=None, timeout=10): - default_headers = {'User-Agent': 'Anki WordQuery', - 'Accept-Encoding': 'gzip'} + default_headers = { + 'User-Agent': 'Mozilla/5.0', + 'Accept-Encoding': 'gzip' + } if headers: default_headers.update(headers) @@ -282,7 +316,7 @@ class WebService(Service): data = zlib.decompress(data, 16 + zlib.MAX_WBITS) return data except: - return '' + return u'' @classmethod def download(cls, url, filename, timeout=15): diff --git a/addons/fastwq/service/dict/LDOCE6.py b/addons/fastwq/service/dict/LDOCE6.py index 618be01..d3840b4 100644 --- a/addons/fastwq/service/dict/LDOCE6.py +++ b/addons/fastwq/service/dict/LDOCE6.py @@ -1,7 +1,7 @@ #-*- coding:utf-8 -*- import re import FastWQ -from ..base import MdxService, export, register, with_styles, parse_html +from ..base import * PATH = FastWQ.LDOCE6_PATH @@ -33,24 +33,11 @@ class Ldoce6(MdxService): def _fld_voice(self, html, voice): """获取发音字段""" - from hashlib import sha1 for regexp in LANG_TO_REGEXPS[voice]: match = regexp.search(html) if match: val = '/' + match.group(1) - hex_digest = sha1( - val.encode('utf-8') if isinstance(val, unicode) - else val - ).hexdigest().lower() - - assert len(hex_digest) == 40, "unexpected output from hash library" - name = '.'.join([ - '-'.join([ - 'mdx', self.unique.lower(), hex_digest[:8], hex_digest[8:16], - hex_digest[16:24], hex_digest[24:32], hex_digest[32:], - ]), - 'mp3', - ]) + name = get_hex_name('mdx-'+self.unique.lower(), val, 'mp3') name = self.save_file(val, name) if name: return self.get_anki_label(name, 'audio') diff --git a/addons/fastwq/service/dict/baicizhan.py b/addons/fastwq/service/dict/baicizhan.py index d54a913..82f05c9 100644 --- a/addons/fastwq/service/dict/baicizhan.py +++ b/addons/fastwq/service/dict/baicizhan.py @@ -2,7 +2,7 @@ import json import os from collections import defaultdict -from ..base import WebService, export, register +from ..base import * @register([u'百词斩', u'Baicizhan']) @@ -15,8 +15,7 @@ class Baicizhan(WebService): super(Baicizhan, self).__init__() def _get_from_api(self): - word = self.word.replace(' ', '_') - url = u"http://mall.baicizhan.com/ws/search?w={}".format(word) + url = u"http://mall.baicizhan.com/ws/search?w={}".format(self.quote_word) result = { "accent": u"", "img": u"", @@ -33,18 +32,14 @@ class Baicizhan(WebService): except: pass return self.cache_this(result) - - def _get_field(self, key, default=u''): - return self.cache_result(key) if self.cached(key) else self._get_from_api().get(key, default) @export('PRON') def fld_phonetic(self): - word = self.word.replace(' ', '_') - url = u'http://baicizhan.qiniucdn.com/word_audios/{}.mp3'.format(word) + url = u'http://baicizhan.qiniucdn.com/word_audios/{}.mp3'.format(self.quote_word) audio_name = 'bcz_%s.mp3' % self.word + audio_name = get_hex_name(self.unique.lower(), audio_name, 'mp3') if self.bcz_download_mp3: if os.path.exists(audio_name) or self.download(url, audio_name, 5): - # urllib.urlretrieve(url, audio_name) with open(audio_name, 'rb') as f: if f.read().strip() == '{"error":"Document not found"}': res = '' diff --git a/addons/fastwq/service/dict/baidu_chinese.py b/addons/fastwq/service/dict/baidu_chinese.py index 1dca696..1bc4960 100644 --- a/addons/fastwq/service/dict/baidu_chinese.py +++ b/addons/fastwq/service/dict/baidu_chinese.py @@ -1,6 +1,6 @@ #-*- coding:utf-8 -*- -from hashlib import sha1 -from ..base import WebService, export, register, with_styles, parse_html +import os +from ..base import * baidu_download_mp3 = True @@ -11,7 +11,7 @@ class Baidu_Chinese(WebService): super(Baidu_Chinese, self).__init__() def _get_content(self): - url = u"http://dict.baidu.com/s?wd={}#basicmean".format(self.word) + url = u"http://dict.baidu.com/s?wd={}#basicmean".format(self.quote_word) html = self.get_response(url, timeout=10) soup = parse_html(html) result = { @@ -72,26 +72,14 @@ class Baidu_Chinese(WebService): audio_url = self._get_field('audio_url') if baidu_download_mp3 and audio_url: filename = u'_baidu_chinese_{}_.mp3'.format(self.word) - hex_digest = sha1( - self.word.encode('utf-8') if isinstance(self.word, unicode) - else self.word - ).hexdigest().lower() - assert len(hex_digest) == 40, "unexpected output from hash library" - filename = '.'.join([ - '-'.join([ - self.unique.lower( - ), hex_digest[:8], hex_digest[8:16], - hex_digest[16:24], hex_digest[24:32], hex_digest[32:], - ]), - 'mp3', - ]) + filename = get_hex_name(self.unique.lower(), filename, 'mp3') try: - self.net_download( + if os.path.exists(filename) or self.net_download( filename, audio_url, - require=dict(mime='audio/mp3', size=512), - ) - return self.get_anki_label(filename, 'audio') + require=dict(mime='audio/mp3', size=512) + ): + return self.get_anki_label(filename, 'audio') except: pass return '' diff --git a/addons/fastwq/service/dict/bing.py b/addons/fastwq/service/dict/bing.py index bbb7b8e..cae3f47 100644 --- a/addons/fastwq/service/dict/bing.py +++ b/addons/fastwq/service/dict/bing.py @@ -1,6 +1,7 @@ #-*- coding:utf-8 -*- import re -from ..base import WebService, export, register, with_styles, parse_html +import os +from ..base import * bing_download_mp3 = True @@ -10,9 +11,8 @@ class Bing(WebService): def __init__(self): super(Bing, self).__init__() - def _get_content(self): - word = self.word.replace(' ', '_') - data = self.get_response(u"http://cn.bing.com/dict/search?q={}&mkt=zh-cn".format(word)) + def _get_from_api(self): + data = self.get_response(u"http://cn.bing.com/dict/search?q={}&mkt=zh-cn".format(self.quote_word)) soup = parse_html(data) result = { 'pronunciation': {'AmE': '', 'BrE': '', 'AmEmp3': '', 'BrEmp3': ''}, @@ -66,9 +66,6 @@ class Bing(WebService): return self.cache_this(result) - def _get_field(self, key, default=u''): - return self.cache_result(key) if self.cached(key) else self._get_content().get(key, default) - @with_styles(css='.pos{font-weight:bold;margin-right:4px;}', need_wrap_css=True, wrap_class='bing') def _css(self, val): return val @@ -86,8 +83,8 @@ class Bing(WebService): def _fld_mp3(self, fld): audio_url = self._get_field('pronunciation')[fld] if bing_download_mp3 and audio_url: - filename = u'bing_' + u''.join(re.findall(r'\w*\.mp3', audio_url)) - if filename and self.net_download(filename, audio_url): + filename = get_hex_name('bing', audio_url, 'mp3') + if os.path.exists(filename) or self.net_download(filename, audio_url): return self.get_anki_label(filename, 'audio') return '' diff --git a/addons/fastwq/service/dict/bing3tp.py b/addons/fastwq/service/dict/bing3tp.py index 6c544d9..2dc8e00 100644 --- a/addons/fastwq/service/dict/bing3tp.py +++ b/addons/fastwq/service/dict/bing3tp.py @@ -1,7 +1,8 @@ #-*- coding:utf-8 -*- import json import re -from ..base import WebService, export, register, with_styles +import os +from ..base import * bing_download_mp3 = True @@ -11,7 +12,7 @@ class BingXtk(WebService): def __init__(self): super(BingXtk, self).__init__() - def _get_content(self): + def _get_from_api(self): result = { 'pronunciation': {'AmE': '', 'BrE': '', 'AmEmp3': '', 'BrEmp3': ''}, 'def': '', @@ -21,17 +22,13 @@ class BingXtk(WebService): 'Accept-Language': 'en-US,zh-CN;q=0.8,zh;q=0.6,en;q=0.4', 'User-Agent': 'WordQuery Addon (Anki)', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'} - word = self.word.replace(' ', '_').encode('utf-8') - url = u'http://xtk.azurewebsites.net/BingDictService.aspx?Word={}'.format(word) + url = u'http://xtk.azurewebsites.net/BingDictService.aspx?Word={}'.format(self.quote_word) try: result.update(json.loads(self.get_response(url, headers=headers, timeout=10))) except: pass return self.cache_this(result) - def _get_field(self, key, default=u''): - return self.cache_result(key) if self.cached(key) else self._get_content().get(key, default) - @export('AME_PHON') def fld_phonetic_us(self): seg = self._get_field('pronunciation') @@ -45,8 +42,8 @@ class BingXtk(WebService): def _fld_mp3(self, fld): audio_url = self._get_field('pronunciation')[fld] if bing_download_mp3 and audio_url: - filename = u'bing_' + u''.join(re.findall(r'\w*\.mp3', audio_url)) - if filename and self.net_download(filename, audio_url): + filename = get_hex_name('bing', audio_url, 'mp3') + if os.path.exists(filename) or self.net_download(filename, audio_url): return self.get_anki_label(filename, 'audio') return '' diff --git a/addons/fastwq/service/dict/cambridge.py b/addons/fastwq/service/dict/cambridge.py index 3770c78..ca7f53e 100644 --- a/addons/fastwq/service/dict/cambridge.py +++ b/addons/fastwq/service/dict/cambridge.py @@ -1,6 +1,6 @@ #-*- coding:utf-8 -*- -from hashlib import sha1 -from ..base import WebService, export, register, with_styles, parse_html +import os +from ..base import * cambridge_download_mp3 = True @@ -10,9 +10,8 @@ class Cambridge(WebService): def __init__(self): super(Cambridge, self).__init__() - def _get_content(self): - word = self.word.replace(' ', '_') - data = self.get_response(u"https://dictionary.cambridge.org/dictionary/english/{}".format(word)) + def _get_from_api(self): + data = self.get_response(u"https://dictionary.cambridge.org/dictionary/english/{}".format(self.quote_word)) soup = parse_html(data) result = { 'pronunciation': {'AmE': '', 'BrE': '', 'AmEmp3': '', 'BrEmp3': ''}, @@ -29,7 +28,8 @@ class Cambridge(WebService): tags = header.find_all('span', class_='pron-info') if tags: for tag in tags: - reg = str(tag.find('span', class_='region').get_text()).decode('utf-8') + r = tag.find('span', class_='region') + reg = str(r.get_text()).decode('utf-8') if r else u'' pn = 'AmE' if reg=='us' else 'BrE' p = tag.find('span', class_='pron') result['pronunciation'][pn] = str(p.get_text()).decode('utf-8') if p else u'' @@ -58,9 +58,6 @@ class Cambridge(WebService): result['def'] = u'' return self.cache_this(result) - - def _get_field(self, key, default=u''): - return self.cache_result(key) if self.cached(key) else self._get_content().get(key, default) @with_styles(need_wrap_css=True, cssfile='_cambridge.css') def _css(self, val): @@ -79,22 +76,8 @@ class Cambridge(WebService): def _fld_mp3(self, fld): audio_url = self._get_field('pronunciation')[fld] if cambridge_download_mp3 and audio_url: - filename = u'_cambridge_{}_.mp3'.format(self.word) - hex_digest = sha1( - self.word.encode('utf-8') if isinstance(self.word, unicode) - else self.word - ).hexdigest().lower() - assert len(hex_digest) == 40, "unexpected output from hash library" - filename = '.'.join([ - '-'.join([ - self.unique.lower( - ), hex_digest[:8], hex_digest[8:16], - hex_digest[16:24], hex_digest[24:32], hex_digest[32:], - ]), - 'mp3', - ]) - - if filename and self.net_download(filename, audio_url): + filename = get_hex_name(self.unique.lower(), audio_url, 'mp3') + if os.path.exists(filename) or self.net_download(filename, audio_url): return self.get_anki_label(filename, 'audio') return '' diff --git a/addons/fastwq/service/dict/esdict.py b/addons/fastwq/service/dict/esdict.py index ab7ef11..95c7c72 100644 --- a/addons/fastwq/service/dict/esdict.py +++ b/addons/fastwq/service/dict/esdict.py @@ -1,32 +1,25 @@ #-*- coding:utf-8 -*- + import base64 import re -import urllib import urllib2 - -from aqt.utils import showInfo -from BeautifulSoup import BeautifulSoup -from ..base import WebService, export, register, with_styles - -# Anki buit-in BeautifulSoup is bs3 not bs4 - +import os +from ..base import * css = '' - @register(u'西语助手') class Esdict(WebService): def __init__(self): super(Esdict, self).__init__() - def _get_content(self): - url = 'https://www.esdict.cn/mdicts/es/{word}'.format( - word=urllib.quote(self.word.encode('utf-8'))) + def _get_from_api(self): + url = 'https://www.esdict.cn/mdicts/es/{}'.format(self.quote_word) try: result = {} - html = urllib2.urlopen(url, timeout=5).read() - soup = BeautifulSoup(html) + html = self.get_response(url, timeout=5) + soup = parse_html(html) def _get_from_element(dict, key, soup, tag, id=None, class_=None): baseURL = 'https://www.esdict.cn/' @@ -62,21 +55,15 @@ class Esdict(WebService): except Exception as e: return {} - def _get_field(self, key, default=u''): - return self.cache_result(key) if self.cached(key) else self._get_content().get(key, default) - @export(u'真人发音') def fld_sound(self): - # base64.b64encode('bonjour') == 'Ym9uam91cg==' - # https://api.frdic.com/api/v2/speech/speakweb?langid=fr&txt=QYNYm9uam91cg%3d%3d url = 'https://api.frdic.com/api/v2/speech/speakweb?langid=es&txt=QYN{word}'.format( - word=urllib.quote(base64.b64encode(self.word.encode('utf-8')))) - audio_name = u'esdict_{word}.mp3'.format(word=self.word) - try: - urllib.urlretrieve(url, audio_name) - return self.get_anki_label(audio_name, 'audio') - except Exception as e: - return '' + word=urllib2.quote(base64.b64encode(self.word.encode('utf-8'))) + ) + filename = get_hex_name(self.unique.lower(), url, 'mp3') + if os.path.exists(filename) or self.net_download(filename, url): + return self.get_anki_label(filename, 'audio') + return '' @export(u'音标') def fld_phonetic(self): diff --git a/addons/fastwq/service/dict/frdic.py b/addons/fastwq/service/dict/frdic.py index a36eb45..0621904 100644 --- a/addons/fastwq/service/dict/frdic.py +++ b/addons/fastwq/service/dict/frdic.py @@ -1,36 +1,28 @@ #-*- coding:utf-8 -*- import base64 import re -import urllib import urllib2 - -from aqt.utils import showInfo -from BeautifulSoup import BeautifulSoup - -from ..base import WebService, export, with_styles, register - -# Anki buit-in BeautifulSoup is bs3 not bs4 +import os +from ..base import * css = '' - @register(u'法语助手') class Frdic(WebService): def __init__(self): super(Frdic, self).__init__() - def _get_content(self): - url = 'https://frdic.com/mdicts/fr/{word}'.format( - word=urllib.quote(self.word.encode('utf-8'))) + def _get_from_api(self): + url = 'http://www.frdic.com/dicts/fr/{}'.format(self.quote_word) try: result = {} - html = urllib2.urlopen(url, timeout=5).read() - soup = BeautifulSoup(html) + html = self.get_response(url, timeout=5) + soup = parse_html(html) def _get_from_element(dict, key, soup, tag, id=None, class_=None): - baseURL = 'https://frdic.com/' + baseURL = 'http://www.frdic.com/' # element = soup.find(tag, id=id, class_=class_) # bs4 if id: element = soup.find(tag, {"id": id}) @@ -49,9 +41,9 @@ class Frdic(WebService): result, 'phonitic', soup, 'span', class_='Phonitic') # '
' result = _get_from_element( - result, 'fccf', soup, 'div', id='FCChild') # 法汉-汉法词典 + result, 'fccf', soup, 'div', id='ExpFCChild') # 法汉-汉法词典 result = _get_from_element( - result, 'example', soup, 'div', id='LJChild') # 法语例句库 + result, 'example', soup, 'div', id='TingLijuChild') # 法语例句库 result = _get_from_element( result, 'syn', soup, 'div', id='SYNChild') # 近义、反义、派生词典 result = _get_from_element( @@ -65,19 +57,14 @@ class Frdic(WebService): @export(u'真人发音') def fld_sound(self): - # base64.b64encode('bonjour') == 'Ym9uam91cg==' - # https://api.frdic.com/api/v2/speech/speakweb?langid=fr&txt=QYNYm9uam91cg%3d%3d url = 'https://api.frdic.com/api/v2/speech/speakweb?langid=fr&txt=QYN{word}'.format( - word=urllib.quote(base64.b64encode(self.word.encode('utf-8')))) - audio_name = u'frdic_{word}.mp3'.format(word=self.word) - try: - urllib.urlretrieve(url, audio_name) - return self.get_anki_label(audio_name, 'audio') - except Exception as e: - return '' - - def _get_field(self, key, default=u''): - return self.cache_result(key) if self.cached(key) else self._get_content().get(key, default) + word=urllib2.quote(base64.b64encode(self.word.encode('utf-8'))) + ) + filename = u'frdic_{word}.mp3'.format(word=self.word) + filename = get_hex_name(self.unique.lower(), filename, 'mp3') + if os.path.exists(filename) or self.net_download(filename, url): + return self.get_anki_label(filename, 'audio') + return '' @export(u'音标') def fld_phonetic(self): diff --git a/addons/fastwq/service/dict/iciba.py b/addons/fastwq/service/dict/iciba.py index 2a279fc..80ea1d9 100644 --- a/addons/fastwq/service/dict/iciba.py +++ b/addons/fastwq/service/dict/iciba.py @@ -1,12 +1,11 @@ #-*- coding:utf-8 -*- import os import re -import urllib2 import json from collections import defaultdict from aqt.utils import showInfo, showText -from ..base import WebService, export, with_styles, register +from ..base import * from ...utils import ignore_exception iciba_download_mp3 = True @@ -18,16 +17,19 @@ class ICIBA(WebService): def __init__(self): super(ICIBA, self).__init__() - def _get_content(self): + def _get_from_api(self): resp = defaultdict(str) headers = { 'Accept-Language': 'en-US,zh-CN;q=0.8,zh;q=0.6,en;q=0.4', 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36', 'Accept': 'text/javascript, application/javascript, application/ecmascript, application/x-ecmascript, */*; q=0.01'} # try: - request = urllib2.Request( - 'http://www.iciba.com/index.php?a=getWordMean&c=search&word=' + self.word.encode('utf-8'), headers=headers) - resp = json.loads(urllib2.urlopen(request).read()) + html = self.get_response( + 'http://www.iciba.com/index.php?a=getWordMean&c=search&word=' + self.quote_word, + headers=headers, + timeout=5 + ) + resp = json.loads(html) # self.cache_this(resp['baesInfo']['symbols'][0]) # self.cache_this(resp['sentence']) # showText(str(self.cache[self.word])) @@ -36,9 +38,6 @@ class ICIBA(WebService): # except Exception as e: # return resp - def _get_field(self, key, default=u''): - return self.cache_result(key) if self.cached(key) else self._get_content().get(key, default) - @ignore_exception @export(u'美式音标') def fld_phonetic_us(self): @@ -130,12 +129,11 @@ class ICIBA(WebService): @ignore_exception @export(u'英文例句') def fld_st(self): - sentences = '' + sentences = '' segs = self._get_field('sentence') for i, seg in enumerate(segs): sentences = u"""{}""".format(seg['Network_en']) - break - + break return sentences @ignore_exception diff --git a/addons/fastwq/service/dict/longman.py b/addons/fastwq/service/dict/longman.py index 1d81648..90aca67 100644 --- a/addons/fastwq/service/dict/longman.py +++ b/addons/fastwq/service/dict/longman.py @@ -1,7 +1,7 @@ #-*- coding:utf-8 -*- -from hashlib import sha1 -from ..base import WebService, export, register, with_styles, parse_html +import os +from ..base import * from ...libs.bs4 import Tag @@ -14,11 +14,8 @@ class Longman(WebService): def __init__(self): super(Longman, self).__init__() - def _get_field(self, key, default=u''): - return self.cache_result(key) if self.cached(key) else self._get_content().get(key, default) - - def _get_content(self): - url = 'https://www.ldoceonline.com/dictionary/{}'.format(self.word) + def _get_from_api(self): + url = 'https://www.ldoceonline.com/dictionary/{}'.format(self.quote_word) data = self.get_response(url) soup = parse_html(data) # Top Container @@ -116,21 +113,8 @@ class Longman(WebService): def _fld_mp3(self, fld): audio_url = self._get_field(fld) if longman_download_mp3 and audio_url: - filename = u'_longman_{}_.mp3'.format(self.word) - hex_digest = sha1( - self.word.encode('utf-8') if isinstance(self.word, unicode) - else self.word - ).hexdigest().lower() - assert len(hex_digest) == 40, "unexpected output from hash library" - filename = '.'.join([ - '-'.join([ - self.unique.lower( - ), hex_digest[:8], hex_digest[8:16], - hex_digest[16:24], hex_digest[24:32], hex_digest[32:], - ]), - 'mp3', - ]) - if self.net_download(filename, audio_url): + filename = get_hex_name(self.unique.lower(), audio_url, 'mp3') + if os.path.exists(filename) or self.net_download(filename, audio_url): return self.get_anki_label(filename, 'audio') return '' diff --git a/addons/fastwq/service/dict/minidict.py b/addons/fastwq/service/dict/minidict.py index e42208c..241a831 100644 --- a/addons/fastwq/service/dict/minidict.py +++ b/addons/fastwq/service/dict/minidict.py @@ -1,6 +1,6 @@ #-*- coding:utf-8 -*- -from ..base import WebService, export, register, with_styles, parse_html +from ..base import * @register(u'海词迷你词典') @@ -9,8 +9,8 @@ class MiniDict(WebService): def __init__(self): super(MiniDict, self).__init__() - def _get_content(self): - data = self.get_response(u"http://apii.dict.cn/mini.php?q={}".format(self.word)) + def _get_from_api(self): + data = self.get_response(u"http://apii.dict.cn/mini.php?q={}".format(self.quote_word)) soup = parse_html(data) result = { 'expressions': u'', @@ -45,9 +45,6 @@ class MiniDict(WebService): return self.cache_this(result) - def _get_field(self, key, default=u''): - return self.cache_result(key) if self.cached(key) else self._get_content().get(key, default) - @export(u'音标') def fld_phonetic(self): return self._get_field('phonetic') diff --git a/addons/fastwq/service/dict/oxford.py b/addons/fastwq/service/dict/oxford.py index 7fc3a69..2602c7e 100644 --- a/addons/fastwq/service/dict/oxford.py +++ b/addons/fastwq/service/dict/oxford.py @@ -1,12 +1,11 @@ #-*- coding:utf-8 -*- + import urllib2 -from urllib2 import quote import json -from aqt.utils import showInfo -from ..base import WebService, export, register, with_styles +from ..base import * -@register("Oxford") +@register(u"Oxford") class Oxford(WebService): def __init__(self): @@ -19,7 +18,7 @@ class Oxford(WebService): app_key = "bb36fd6a1259e5baf8df6110a2f7fc8f" headers = {"app_id": app_id, "app_key": app_key} - word_id = quote(word.lower().replace(" ", "_")) + word_id = urllib2.quote(word.lower().replace(" ", "_")) url = baseurl + "/entries/" + lang + "/" + word_id url = urllib2.Request(url, headers=headers) response = json.loads(urllib2.urlopen(url).read()) diff --git a/addons/fastwq/service/dict/oxford_learning.py b/addons/fastwq/service/dict/oxford_learning.py index 9c82798..561a721 100644 --- a/addons/fastwq/service/dict/oxford_learning.py +++ b/addons/fastwq/service/dict/oxford_learning.py @@ -1,14 +1,9 @@ # coding=utf-8 #from warnings import filterwarnings + +from ..base import * from ...libs.bs4 import Tag -from ..base import WebService, export, register, with_styles, parse_html -#filterwarnings('ignore') - -import sys - -#reload(sys) -#sys.setdefaultencoding('utf8') @register([u'牛津学习词典', u'Oxford Learner']) class OxfordLearning(WebService): @@ -36,7 +31,7 @@ class OxfordLearning(WebService): def _get_single_dict(self, single_dict): if not (self.cached(single_dict) and self.cache_result(single_dict)): - web_word = self.query(self.word) + web_word = self.query(self.quote_word) if web_word: self.cache_this( { diff --git a/addons/fastwq/service/dict/remotemdx.py b/addons/fastwq/service/dict/remotemdx.py index 24b8141..36db883 100644 --- a/addons/fastwq/service/dict/remotemdx.py +++ b/addons/fastwq/service/dict/remotemdx.py @@ -21,12 +21,10 @@ import os # import ntpath import re import urllib -import urllib2 -import urlparse from collections import defaultdict from aqt.utils import showInfo, showText -from ..base import QueryResult, WebService, export, register, with_styles +from ..base import * @register('MDX_SERVER') @@ -41,8 +39,8 @@ class RemoteMdx(WebService): self.url = dict_path + \ '/' if not dict_path.endswith('/') else dict_path try: - req = urllib2.urlopen(self.url + word) - result, js = self.adapt_to_anki(req.read()) + html = self.get_response(self.url + word) + result, js = self.adapt_to_anki(html) return QueryResult(result=result, js=js) except: return QueryResult.default() @@ -54,12 +52,12 @@ class RemoteMdx(WebService): for each in diff: basename = os.path.basename(each.replace('\\', os.path.sep)) saved_basename = '_' + basename - abs_url = urlparse.urljoin(self.url, each) + abs_url = urllib.parse.urljoin(self.url, each) if basename.endswith('.css') or basename.endswith('.js'): styles.append(saved_basename) if not os.path.exists(saved_basename): try: - urllib.urlretrieve(abs_url, saved_basename) + self.download(abs_url, saved_basename) except: errors.append(each) return errors, styles diff --git a/addons/fastwq/service/dict/yahoo.py b/addons/fastwq/service/dict/yahoo.py index 7ea1f47..99c159d 100644 --- a/addons/fastwq/service/dict/yahoo.py +++ b/addons/fastwq/service/dict/yahoo.py @@ -1,7 +1,6 @@ # -*- coding:utf-8 -*- import os -from hashlib import sha1 -from ..base import WebService, export, register, with_styles, parse_html +from ..base import * yahoo_download_mp3 = True @@ -12,9 +11,9 @@ class Yahoo_Dict(WebService): def __init__(self): super(Yahoo_Dict, self).__init__() - def _get_content(self): + def _get_from_api(self): url = u"https://tw.dictionary.search.yahoo.com/search?p={}".format( - self.word) + self.quote_word) html = self.get_response(url, timeout=10) soup = parse_html(html) result = { @@ -51,9 +50,6 @@ class Yahoo_Dict(WebService): return self.cache_this(result) - def _get_field(self, key, default=u''): - return self.cache_result(key) if self.cached(key) else self._get_content().get(key, default) - @with_styles(need_wrap_css=True, cssfile='_yahoo.css') def _css(self, val): return val @@ -66,20 +62,7 @@ class Yahoo_Dict(WebService): def fld_pron(self): audio_url = self._get_field('audio_url') if yahoo_download_mp3 and audio_url: - filename = u'_yahoo_dict_{}_.mp3'.format(self.word) - hex_digest = sha1( - self.word.encode('utf-8') if isinstance(self.word, unicode) - else self.word - ).hexdigest().lower() - assert len(hex_digest) == 40, "unexpected output from hash library" - filename = '.'.join([ - '-'.join([ - self.unique.lower( - ), hex_digest[:8], hex_digest[8:16], - hex_digest[16:24], hex_digest[24:32], hex_digest[32:], - ]), - 'mp3', - ]) + filename = get_hex_name(self.unique.lower(), audio_url, 'mp3') if os.path.exists(filename) or self.download(audio_url, filename, 5): return self.get_anki_label(filename, 'audio') diff --git a/addons/fastwq/service/dict/youdao.py b/addons/fastwq/service/dict/youdao.py index 92f1d18..15fab7e 100644 --- a/addons/fastwq/service/dict/youdao.py +++ b/addons/fastwq/service/dict/youdao.py @@ -1,10 +1,8 @@ #-*- coding:utf-8 -*- -import re -import urllib2 -import xml.etree.ElementTree -from aqt.utils import showInfo -from ..base import WebService, export, register, with_styles +import os +import xml.etree.ElementTree +from ..base import * js = ''' var initVoice = function () { @@ -38,7 +36,7 @@ class Youdao(WebService): '&doctype=xml&xmlVersion=3.2' '&dogVersion=1.0&vendor=unknown' '&appVer=3.1.17.4208' - '&le={0}&q={1}').format(lang, self.word) + '&le={0}&q={1}').format(lang, self.quote_word) result ={ 'phonetic': '', 'explains':'', @@ -66,9 +64,6 @@ class Youdao(WebService): pass return self.cache_this(result) - def _get_field(self, key, default=u''): - return self.cache_result(key) if self.cached(key) else self._get_from_api().get(key, default) - @export(u'音标') def fld_phonetic(self): return self._get_field('phonetic') @@ -80,12 +75,12 @@ class Youdao(WebService): @with_styles(cssfile='_youdao.css', js=js, need_wrap_css=True, wrap_class='youdao') def _get_singledict(self, single_dict, lang='eng'): url = u"http://m.youdao.com/singledict?q={0}&dict={1}&le={2}&more=false".format( - self.word, + self.quote_word, single_dict, lang ) try: - result = urllib2.urlopen(url, timeout=5).read() + result = self.get_response(url, timeout=5) return (u'
' '
{1}
' '
' @@ -100,19 +95,19 @@ class Youdao(WebService): @export(u'英式发音') def fld_british_audio(self): - audio_url = u'http://dict.youdao.com/dictvoice?audio={}&type=1'.format(self.word) + audio_url = u'http://dict.youdao.com/dictvoice?audio={}&type=1'.format(self.quote_word) if youdao_download_mp3: - filename = u'_youdao_{}_uk.mp3'.format(self.word) - if self.download(audio_url, filename): + filename = get_hex_name(self.unique.lower(), audio_url, 'mp3') + if os.path.exists(filename) or self.download(audio_url, filename): return self.get_anki_label(filename, 'audio') return audio_url @export(u'美式发音') def fld_american_audio(self): - audio_url = u'http://dict.youdao.com/dictvoice?audio={}&type=2'.format(self.word) + audio_url = u'http://dict.youdao.com/dictvoice?audio={}&type=2'.format(self.quote_word) if youdao_download_mp3: - filename = u'_youdao_{}_us.mp3'.format(self.word) - if self.download(audio_url, filename): + filename = get_hex_name(self.unique.lower(), audio_url, 'mp3') + if os.path.exists(filename) or self.download(audio_url, filename): return self.get_anki_label(filename, 'audio') return audio_url diff --git a/addons/fastwq/service/dict/youdaofr.py b/addons/fastwq/service/dict/youdaofr.py index 9751f70..599471e 100644 --- a/addons/fastwq/service/dict/youdaofr.py +++ b/addons/fastwq/service/dict/youdaofr.py @@ -1,10 +1,7 @@ #-*- coding:utf-8 -*- -import re -import urllib2 -import xml.etree.ElementTree -from aqt.utils import showInfo -from ..base import WebService, export, register, with_styles +import xml.etree.ElementTree +from ..base import * @register([u'有道词典-法语', u'Youdao-French']) @@ -19,7 +16,7 @@ class Youdaofr(WebService): '&doctype=xml&xmlVersion=3.2' '&dogVersion=1.0&vendor=unknown' '&appVer=3.1.17.4208' - '&le={0}&q={1}').format(lang, self.word) + '&le={0}&q={1}').format(lang, self.quote_word) result ={ 'phonetic': '', 'explains':'', @@ -44,10 +41,10 @@ class Youdaofr(WebService): @with_styles(cssfile='_youdao.css', need_wrap_css=True, wrap_class='youdao') def _get_singledict(self, single_dict, lang='fr'): url = u"http://m.youdao.com/singledict?q={0}&dict={1}&le={2}&more=false".format( - self.word, single_dict, lang + self.quote_word, single_dict, lang ) try: - html = urllib2.urlopen(url, timeout=5).read() + html = self.get_response(url, timeout=5) return (u'
' '
{3}
' '
' @@ -57,7 +54,7 @@ class Youdaofr(WebService): single_dict, single_dict, single_dict, - html + html.decode('utf-8') ) except: return '' diff --git a/addons/fastwq/service/dict/youdaoko.py b/addons/fastwq/service/dict/youdaoko.py index 32bb79e..3d6a033 100644 --- a/addons/fastwq/service/dict/youdaoko.py +++ b/addons/fastwq/service/dict/youdaoko.py @@ -1,10 +1,7 @@ #-*- coding:utf-8 -*- -import re -import urllib2 -import xml.etree.ElementTree -from aqt.utils import showInfo -from ..base import WebService, export, register, with_styles +import xml.etree.ElementTree +from ..base import * @register([u'有道词典-韩语', u'Youdao-Korean']) @@ -19,10 +16,10 @@ class Youdaoko(WebService): '&doctype=xml&xmlVersion=3.2' '&dogVersion=1.0&vendor=unknown' '&appVer=3.1.17.4208' - '&le={0}&q={1}').format(lang, self.word) + '&le={0}&q={1}').format(lang, self.quote_word) result ={ - 'phonetic': '', - 'explains':'', + 'phonetic': u'', + 'explains': u'', } try: html = self.get_response(url, timeout=5) @@ -44,10 +41,10 @@ class Youdaoko(WebService): @with_styles(cssfile='_youdao.css', need_wrap_css=True, wrap_class='youdao') def _get_singledict(self, single_dict, lang='ko'): url = u"http://m.youdao.com/singledict?q={0}&dict={1}&le={2}&more=false".format( - self.word, single_dict, lang + self.quote_word, single_dict, lang ) try: - html = urllib2.urlopen(url, timeout=5).read() + html = self.get_response(url, timeout=5) return (u'
' '
{3}
' '
' diff --git a/addons21/fastwq/service/base.py b/addons21/fastwq/service/base.py index 434f992..60002af 100644 --- a/addons21/fastwq/service/base.py +++ b/addons21/fastwq/service/base.py @@ -305,8 +305,10 @@ class WebService(Service): return getattr(self, '__register_label__', self.unique) def get_response(self, url, data=None, headers=None, timeout=10): - default_headers = {'User-Agent': 'Anki WordQuery', - 'Accept-Encoding': 'gzip'} + default_headers = { + 'User-Agent': 'Mozilla/5.0', + 'Accept-Encoding': 'gzip' + } if headers: default_headers.update(headers) diff --git a/addons21/fastwq/service/dict/bing.py b/addons21/fastwq/service/dict/bing.py index 9e969e5..170f3e7 100644 --- a/addons21/fastwq/service/dict/bing.py +++ b/addons21/fastwq/service/dict/bing.py @@ -83,8 +83,7 @@ class Bing(WebService): def _fld_mp3(self, fld): audio_url = self._get_field('pronunciation')[fld] if bing_download_mp3 and audio_url: - filename = u'bing_' + u''.join(re.findall(r'\w*\.mp3', audio_url)) - filename = get_hex_name('bing', filename, 'mp3') + filename = get_hex_name('bing', audio_url, 'mp3') if os.path.exists(filename) or self.net_download(filename, audio_url): return self.get_anki_label(filename, 'audio') return '' diff --git a/addons21/fastwq/service/dict/bing3tp.py b/addons21/fastwq/service/dict/bing3tp.py index ce2c94c..2dc8e00 100644 --- a/addons21/fastwq/service/dict/bing3tp.py +++ b/addons21/fastwq/service/dict/bing3tp.py @@ -42,8 +42,7 @@ class BingXtk(WebService): def _fld_mp3(self, fld): audio_url = self._get_field('pronunciation')[fld] if bing_download_mp3 and audio_url: - filename = u'bing_' + u''.join(re.findall(r'\w*\.mp3', audio_url)) - filename = get_hex_name('bing', filename, 'mp3') + filename = get_hex_name('bing', audio_url, 'mp3') if os.path.exists(filename) or self.net_download(filename, audio_url): return self.get_anki_label(filename, 'audio') return '' diff --git a/addons21/fastwq/service/dict/cambridge.py b/addons21/fastwq/service/dict/cambridge.py index dcfdd9d..c3ca32b 100644 --- a/addons21/fastwq/service/dict/cambridge.py +++ b/addons21/fastwq/service/dict/cambridge.py @@ -76,8 +76,7 @@ class Cambridge(WebService): def _fld_mp3(self, fld): audio_url = self._get_field('pronunciation')[fld] if cambridge_download_mp3 and audio_url: - filename = u'_cambridge_{}_.mp3'.format(self.word) - filename = get_hex_name(self.unique.lower(), filename, 'mp3') + filename = get_hex_name(self.unique.lower(), audio_url, 'mp3') if os.path.exists(filename) or self.net_download(filename, audio_url): return self.get_anki_label(filename, 'audio') return '' diff --git a/addons21/fastwq/service/dict/esdict.py b/addons21/fastwq/service/dict/esdict.py index 56a8680..68dde69 100644 --- a/addons21/fastwq/service/dict/esdict.py +++ b/addons21/fastwq/service/dict/esdict.py @@ -60,8 +60,7 @@ class Esdict(WebService): url = 'https://api.frdic.com/api/v2/speech/speakweb?langid=es&txt=QYN{word}'.format( word=urllib2.quote(base64.b64encode(self.word.encode('utf-8'))) ) - filename = u'esdict_{word}.mp3'.format(word=self.word) - filename = get_hex_name(self.unique.lower(), filename, 'mp3') + filename = get_hex_name(self.unique.lower(), url, 'mp3') if os.path.exists(filename) or self.net_download(filename, url): return self.get_anki_label(filename, 'audio') return '' diff --git a/addons21/fastwq/service/dict/longman.py b/addons21/fastwq/service/dict/longman.py index 94ba5db..9ab5e61 100644 --- a/addons21/fastwq/service/dict/longman.py +++ b/addons21/fastwq/service/dict/longman.py @@ -113,8 +113,7 @@ class Longman(WebService): def _fld_mp3(self, fld): audio_url = self._get_field(fld) if longman_download_mp3 and audio_url: - filename = u'_longman_{}_.mp3'.format(self.word) - filename = get_hex_name(self.unique.lower(), filename, 'mp3') + filename = get_hex_name(self.unique.lower(), audio_url, 'mp3') if os.path.exists(filename) or self.net_download(filename, audio_url): return self.get_anki_label(filename, 'audio') return '' diff --git a/addons21/fastwq/service/dict/yahoo.py b/addons21/fastwq/service/dict/yahoo.py index dce69e3..0b51b73 100644 --- a/addons21/fastwq/service/dict/yahoo.py +++ b/addons21/fastwq/service/dict/yahoo.py @@ -11,7 +11,7 @@ class Yahoo_Dict(WebService): def __init__(self): super(Yahoo_Dict, self).__init__() - def _get_content(self): + def _get_from_api(self): url = u"https://tw.dictionary.search.yahoo.com/search?p={}".format( self.quote_word) html = self.get_response(url, timeout=10) @@ -50,9 +50,6 @@ class Yahoo_Dict(WebService): return self.cache_this(result) - def _get_field(self, key, default=u''): - return self.cache_result(key) if self.cached(key) else self._get_content().get(key, default) - @with_styles(need_wrap_css=True, cssfile='_yahoo.css') def _css(self, val): return val @@ -65,8 +62,7 @@ class Yahoo_Dict(WebService): def fld_pron(self): audio_url = self._get_field('audio_url') if yahoo_download_mp3 and audio_url: - filename = u'_yahoo_dict_{}_.mp3'.format(self.word) - filename = get_hex_name(self.unique.lower(), filename, 'mp3') + filename = get_hex_name(self.unique.lower(), audio_url, 'mp3') if os.path.exists(filename) or self.download(audio_url, filename, 5): return self.get_anki_label(filename, 'audio') diff --git a/addons21/fastwq/service/dict/youdao.py b/addons21/fastwq/service/dict/youdao.py index b5a001e..15fab7e 100644 --- a/addons21/fastwq/service/dict/youdao.py +++ b/addons21/fastwq/service/dict/youdao.py @@ -97,7 +97,7 @@ class Youdao(WebService): def fld_british_audio(self): audio_url = u'http://dict.youdao.com/dictvoice?audio={}&type=1'.format(self.quote_word) if youdao_download_mp3: - filename = u'_youdao_{}_uk.mp3'.format(self.word) + filename = get_hex_name(self.unique.lower(), audio_url, 'mp3') if os.path.exists(filename) or self.download(audio_url, filename): return self.get_anki_label(filename, 'audio') return audio_url @@ -106,7 +106,7 @@ class Youdao(WebService): def fld_american_audio(self): audio_url = u'http://dict.youdao.com/dictvoice?audio={}&type=2'.format(self.quote_word) if youdao_download_mp3: - filename = u'_youdao_{}_us.mp3'.format(self.word) + filename = get_hex_name(self.unique.lower(), audio_url, 'mp3') if os.path.exists(filename) or self.download(audio_url, filename): return self.get_anki_label(filename, 'audio') return audio_url