From 4b2ddcd69af584cfcfa7864df3b6dca2cd08a183 Mon Sep 17 00:00:00 2001 From: "St.Huang" Date: Mon, 9 Jul 2018 14:12:54 +0800 Subject: [PATCH] improve baicizhan query speed. --- src/fastwq/service/baicizhan.py | 29 ++++++++++++------ src/fastwq/service/base.py | 52 +++++++++++++++++++-------------- 2 files changed, 50 insertions(+), 31 deletions(-) diff --git a/src/fastwq/service/baicizhan.py b/src/fastwq/service/baicizhan.py index 556d26e..5178deb 100644 --- a/src/fastwq/service/baicizhan.py +++ b/src/fastwq/service/baicizhan.py @@ -17,11 +17,25 @@ class Baicizhan(WebService): def _get_from_api(self): word = self.word.replace(' ', '_') url = u"http://mall.baicizhan.com/ws/search?w={}".format(word) + result = { + "accent": u"", + "img": u"", + "mean_cn": u"", + "st": u"", + "sttr": u"", + "tv": u"", + "word": u"", + "df": u'', + } try: html = self.get_response(url, timeout=5)#urllib2.urlopen(url, timeout=5).read() - return self.cache_this(json.loads(html)) + result.update(json.loads(html)) except: - return defaultdict(str) + pass + return self.cache_this(result) + + def _get_field(self, key, default=u''): + return self.cache_result(key) if self.cached(key) else self._get_from_api().get(key, default) @export('PRON', 0) def fld_phonetic(self): @@ -29,7 +43,7 @@ class Baicizhan(WebService): url = u'http://baicizhan.qiniucdn.com/word_audios/{}.mp3'.format(word) audio_name = 'bcz_%s.mp3' % self.word if self.bcz_download_mp3: - if self.download(url, audio_name): + if os.path.exists(audio_name) or self.download(url, audio_name, 5): # urllib.urlretrieve(url, audio_name) with open(audio_name, 'rb') as f: if f.read().strip() == '{"error":"Document not found"}': @@ -44,11 +58,8 @@ class Baicizhan(WebService): else: return url - def _get_field(self, key, default=u''): - return self.cache_result(key) if self.cached(key) else self._get_from_api().get(key, default) - @export('PHON', 1) - def fld_explains(self): + def fld_phon(self): return self._get_field('accent') @export('IMAGE', 2) @@ -56,7 +67,7 @@ class Baicizhan(WebService): url = self._get_field('img') if url and self.bcz_download_img: filename = url[url.rindex('/') + 1:] - if self.download(url, filename): + if os.path.exists(filename) or self.download(url, filename): return self.get_anki_label(filename, 'img') #return self.get_anki_label(url, 'img') return '' @@ -66,7 +77,7 @@ class Baicizhan(WebService): url = self._get_field('df') if url and self.bcz_download_img: filename = url[url.rindex('/') + 1:] - if self.download(url, filename): + if os.path.exists(filename) or self.download(url, filename): return self.get_anki_label(filename, 'img') #return self.get_anki_label(url, 'img') return '' diff --git a/src/fastwq/service/base.py b/src/fastwq/service/base.py index 855c9d4..b195c31 100644 --- a/src/fastwq/service/base.py +++ b/src/fastwq/service/base.py @@ -32,6 +32,7 @@ import zlib import random from collections import defaultdict from functools import wraps +from hashlib import md5 import cookielib from aqt import mw @@ -261,7 +262,9 @@ class WebService(Service): return '' @classmethod - def download(cls, url, filename): + def download(cls, url, filename, timeout=15): + import socket + socket.setdefaulttimeout(timeout) try: return urllib.urlretrieve(url, filename) except Exception as e: @@ -401,21 +404,19 @@ class WebService(Service): response_output.write(payload) -# MdxBuilder instances map -mdx_builders = defaultdict(dict) -mutex_builder = QMutex() - -class DictBuilder(QThread): +class _DictBuildWorker(QThread): """Local Dictionary Builder""" def __init__(self, func): - super(DictBuilder, self).__init__() - self.index = 0 - self.exit = False - self.builder = None - self.func = func + super(_DictBuildWorker, self).__init__() + self._builder = None + self._func = func def run(self): - self.builder = self.func() + self._builder = self._func() + + @property + def builder(self): + return self._builder class LocalService(Service): @@ -428,18 +429,25 @@ class LocalService(Service): self.dict_path = dict_path self.builder = None self.missed_css = set() - - def get_builer(self, key, func): - mutex_builder.lock() - if not mdx_builders.has_key(key) or not mdx_builders[key]: - worker = DictBuilder(func) + + # MdxBuilder instances map + _mdx_builders = defaultdict(dict) + _mutex_builder = QMutex() + + @staticmethod + def _get_builer(key, func=None): + LocalService._mutex_builder.lock() + key = md5(key).hexdigest() + if not func is None: + if not LocalService._mdx_builders.has_key(key) or not LocalService._mdx_builders[key]: + worker = _DictBuildWorker(func) worker.start() while not worker.isFinished(): mw.app.processEvents() worker.wait(100) - mdx_builders[key] = worker.builder - mutex_builder.unlock() - return mdx_builders[key] + LocalService._mdx_builders[key] = worker.builder + LocalService._mutex_builder.unlock() + return LocalService._mdx_builders[key] @property def support(self): @@ -471,7 +479,7 @@ class MdxService(LocalService): self.query_interval = 0.01 self.styles = [] if self.support: - self.builder = self.get_builer(dict_path, service_wrap(MdxBuilder, dict_path)) + self.builder = self._get_builer(dict_path, service_wrap(MdxBuilder, dict_path)) @staticmethod def check(dict_path): @@ -640,7 +648,7 @@ class StardictService(LocalService): super(StardictService, self).__init__(dict_path) self.query_interval = 0.05 if self.support: - self.builder = self.get_builer( + self.builder = self._get_builer( dict_path, service_wrap(StardictBuilder, dict_path, in_memory=False) )