commit 549d9b1d2cbef9cca8ada14a2aee53b67b3a3904 Author: David Snopek Date: Wed Apr 3 14:31:44 2013 +0100 Committed old Anki 1.2 sync server code. It's not very generic. :-/ diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..eef29c1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +*~ +*.pyc + diff --git a/AnkiServer/__init__.py b/AnkiServer/__init__.py new file mode 100644 index 0000000..43c0a7e --- /dev/null +++ b/AnkiServer/__init__.py @@ -0,0 +1,12 @@ + +def server_runner(app, global_conf, **kw): + """ Special version of paste.httpserver.server_runner which shuts down + the AnkiServer.deck.thread_pool on server exit. """ + + from paste.httpserver import server_runner as paste_server_runner + from AnkiServer.deck import thread_pool + try: + paste_server_runner(app, global_conf, **kw) + finally: + thread_pool.shutdown() + diff --git a/AnkiServer/deck.py b/AnkiServer/deck.py new file mode 100644 index 0000000..749c36a --- /dev/null +++ b/AnkiServer/deck.py @@ -0,0 +1,549 @@ + +from webob.dec import wsgify +from webob.exc import * +from webob import Response + +import anki +from anki.facts import Fact +from anki.models import Model, CardModel, FieldModel + +from threading import Thread +from Queue import Queue + +try: + import simplejson as json +except ImportError: + import json + +import os, errno, time, logging + +__all__ = ['DeckThread'] + +def ExternalModel(): + m = Model(u'External') + # we can only guarantee that the Front will be unique because it will + # be based on the headword, language, pos. The Back could be anything! + m.addFieldModel(FieldModel(u'Front', True, True)) + # while I think that Back should be required, I don't really want this to + # fail just because of that!! + m.addFieldModel(FieldModel(u'Back', False, False)) + m.addFieldModel(FieldModel(u'External ID', True, True)) + + front = u'{{{Front}}}' + back = u'{{{Back}}}' + m.addCardModel(CardModel(u'Forward', front, back)) + m.addCardModel(CardModel(u'Reverse', back, front)) + + m.tags = u"External" + return m + +class DeckWrapper(object): + def __init__(self, path): + self.path = os.path.abspath(path) + self._deck = None + + def _create_deck(self): + # mkdir -p the path, because it might not exist + dir = os.path.dirname(self.path) + try: + os.makedirs(dir) + except OSError, exc: + if exc.errno == errno.EEXIST: + pass + else: + raise + + deck = anki.DeckStorage.Deck(self.path) + try: + deck.initUndo() + deck.addModel(ExternalModel()) + deck.save() + except Exception, e: + deck.close() + deck = None + raise e + + return deck + + def open(self): + if self._deck is None: + if os.path.exists(self.path): + self._deck = anki.DeckStorage.Deck(self.path) + else: + self._deck = self._create_deck() + return self._deck + + def close(self): + if self._deck is None: + return + + self._deck.close() + self._deck = None + + # delete the cache for 'External ID' on this deck + if hasattr(self, '_external_field_id'): + delattr(self, '_external_field_id') + + def opened(self): + return self._deck is not None + + @property + def external_field_id(self): + if not hasattr(self, '_external_field_id'): + # find a field model id for a field named "External ID" + deck = self.open() + self._external_field_id = deck.s.scalar("SELECT id FROM fieldModels WHERE name = :name", name=u'External ID') + if self._external_field_id is None: + raise HTTPBadRequest("No field model named 'External ID'") + return self._external_field_id + + def find_fact(self, external_id): + deck = self.open() + return deck.s.scalar(""" + SELECT factId FROM fields WHERE fieldModelId = :fieldModelId AND + value = :externalId""", fieldModelId=self.external_field_id, externalId=external_id) + +class DeckThread(object): + def __init__(self, path): + self.path = os.path.abspath(path) + self.wrapper = DeckWrapper(path) + + self._queue = Queue() + self._thread = None + self._running = False + self.last_timestamp = time.time() + + @property + def running(self): + return self._running + + def qempty(self): + return self._queue.empty() + + def current(self): + from threading import current_thread + return current_thread() == self._thread + + def execute(self, func, args=[], kw={}, waitForReturn=True): + """ Executes a given function on this thread with the *args and **kw. + + If 'waitForReturn' is True, then it will block until the function has + executed and return its return value. If False, it will return None + immediately and the function will be executed sometime later. + """ + + if waitForReturn: + return_queue = Queue() + else: + return_queue = None + + self._queue.put((func, args, kw, return_queue)) + + if return_queue is not None: + ret = return_queue.get(True) + if isinstance(ret, Exception): + raise ret + return ret + + def _run(self): + logging.info('DeckThread[%s]: Starting...', self.path) + + try: + while self._running: + func, args, kw, return_queue = self._queue.get(True) + + logging.info('DeckThread[%s]: Running %s(*%s, **%s)', self.path, func.func_name, repr(args), repr(kw)) + self.last_timestamp = time.time() + + try: + ret = func(*args, **kw) + except Exception, e: + logging.error('DeckThread[%s]: Unable to %s(*%s, **%s): %s', + self.path, func.func_name, repr(args), repr(kw), e, exc_info=True) + # we return the Exception which will be raise'd on the other end + ret = e + + if return_queue is not None: + return_queue.put(ret) + except Exception, e: + logging.error('DeckThread[%s]: Thread crashed! Exception: %s', e, exc_info=True) + finally: + self.wrapper.close() + # clean out old thread object + self._thread = None + # in case we got here via an exception + self._running = False + + logging.info('DeckThread[%s]: Stopped!' % self.path) + + def start(self): + if not self._running: + self._running = True + assert self._thread is None + self._thread = Thread(target=self._run) + self._thread.start() + + def stop(self): + def _stop(): + self._running = False + self.execute(_stop, waitForReturn=False) + + def stop_and_wait(self): + """ Tell the thread to stop and wait for it to happen. """ + self.stop() + if self._thread is not None: + self._thread.join() + +class DeckThreadPool(object): + def __init__(self): + self.threads = {} + + self.monitor_frequency = 15 + self.monitor_inactivity = 90 + + monitor = Thread(target=self._monitor_run) + monitor.daemon = True + monitor.start() + self._monitor_thread = monitor + + # TODO: it would be awesome to have a safe way to stop inactive threads completely! + def _monitor_run(self): + """ Monitors threads for inactivity and closes the deck on them + (leaves the thread itself running -- hopefully waiting peacefully with only a + small memory footprint!) """ + while True: + cur = time.time() + for path, thread in self.threads.items(): + if thread.running and thread.wrapper.opened() and thread.qempty() and cur - thread.last_timestamp >= self.monitor_inactivity: + logging.info('Monitor is closing deck on inactive DeckThread[%s]' % thread.path) + def closeDeck(wrapper): + wrapper.close() + thread.execute(closeDeck, [thread.wrapper], waitForReturn=False) + time.sleep(self.monitor_frequency) + + def start(self, path): + path = os.path.abspath(path) + + try: + thread = self.threads[path] + except KeyError: + thread = self.threads[path] = DeckThread(path) + + thread.start() + + return thread + + def shutdown(self): + for thread in self.threads.values(): + thread.stop() + self.threads = {} + +thread_pool = DeckThreadPool() + +#def defer(*func, **opts): +# def decorator(func): +# def newFunc(*args, **kw): +# (self, thread) = args[0:2] +# if thread.current(): +# ret = func(*args, **kw) +# # don't return 'ret' if this isn't a wait function, to keep the API +# # consistent even when inside the thread itself (hopefully, help +# # avoid weird problems in the future) +# if opts.get('waitForReturn', True): +# return ret +# else: +# return thread.execute(func, args, kw, **opts) +# newFunc.func_name = func.func_name +# return newFunc +# +# if len(func) == 1: +# return decorator(func[0]) +# elif len(func) > 1: +# raise TypeError +# +# return decorator + +def opts(**opts): + def dec(func): + func.opts = opts + return func + return dec + +class DeckAppHandler(object): + def __init__(self, wrapper): + self.wrapper = wrapper + + def _output_fact(self, fact): + res = dict(zip(fact.keys(), fact.values())) + res['id'] = str(fact.id) + return res + + def _output_card(self, card): + return { + 'id': card.id, + 'question': card.question, + 'answer': card.answer, + } + + @opts(waitForReturn=False) + def setup(self): + # will create the deck if it doesn't exist + self.wrapper.open() + + @opts(waitForReturn=False) + def add_fact(self, fields): + fact_id = self.wrapper.find_fact(fields['External ID']) + if fact_id is not None: + fields['id'] = fact_id + self.save_fact(fields) + else: + deck = self.wrapper.open() + fact = deck.newFact() + for key in fact.keys(): + fact[key] = unicode(fields[key]) + + deck.addFact(fact) + deck.save() + + @opts(waitForReturn=False) + def save_fact(self, fact): + deck = self.wrapper.open() + newFact = deck.s.query(Fact).get(int(fact['id'])) + for key in newFact.keys(): + newFact[key] = fact[key] + + newFact.setModified(textChanged=True, deck=deck) + deck.setModified() + deck.save() + + def find_fact(self, external_id): + factId = self.wrapper.find_fact(external_id) + if not factId: + # we need to signal somehow to the calling application that no such + # deck exists, but without it being considered a "bad error". 404 is + # inappropriate that refers to the resource (ie. /find_fact) which is + # here obviously. + return None + + deck = self.wrapper.open() + fact = deck.s.query(Fact).get(factId) + return self._output_fact(fact) + + @opts(waitForReturn=False) + def delete_fact(self, fact_id=None, external_id=None): + if fact_id is None and external_id is not None: + fact_id = self.wrapper.find_fact(external_id) + if fact_id is not None: + deck = self.wrapper.open() + deck.deleteFact(int(fact_id)) + deck.save() + + def resync_facts(self, external_ids): + from anki.facts import fieldsTable + from sqlalchemy.sql import select, and_, not_ + + deck = self.wrapper.open() + + # remove extra cards + selectExtra = select([fieldsTable.c.factId], + and_( + fieldsTable.c.fieldModelId == self.wrapper.external_field_id, + not_(fieldsTable.c.value.in_(external_ids)) + ) + ) + for factId, in deck.s.execute(selectExtra).fetchall(): + deck.deleteFact(factId) + deck.save() + + # find ids that should be on this deck but which aren't + missing_ids = [] + for external_id in external_ids: + if self.wrapper.find_fact(external_id) is None: + missing_ids.append(external_id) + + return {'missing':missing_ids} + + def get_card(self): + deck = self.wrapper.open() + card = deck.getCard() + if card: + # grab the interval strings + intervals = [] + for i in range(1, 5): + intervals.append(deck.nextIntervalStr(card, i)) + + card = self._output_card(card) + card['intervals'] = intervals + card['finished'] = False + else: + # copied from Deck.nextDueMsg() in libanki/anki/deck.py + newCount = deck.newCardsDueBy(deck.dueCutoff + 86400) + newCardsTomorrow = min(newCount, deck.newCardsPerDay) + cards = deck.cardsDueBy(deck.dueCutoff + 86400) + + card = { + 'finished': True, + 'new_count': newCardsTomorrow, + 'reviews_count': cards + } + + # TODO: clean up a bit, now that we've finished this review + + return card + + @opts(waitForReturn=False) + def setup_scheduler(self, name): + deck = self.wrapper.open() + if name == 'standard': + deck.setupStandardScheduler() + elif name == 'reviewEarly': + deck.setupReviewEarlyScheduler() + elif name == 'learnMore': + deck.setupLearnMoreScheduler() + deck.refreshSession() + deck.reset() + + def get_options(self): + deck = self.wrapper.open() + + return { + 'new_cards': { + 'cards_per_day': deck.newCardsPerDay, + 'order': deck.newCardOrder, + 'spacing': deck.newCardSpacing, + }, + 'reviews': { + 'failed_card_max': deck.failedCardMax, + 'order': deck.revCardOrder, + 'failed_policy': deck.getFailedCardPolicy(), + } + } + + @opts(waitForReturn=False) + def set_options(self, study_options): + deck = self.wrapper.open() + + # new card options + deck.newCardsPerDay = int(study_options['new_cards']['cards_per_day']) + deck.newCardOrder = int(study_options['new_cards']['order']) + if deck.newCardOrder == anki.deck.NEW_CARDS_RANDOM: + deck.randomizeNewCards() + deck.newCardSpacing = int(study_options['new_cards']['spacing']) + + # reviews options + deck.setFailedCardPolicy(int(study_options['reviews']['failed_policy'])) + deck.failedCardMax = int(study_options['reviews']['failed_card_max']) + deck.revCardOrder = int(study_options['reviews']['order']) + + deck.flushMod() + deck.reset() + deck.save() + + def answer_card(self, card_id, ease): + ease = int(ease) + deck = self.wrapper.open() + card = deck.cardFromId(card_id) + if card: + try: + deck.answerCard(card, ease) + except: + import sys, traceback + exc_info = sys.exc_info() + print exc_info[1] + print traceback.print_tb(exc_info[2]) + return False + deck.save() + return True + +class DeckApp(object): + """ Our WSGI app. """ + + direct_operations = ['add_fact', 'save_fact', 'find_fact', 'delete_fact', 'resync_facts', + 'get_card', 'answer_card'] + + def __init__(self, data_root, allowed_hosts): + self.data_root = os.path.abspath(data_root) + self.allowed_hosts = allowed_hosts + + def _get_path(self, path): + npath = os.path.normpath(os.path.join(self.data_root, path)) + if npath[0:len(self.data_root)] != self.data_root: + # attempting to escape our data jail! + raise HTTPBadRequest('"%s" is not a valid path/id' % path) + return npath + + @wsgify + def __call__(self, req): + global thread_pool + + if self.allowed_hosts != '*': + try: + remote_addr = req.headers['X-Forwarded-For'] + except KeyError: + remote_addr = req.remote_addr + if remote_addr != self.allowed_hosts: + raise HTTPForbidden() + + if req.method != 'POST': + raise HTTPMethodNotAllowed(allow=['POST']) + + # get the deck and function to call from the path + func = req.path + if func[0] == '/': + func = func[1:] + parts = func.split('/') + path = '/'.join(parts[:-1]) + func = parts[-1] + if func[0] == '_' or not hasattr(DeckAppHandler, func) or not callable(getattr(DeckAppHandler, func)): + raise HTTPNotFound() + thread = thread_pool.start(self._get_path(path)) + handler = DeckAppHandler(thread.wrapper) + func = getattr(handler, func) + try: + opts = func.opts + except AttributeError: + opts = {} + + try: + input = json.loads(req.body) + except ValueError, e: + logging.error(req.path+': Unable to parse JSON: '+str(e), exc_info=True) + raise HTTPBadRequest() + # make the keys into non-unicode strings + input = dict([(str(k), v) for k, v in input.items()]) + + # debug + from pprint import pprint + pprint(input) + + # run it! + try: + output = thread.execute(func, [], input, **opts) + except Exception, e: + logging.error(e) + return HTTPInternalServerError() + + if output is None: + return Response('', content_type='text/plain') + else: + return Response(json.dumps(output), content_type='application/json') + +# Our entry point +def make_app(global_conf, **local_conf): + # setup the logger + logging_config_file = local_conf.get('logging.config_file') + if logging_config_file: + # monkey patch the logging.config.SMTPHandler if necessary + import sys + if sys.version_info[0] == 2 and sys.version_info[1] == 5: + import AnkiServer.logpatch + + # load the config file + import logging.config + logging.config.fileConfig(logging_config_file) + + return DeckApp( + data_root=local_conf.get('data_root', '.'), + allowed_hosts=local_conf.get('allowed_hosts', '*') + ) + diff --git a/AnkiServer/logpatch.py b/AnkiServer/logpatch.py new file mode 100644 index 0000000..eeeda58 --- /dev/null +++ b/AnkiServer/logpatch.py @@ -0,0 +1,96 @@ + +import logging +import logging.handlers +import types + +# The SMTPHandler taken from python 2.6 +class SMTPHandler(logging.Handler): + """ + A handler class which sends an SMTP email for each logging event. + """ + def __init__(self, mailhost, fromaddr, toaddrs, subject, credentials=None): + """ + Initialize the handler. + + Initialize the instance with the from and to addresses and subject + line of the email. To specify a non-standard SMTP port, use the + (host, port) tuple format for the mailhost argument. To specify + authentication credentials, supply a (username, password) tuple + for the credentials argument. + """ + logging.Handler.__init__(self) + if type(mailhost) == types.TupleType: + self.mailhost, self.mailport = mailhost + else: + self.mailhost, self.mailport = mailhost, None + if type(credentials) == types.TupleType: + self.username, self.password = credentials + else: + self.username = None + self.fromaddr = fromaddr + if type(toaddrs) == types.StringType: + toaddrs = [toaddrs] + self.toaddrs = toaddrs + self.subject = subject + + def getSubject(self, record): + """ + Determine the subject for the email. + + If you want to specify a subject line which is record-dependent, + override this method. + """ + return self.subject + + weekdayname = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'] + + monthname = [None, + 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', + 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] + + def date_time(self): + """ + Return the current date and time formatted for a MIME header. + Needed for Python 1.5.2 (no email package available) + """ + year, month, day, hh, mm, ss, wd, y, z = time.gmtime(time.time()) + s = "%s, %02d %3s %4d %02d:%02d:%02d GMT" % ( + self.weekdayname[wd], + day, self.monthname[month], year, + hh, mm, ss) + return s + + def emit(self, record): + """ + Emit a record. + + Format the record and send it to the specified addressees. + """ + try: + import smtplib + try: + from email.utils import formatdate + except ImportError: + formatdate = self.date_time + port = self.mailport + if not port: + port = smtplib.SMTP_PORT + smtp = smtplib.SMTP(self.mailhost, port) + msg = self.format(record) + msg = "From: %s\r\nTo: %s\r\nSubject: %s\r\nDate: %s\r\n\r\n%s" % ( + self.fromaddr, + string.join(self.toaddrs, ","), + self.getSubject(record), + formatdate(), msg) + if self.username: + smtp.login(self.username, self.password) + smtp.sendmail(self.fromaddr, self.toaddrs, msg) + smtp.quit() + except (KeyboardInterrupt, SystemExit): + raise + except: + self.handleError(record) + +# Monkey patch logging.handlers +logging.handlers.SMTPHandler = SMTPHandler + diff --git a/AnkiServer/sync.py b/AnkiServer/sync.py new file mode 100644 index 0000000..d504993 --- /dev/null +++ b/AnkiServer/sync.py @@ -0,0 +1,377 @@ + +from webob.dec import wsgify +from webob.exc import * +from webob import Response + +import anki +from anki.sync import HttpSyncServer, CHUNK_SIZE +from anki.db import sqlite +from anki.utils import checksum + +import AnkiServer.deck + +import MySQLdb + +try: + import simplejson as json +except ImportError: + import json + +import os, zlib, tempfile, time + +def makeArgs(mdict): + d = dict(mdict.items()) + # TODO: use password/username/version for something? + for k in ['p','u','v','d']: + if d.has_key(k): + del d[k] + return d + +class FileIterable(object): + def __init__(self, fn): + self.fn = fn + def __iter__(self): + return FileIterator(self.fn) + +class FileIterator(object): + def __init__(self, fn): + self.fn = fn + self.fo = open(self.fn, 'rb') + self.c = zlib.compressobj() + self.flushed = False + def __iter__(self): + return self + def next(self): + data = self.fo.read(CHUNK_SIZE) + if not data: + if not self.flushed: + self.flushed = True + return self.c.flush() + else: + raise StopIteration + return self.c.compress(data) + +def lock_deck(path): + """ Gets exclusive access to this deck path. If there is a DeckThread running on this + deck, this will wait for its current operations to complete before temporarily stopping + it. """ + + from AnkiServer.deck import thread_pool + + if thread_pool.decks.has_key(path): + thread_pool.decks[path].stop_and_wait() + thread_pool.lock(path) + +def unlock_deck(path): + """ Release exclusive access to this deck path. """ + from AnkiServer.deck import thread_pool + thread_pool.unlock(path) + +class SyncAppHandler(HttpSyncServer): + operations = ['summary','applyPayload','finish','createDeck','getOneWayPayload'] + + def __init__(self): + HttpSyncServer.__init__(self) + + def createDeck(self, name): + # The HttpSyncServer.createDeck doesn't return a valid value! This seems to be + # a bug in libanki.sync ... + return self.stuff({"status": "OK"}) + + def finish(self): + # The HttpSyncServer has no finish() function... I can only assume this is a bug too! + return self.stuff("OK") + +class SyncApp(object): + valid_urls = SyncAppHandler.operations + ['getDecks','fullup','fulldown'] + + def __init__(self, **kw): + self.data_root = os.path.abspath(kw.get('data_root', '.')) + self.base_url = kw.get('base_url', '/') + self.users = {} + + # make sure the base_url has a trailing slash + if len(self.base_url) == 0: + self.base_url = '/' + elif self.base_url[-1] != '/': + self.base_url = base_url + '/' + + # setup mysql connection + mysql_args = {} + for k, v in kw.items(): + if k.startswith('mysql.'): + mysql_args[k[6:]] = v + self.mysql_args = mysql_args + self.conn = None + + # get SQL statements + self.sql_check_password = kw.get('sql_check_password') + self.sql_username2dirname = kw.get('sql_username2dirname') + + default_libanki_version = '.'.join(anki.version.split('.')[:2]) + + def user_libanki_version(self, u): + try: + s = self.users[u]['libanki'] + except KeyError: + return self.default_libanki_version + + parts = s.split('.') + if parts[0] == '1': + if parts[1] == '0': + return '1.0' + elif parts[1] in ('1','2'): + return '1.2' + + return self.default_libanki_version + + # Mimcs from anki.sync.SyncTools.stuff() + def _stuff(self, data): + return zlib.compress(json.dumps(data)) + + def _connect_mysql(self): + if self.conn is None and len(self.mysql_args) > 0: + self.conn = MySQLdb.connect(**self.mysql_args) + + def _execute_sql(self, sql, args=()): + self._connect_mysql() + try: + cur = self.conn.cursor() + cur.execute(sql, args) + except MySQLdb.OperationalError, e: + if e.args[0] == 2006: + # MySQL server has gone away message + self.conn = None + self._connect_mysql() + cur = self.conn.cursor() + cur.execute(sql, args) + return cur + + def check_password(self, username, password): + if len(self.mysql_args) > 0 and self.sql_check_password is not None: + cur = self._execute_sql(self.sql_check_password, (username, password)) + row = cur.fetchone() + return row is not None + + return True + + def username2dirname(self, username): + if len(self.mysql_args) > 0 and self.sql_username2dirname is not None: + cur = self._execute_sql(self.sql_username2dirname, (username,)) + row = cur.fetchone() + if row is None: + return None + return str(row[0]) + + return username + + def _getDecks(self, user_path): + decks = {} + + if os.path.exists(user_path): + # It is a dict of {'deckName':[modified,lastSync]} + for fn in os.listdir(unicode(user_path, 'utf-8')): + if len(fn) > 5 and fn[-5:] == '.anki': + d = os.path.abspath(os.path.join(user_path, fn)) + + # For simplicity, we will always open a thread. But this probably + # isn't necessary! + thread = AnkiServer.deck.thread_pool.start(d) + def lookupModifiedLastSync(wrapper): + deck = wrapper.open() + return [deck.modified, deck.lastSync] + res = thread.execute(lookupModifiedLastSync, [thread.wrapper]) + +# if thread_pool.threads.has_key(d): +# thread = thread_pool.threads[d] +# def lookupModifiedLastSync(wrapper): +# deck = wrapper.open() +# return [deck.modified, deck.lastSync] +# res = thread.execute(lookup, [thread.wrapper]) +# else: +# conn = sqlite.connect(d) +# cur = conn.cursor() +# cur.execute("select modified, lastSync from decks") +# +# res = list(cur.fetchone()) +# +# cur.close() +# conn.close() + + #self.decks[fn[:-5]] = ["%.5f" % x for x in res] + decks[fn[:-5]] = res + + # same as HttpSyncServer.getDecks() + return self._stuff({ + "status": "OK", + "decks": decks, + "timestamp": time.time(), + }) + + def _fullup(self, wrapper, infile, version): + wrapper.close() + path = wrapper.path + + # DRS: most of this function was graciously copied + # from anki.sync.SyncTools.fullSyncFromServer() + (fd, tmpname) = tempfile.mkstemp(dir=os.getcwd(), prefix="fullsync") + outfile = open(tmpname, 'wb') + decomp = zlib.decompressobj() + while 1: + data = infile.read(CHUNK_SIZE) + if not data: + outfile.write(decomp.flush()) + break + outfile.write(decomp.decompress(data)) + infile.close() + outfile.close() + os.close(fd) + # if we were successful, overwrite old deck + if os.path.exists(path): + os.unlink(path) + os.rename(tmpname, path) + # reset the deck name + c = sqlite.connect(path) + lastSync = time.time() + if version == '1': + c.execute("update decks set lastSync = ?", [lastSync]) + elif version == '2': + c.execute("update decks set syncName = ?, lastSync = ?", + [checksum(path.encode("utf-8")), lastSync]) + c.commit() + c.close() + + return lastSync + + def _stuffedResp(self, data): + return Response( + status='200 OK', + content_type='application/json', + content_encoding='deflate', + body=data) + + @wsgify + def __call__(self, req): + if req.path.startswith(self.base_url): + url = req.path[len(self.base_url):] + if url not in self.valid_urls: + raise HTTPNotFound() + + # get and check username and password + try: + u = req.str_params.getone('u') + p = req.str_params.getone('p') + except KeyError: + raise HTTPBadRequest('Must pass username and password') + if not self.check_password(u, p): + #raise HTTPBadRequest('Incorrect username or password') + return self._stuffedResp(self._stuff({'status':'invalidUserPass'})) + dirname = self.username2dirname(u) + if dirname is None: + raise HTTPBadRequest('Incorrect username or password') + user_path = os.path.join(self.data_root, dirname) + + # get and lock the (optional) deck for this request + d = None + try: + d = unicode(req.str_params.getone('d'), 'utf-8') + # AnkiDesktop actually passes us the string value 'None'! + if d == 'None': + d = None + except KeyError: + pass + if d is not None: + # get the full deck path name + d = os.path.abspath(os.path.join(user_path, d)+'.anki') + if d[:len(user_path)] != user_path: + raise HTTPBadRequest('Bad deck name') + thread = AnkiServer.deck.thread_pool.start(d) + else: + thread = None + + if url == 'getDecks': + # force the version up to 1.2.x + v = req.str_params.getone('libanki') + if v.startswith('0.') or v.startswith('1.0'): + return self._stuffedResp(self._stuff({'status':'oldVersion'})) + + # store the data the user passes us keyed with the username. This + # will be used later by SyncAppHandler for version compatibility. + self.users[u] = makeArgs(req.str_params) + return self._stuffedResp(self._getDecks(user_path)) + + elif url in SyncAppHandler.operations: + handler = SyncAppHandler() + func = getattr(handler, url) + args = makeArgs(req.str_params) + + if thread is not None: + # If this is for a specific deck, then it needs to run + # inside of the DeckThread. + def runFunc(wrapper): + handler.deck = wrapper.open() + ret = func(**args) + handler.deck.save() + return ret + runFunc.func_name = url + ret = thread.execute(runFunc, [thread.wrapper]) + else: + # Otherwise, we can simply execute it in this thread. + ret = func(**args) + + # clean-up user data stored in getDecks + if url == 'finish': + del self.users[u] + + return self._stuffedResp(ret) + + elif url == 'fulldown': + # set the syncTime before we send it + def setupForSync(wrapper): + wrapper.close() + c = sqlite.connect(d) + lastSync = time.time() + c.execute("update decks set lastSync = ?", [lastSync]) + c.commit() + c.close() + thread.execute(setupForSync, [thread.wrapper]) + + return Response(status='200 OK', content_type='application/octet-stream', content_encoding='deflate', content_disposition='attachment; filename="'+os.path.basename(d).encode('utf-8')+'"', app_iter=FileIterable(d)) + elif url == 'fullup': + #version = self.user_libanki_version(u) + try: + version = req.str_params.getone('v') + except KeyError: + version = '1' + + infile = req.str_params['deck'].file + lastSync = thread.execute(self._fullup, [thread.wrapper, infile, version]) + + # append the 'lastSync' value for libanki 1.1 and 1.2 + if version == '2': + body = 'OK '+str(lastSync) + else: + body = 'OK' + + return Response(status='200 OK', content_type='application/text', body=body) + + return Response(status='200 OK', content_type='text/plain', body='Anki Server') + +# Our entry point +def make_app(global_conf, **local_conf): + return SyncApp(**local_conf) + +def main(): + from wsgiref.simple_server import make_server + + ankiserver = DeckApp('.', '/sync/') + httpd = make_server('', 8001, ankiserver) + try: + httpd.serve_forever() + except KeyboardInterrupt: + print "Exiting ..." + finally: + AnkiServer.deck.thread_pool.shutdown() + +if __name__ == '__main__': main() + diff --git a/INSTALL.txt b/INSTALL.txt new file mode 100644 index 0000000..8e9095e --- /dev/null +++ b/INSTALL.txt @@ -0,0 +1,37 @@ + +Instructions for installing and running AnkiServer: + + 1. First, you need to install "virtualenv". If your system has easy_install, this is + just a matter of: + + $ easy_install virtualenv + + If your system doesn't have easy_install, I recommend getting it! + + 2. Next, you need to create a Python environment for running AnkiServer and install some of + the dependencies we need there: + + $ virtualenv AnkiServer.env + $ AnkiServer.env/bin/easy_install webob PasteDeploy PasteScript sqlalchemy simplejson MySQL-python + + 3. Download and install libanki. You can find the latest release of Anki here: + + http://code.google.com/p/anki/downloads/list + + Look for a *.tgz file with a Summary of "Anki Source". At the time of this writing + that is anki-1.0.1.tgz. + + Download this file and extract. Inside you will find a libanki/ directory with a + setup.py. You will want to run setup.py with the python executable inside our virtualenv, + or AnkiServer.env/bin/python, like so: + + anki-1.0.1/libanki$ ../../AnkiServer.env/bin/python setup.py install + + 4. Make the egg info files (so paster can see our app): + + $ AnkiServer.env/bin/python setup.py egg_info + + 5. Then we can run AnkiServer like so: + + $ AnkiServer.env/bin/paster serve development.ini + diff --git a/example.ini b/example.ini new file mode 100644 index 0000000..a620063 --- /dev/null +++ b/example.ini @@ -0,0 +1,33 @@ + +[server:main] +#use = egg:Paste#http +use = egg:AnkiServer#server +host = 127.0.0.1 +port = 27701 + +[filter-app:main] +use = egg:Paste#translogger +next = real + +[app:real] +use = egg:Paste#urlmap +/decks = deckapp +/sync = syncapp + +[app:deckapp] +use = egg:AnkiServer#deckapp +data_root = ./decks +allowed_hosts = 127.0.0.1 +logging.config_file = logging.conf + +[app:syncapp] +use = egg:AnkiServer#syncapp +data_root = ./decks +base_url = /sync/ +mysql.host = 127.0.0.1 +mysql.user = db_user +mysql.passwd = db_password +mysql.db = db +sql_check_password = SELECT uid FROM users WHERE name=%s AND pass=MD5(%s) AND status=1 +sql_username2dirname = SELECT uid AS dirname FROM users WHERE name=%s + diff --git a/logging.conf b/logging.conf new file mode 100644 index 0000000..fb94bf0 --- /dev/null +++ b/logging.conf @@ -0,0 +1,41 @@ + +[loggers] +keys=root + +[handlers] +keys=screen,file,email + +[formatters] +keys=normal,email + +[logger_root] +level=INFO +handlers=screen +#handlers=file +#handlers=file,email + +[handler_file] +class=FileHandler +formatter=normal +args=('server.log','a') + +[handler_screen] +class=StreamHandler +level=NOTSET +formatter=normal +args=(sys.stdout,) + +[handler_email] +class=handlers.SMTPHandler +level=ERROR +formatter=email +args=('smtp.example.com', 'support@example.com', ['support_guy1@example.com', 'support_guy2@example.com'], 'AnkiServer error', ('smtp_user', 'smtp_password')) + +[formatter_normal] +format=%(asctime)s:%(name)s:%(levelname)s:%(message)s +datefmt= + +[formatter_email] +format=%(asctime)s - %(name)s - %(levelname)s - %(message)s +datefmt= + diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..8688fc4 --- /dev/null +++ b/setup.py @@ -0,0 +1,23 @@ + +from setuptools import setup, find_packages + +setup( + name="AnkiServer", + version="0.0.1", + description="Provides the a RESTful API to manipulating Anki decks", + author="David Snopek", + author_email="dsnopek@gmail.com", + install_requires=["PasteDeploy>=1.3.2"], + # TODO: should these really be in install_requires? + requires=["webob(>=0.9.7)"], + test_suite='nose.collector', + entry_points=""" + [paste.app_factory] + deckapp = AnkiServer.deck:make_app + syncapp = AnkiServer.sync:make_app + + [paste.server_runner] + server = AnkiServer:server_runner + """, +) +