Squashed commit of the following:

commit cb509e8f75e3dcdbc66327be4bfbf6661aa084b5
Author: David Snopek <dsnopek@gmail.com>
Date:   Fri Jul 12 22:06:28 2013 +0100

    Cut down 'import' statements to only modules actually used.

commit 0ea255115e095e31af5a991e9cce2b5b15cb496d
Author: David Snopek <dsnopek@gmail.com>
Date:   Fri Jul 12 22:00:06 2013 +0100

     * Add getCollectionManager() so that the whole process can share the same ThreadingCollectionManager object.

     * Got the RestApp actually working!

commit 00997bab600b13d4b430ed2c2839b1d2232f55ed
Author: David Snopek <dsnopek@gmail.com>
Date:   Fri Jul 12 21:04:58 2013 +0100

    Got the sync_app working again (more or less)

commit 459c69566bb92d2c0195a384e067d98c059bdea7
Author: David Snopek <dsnopek@gmail.com>
Date:   Fri Jul 12 19:47:40 2013 +0100

    Started implementing test for the RESTful callbacks that PrepECN is going to need.

commit 7ffbac793f9bf45ab9056c1de475422b8742e107
Author: David Snopek <dsnopek@gmail.com>
Date:   Fri Jul 12 17:19:06 2013 +0100

    Started work on a WSGI app for RESTful access to Anki based on Bibliobird code here:

      https://raw.github.com/dsnopek/bbcom/master/AnkiServer/AnkiServer/deck.py

commit 8820411388ce0c2b7b14769c614c22c675d2dbdd
Author: David Snopek <dsnopek@gmail.com>
Date:   Fri Jul 12 15:03:56 2013 +0100

     * Seperated the collection and threading code.

     * Implemented a new interface to interact with the collections, which will hopefully be more transparent and testable.

.gitignore

@@ -1,4 +1,6 @@
 *~
 *.pyc
+AnkiServer.egg-info
 development.ini
-
+server.log
+collections

AnkiServer/__init__.py

@@ -1,12 +1,15 @@
 
+import sys
+sys.path.insert(0, "/usr/share/anki")
+
 def server_runner(app, global_conf, **kw):
-    """ Special version of paste.httpserver.server_runner which shuts down
+    """ Special version of paste.httpserver.server_runner which calls 
-    the AnkiServer.deck.thread_pool on server exit. """
+    AnkiServer.threading.shutdown() on server exit."""
 
     from paste.httpserver import server_runner as paste_server_runner
-    from AnkiServer.deck import thread_pool
+    from AnkiServer.threading import shutdown
     try:
         paste_server_runner(app, global_conf, **kw)
     finally:
-        thread_pool.shutdown()
+        shutdown()
 

AnkiServer/apps/__init__.py

@@ -0,0 +1,2 @@
+# package
+

AnkiServer/apps/rest_app.py

@@ -0,0 +1,225 @@
+
+from webob.dec import wsgify
+from webob.exc import *
+from webob import Response
+
+try:
+    import simplejson as json
+except ImportError:
+    import json
+
+import os, logging
+
+__all__ = ['RestApp', 'RestHandlerBase', 'hasReturnValue', 'noReturnValue']
+
+def hasReturnValue(func):
+    func.hasReturnValue = True
+    return func
+
+def noReturnValue(func):
+    func.hasReturnValue = False
+    return func
+
+class RestHandlerBase(object):
+    """Parent class for single handler callbacks."""
+    hasReturnValue = True
+    def __call__(self, collection, data, ids):
+        pass
+
+class RestHandlerGroupBase(object):
+    """Parent class for a handler group."""
+    hasReturnValue = True
+
+class _RestHandlerWrapper(RestHandlerBase):
+    def __init__(self, func_name, func, hasReturnValue=True):
+        self.func_name = func_name
+        self.func = func
+        self.hasReturnValue = hasReturnValue
+    def __call__(self, *args, **kw):
+        return self.func(*args, **kw)
+
+class RestApp(object):
+    """A WSGI app that implements RESTful operations on Collections, Decks and Cards."""
+
+    handler_types = ['collection', 'deck', 'note']
+
+    def __init__(self, data_root, allowed_hosts='*', use_default_handlers=True, collection_manager=None):
+        from AnkiServer.threading import getCollectionManager
+
+        self.data_root = os.path.abspath(data_root)
+        self.allowed_hosts = allowed_hosts
+
+        if collection_manager is not None:
+            self.collection_manager = collection_manager
+        else:
+            self.collection_manager = getCollectionManager()
+
+        self.handlers = {}
+        for type in self.handler_types:
+            self.handlers[type] = {}
+
+        if use_default_handlers:
+            self.add_handler_group('collection', CollectionHandlerGroup())
+            self.add_handler_group('deck', DeckHandlerGroup())
+            self.add_handler_group('note', NoteHandlerGroup())
+
+    def _get_path(self, path):
+        npath = os.path.normpath(os.path.join(self.data_root, path, 'collection.anki2'))
+        if npath[0:len(self.data_root)] != self.data_root:
+            # attempting to escape our data jail!
+            raise HTTPBadRequest('"%s" is not a valid path/id' % path)
+        return npath
+
+    def add_handler(self, type, name, handler):
+        """Adds a callback handler for a type (collection, deck, card) with a unique name.
+        
+         - 'type' is the item that will be worked on, for example: collection, deck, and card.
+
+         - 'name' is a unique name for the handler that gets used in the URL.
+
+         - 'handler' handler can be a Python method or a subclass of the RestHandlerBase class.
+        """
+
+        if self.handlers[type].has_key(name):
+            raise "Handler already for %(type)s/%(name)s exists!"
+        self.handlers[type][name] = handler
+
+    def add_handler_group(self, type, group):
+        """Adds several handlers for every public method on an object descended from RestHandlerGroup.
+        
+        This allows you to create a single class with several methods, so that you can quickly
+        create a group of related handlers."""
+
+        import inspect
+        for name, method in inspect.getmembers(group, predicate=inspect.ismethod):
+            if not name.startswith('_'):
+                if hasattr(group, 'hasReturnValue') and not hasattr(method, 'hasReturnValue'):
+                    method = _RestHandlerWrapper(group.__class__.__name__ + '.' + name, method, group.hasReturnValue)
+                self.add_handler(type, name, method)
+
+    @wsgify
+    def __call__(self, req):
+        if self.allowed_hosts != '*':
+            try:
+                remote_addr = req.headers['X-Forwarded-For']
+            except KeyError:
+                remote_addr = req.remote_addr
+            if remote_addr != self.allowed_hosts:
+                raise HTTPForbidden()
+
+        if req.method != 'POST':
+            raise HTTPMethodNotAllowed(allow=['POST'])
+
+        # split the URL into a list of parts
+        path = req.path
+        if path[0] == '/':
+            path = path[1:]
+        parts = path.split('/')
+
+        # pull the type and context from the URL parts
+        type = None
+        ids = []
+        for type in self.handler_types:
+            if len(parts) == 0 or parts.pop(0) != type:
+                break
+            if len(parts) > 0:
+                ids.append(parts.pop(0))
+            if len(parts) < 2:
+                break
+        # sanity check to make sure the URL is valid
+        if type is None or len(parts) > 1 or len(ids) == 0:
+            raise HTTPNotFound()
+
+        # get the handler name
+        if len(parts) == 0:
+            name = 'index'
+        else:
+            name = parts[0]
+
+        # get the collection path
+        collection_path = self._get_path(ids[0])
+        print collection_path
+
+        # get the handler function
+        try:
+            handler = self.handlers[type][name]
+        except KeyError:
+            raise HTTPNotFound()
+         
+        # get if we have a return value
+        hasReturnValue = True
+        if hasattr(handler, 'hasReturnValue'):
+            hasReturnValue = handler.hasReturnValue
+
+        try:
+            data = json.loads(req.body)
+        except ValueError, e:
+            logging.error(req.path+': Unable to parse JSON: '+str(e), exc_info=True)
+            raise HTTPBadRequest()
+        # make the keys into non-unicode strings
+        data = dict([(str(k), v) for k, v in data.items()])
+
+        # debug
+        from pprint import pprint
+        pprint(data)
+
+        # run it!
+        col = self.collection_manager.get_collection(collection_path)
+        try:
+            output = col.execute(handler, [data, ids], {}, hasReturnValue)
+        except Exception, e:
+            logging.error(e)
+            return HTTPInternalServerError()
+
+        if output is None:
+            return Response('', content_type='text/plain')
+        else:
+            return Response(json.dumps(output), content_type='application/json')
+
+class CollectionHandlerGroup(RestHandlerGroupBase):
+    """Default handler group for 'collection' type."""
+
+    def list_decks(self, col, data, ids):
+        return col.decks.all()
+
+    @noReturnValue
+    def select_deck(self, col, data, ids):
+        col.decks.select(data['deck_id'])
+
+class DeckHandlerGroup(RestHandlerGroupBase):
+    """Default handler group for 'deck' type."""
+
+    def next_card(self, col, data, ids):
+        deck_id = ids[1]
+
+        col.decks.select(deck_id)
+        card = col.sched.getCard()
+
+        return card
+
+class NoteHandlerGroup(RestHandlerGroupBase):
+    """Default handler group for 'note' type."""
+
+    def add_new(self, col, data, ids):
+        # col.addNote(...)
+        pass
+
+# Our entry point
+def make_app(global_conf, **local_conf):
+    # setup the logger
+    logging_config_file = local_conf.get('logging.config_file')
+    if logging_config_file:
+        # monkey patch the logging.config.SMTPHandler if necessary
+        import sys
+        if sys.version_info[0] == 2 and sys.version_info[1] == 5:
+            import AnkiServer.logpatch
+
+        # load the config file
+        import logging.config
+        logging.config.fileConfig(logging_config_file)
+
+    return RestApp(
+        data_root=local_conf.get('data_root', '.'),
+        allowed_hosts=local_conf.get('allowed_hosts', '*')
+    )
+

AnkiServer/apps/sync_app.py

@@ -3,21 +3,17 @@ from webob.dec import wsgify
 from webob.exc import *
 from webob import Response
 
-# TODO: I don't think this should have to be at the top of every module!
+import AnkiServer
-import sys
-sys.path.insert(0, "/usr/share/anki")
 
 import anki
 from anki.sync import LocalServer, MediaSyncer
-# TODO: shouldn't use this directly! This should be through the thread pool
-from anki.storage import Collection
 
 try:
     import simplejson as json
 except ImportError:
     import json
 
-import os, tempfile
+import os
 
 class SyncCollectionHandler(LocalServer):
     operations = ['meta', 'applyChanges', 'start', 'chunk', 'applyChunk', 'sanityCheck2', 'finish']
@@ -61,10 +57,11 @@ class SyncMediaHandler(MediaSyncer):
         return fd.getvalue()
 
 class SyncUserSession(object):
-    def __init__(self, name, path):
+    def __init__(self, name, path, collection_manager):
         import time
         self.name = name
         self.path = path
+        self.collection_manager = collection_manager
         self.version = 0
         self.created = time.time()
 
@@ -79,8 +76,7 @@ class SyncUserSession(object):
         return os.path.realpath(os.path.join(self.path, 'collection.anki2'))
     
     def get_thread(self):
-        from AnkiServer.collection import thread_pool
+        return self.collection_manager.get_collection(self.get_collection_path())
-        return thread_pool.start(self.get_collection_path())
 
     def get_handler_for_operation(self, operation, col):
         if operation in SyncCollectionHandler.operations:
@@ -96,10 +92,17 @@ class SyncApp(object):
     valid_urls = SyncCollectionHandler.operations + SyncMediaHandler.operations + ['hostKey', 'upload', 'download', 'getDecks']
 
     def __init__(self, **kw):
+        from AnkiServer.threading import getCollectionManager
+
         self.data_root = os.path.abspath(kw.get('data_root', '.'))
         self.base_url  = kw.get('base_url', '/')
         self.sessions = {}
 
+        try:
+            self.collection_manager = kw['collection_manager']
+        except KeyError:
+            self.collection_manager = getCollectionManager()
+
         # make sure the base_url has a trailing slash
         if len(self.base_url) == 0:
             self.base_url = '/'
@@ -137,7 +140,7 @@ class SyncApp(object):
     def create_session(self, hkey, username, user_path):
         """Creates, stores and returns a new session for the given hkey and username."""
 
-        session = self.sessions[hkey] = SyncUserSession(username, user_path)
+        session = self.sessions[hkey] = SyncUserSession(username, user_path, self.collection_manager)
         return session
 
     def load_session(self, hkey):
@@ -165,14 +168,14 @@ class SyncApp(object):
 
         return data
 
-    def operation_upload(self, wrapper, data, session):
+    def operation_upload(self, col, data, session):
         # TODO: deal with thread pool
 
         fd = open(session.get_collection_path(), 'wb')
         fd.write(data)
         fd.close()
 
-    def operation_download(self, wrapper, data, session):
+    def operation_download(self, col, data, session):
         pass
 
     @wsgify
@@ -247,8 +250,8 @@ class SyncApp(object):
                     del data['v']
 
                 # Create a closure to run this operation inside of the thread allocated to this collection
-                def runFunc(wrapper):
+                def runFunc(col):
-                    handler = session.get_handler_for_operation(url, wrapper.open())
+                    handler = session.get_handler_for_operation(url, col)
                     func = getattr(handler, url)
                     result = func(**data)
                     handler.col.save()
@@ -257,7 +260,7 @@ class SyncApp(object):
 
                 # Send to the thread to execute
                 thread = session.get_thread()
-                result = thread.execute(runFunc, [thread.wrapper])
+                result = thread.execute(runFunc)
 
                 # If it's a complex data type, we convert it to JSON
                 if type(result) not in (str, unicode):
@@ -278,7 +281,7 @@ class SyncApp(object):
                     func = self.operation_download
 
                 thread = session.get_thread()
-                thread.execute(self.operation_upload, [thread.wrapper, data['data'], session])
+                thread.execute(self.operation_upload, [data['data'], session])
 
                 return Response(
                     status='200 OK',
@@ -296,7 +299,7 @@ def make_app(global_conf, **local_conf):
 
 def main():
     from wsgiref.simple_server import make_server
-    from AnkiServer.collection import thread_pool
+    from AnkiServer.threading import shutdown
 
     ankiserver = SyncApp()
     httpd = make_server('', 8001, ankiserver)
@@ -306,7 +309,7 @@ def main():
     except KeyboardInterrupt:
         print "Exiting ..."
     finally:
-        thread_pool.shutdown()
+        shutdown()
 
 if __name__ == '__main__': main()
 

AnkiServer/collection.py

@@ -2,28 +2,49 @@
 import anki
 import anki.storage
 
-from threading import Thread
+import os, errno
-from Queue import Queue
 
-try:
+__all__ = ['CollectionWrapper', 'CollectionManager']
-    import simplejson as json
-except ImportError:
-    import json
 
-import os, errno, time, logging
-
-__all__ = ['CollectionThread']
-
-# TODO: I feel like we shouldn't need this wrapper...
 class CollectionWrapper(object):
-    """A simple wrapper around a collection for the purpose of opening and closing on demand
+    """A simple wrapper around an anki.storage.Collection object.
-    as well as doing special initialization."""
 
-    def __init__(self, path):
+    This allows us to manage and refer to the collection, whether it's open or not. It
+    also provides a special "continuation passing" interface for executing functions
+    on the collection, which makes it easy to switch to a threading mode.
+    
+    See ThreadingCollectionWrapper for a version that maintains a seperate thread for
+    interacting with the collection.
+    """
+
+    def __init__(self, path, setup_new_collection=None):
         self.path = os.path.realpath(path)
-        self._col = None
+        self.setup_new_collection = setup_new_collection
+        self.__col = None
+
+    def execute(self, func, args=[], kw={}, waitForReturn=True):
+        """ Executes the given function with the underlying anki.storage.Collection
+        object as the first argument and any additional arguments specified by *args
+        and **kw.
+
+        If 'waitForReturn' is True, then it will block until the function has
+        executed and return its return value.  If False, the function MAY be
+        executed some time later and None will be returned.
+        """
+
+        # Open the collection and execute the function
+        self.open()
+        args = [self.__col] + args
+        ret = func(*args, **kw)
+
+        # Only return the value if they requested it, so the interface remains
+        # identical between this class and ThreadingCollectionWrapper
+        if waitForReturn:
+            return ret
+
+    def __create_collection(self):
+        """Creates a new collection and runs any special setup."""
 
-    def _create_colection(self):
         # mkdir -p the path, because it might not exist
         dirname = os.path.dirname(self.path)
         try:
@@ -34,175 +55,55 @@ class CollectionWrapper(object):
             else:
                 raise
 
-        col = anki.Storage.Collection(self.path)
+        self.__col = ank.storage.Collection(self.path)
 
         # Do any special setup
-        self.setup_new_collection(col)
+        if self.setup_new_collection is not None:
-
+            self.setup_new_collection(self.__col)
-        return col
-
-    def setup_new_collection(self, col):
-        """Override this function to initial collections in some special way."""
-        pass
 
     def open(self):
-        if self._col is None:
+        """Open the collection, or create it if it doesn't exist."""
+        if self.__col is None:
             if os.path.exists(self.path):
-                self._col = anki.storage.Collection(self.path)
+                self.__col = anki.storage.Collection(self.path)
             else:
-                self._col = self._create_collection()
+                self.__col = self.__create_collection()
-        return self._col
 
     def close(self):
-        if self._col is None:
+        """Close the collection if opened."""
+        if not self.opened():
             return
 
-        self._col.close()
+        self.__col.close()
-        self._col = None
+        self.__col = None
 
     def opened(self):
-        return self._col is not None
+        """Returns True if the collection is open, False otherwise."""
+        return self.__col is not None
 
-class CollectionThread(object):
+class CollectionManager(object):
-    def __init__(self, path, wrapper_class=CollectionWrapper):
+    """Manages a set of CollectionWrapper objects."""
-        self.path = os.path.realpath(path)
-        self.wrapper = wrapper_class(path)
 
-        self._queue = Queue()
+    collection_wrapper = CollectionWrapper
-        self._thread = None
-        self._running = False
-        self.last_timestamp = time.time()
 
-    @property
+    def __init__(self):
-    def running(self):
+        self.collections = {}
-        return self._running
 
-    def qempty(self):
+    def get_collection(self, path, setup_new_collection=None):
-        return self._queue.empty()
+        """Gets a CollectionWrapper for the given path."""
 
-    def current(self):
-        from threading import current_thread
-        return current_thread() == self._thread
-
-    def execute(self, func, args=[], kw={}, waitForReturn=True):
-        """ Executes a given function on this thread with the *args and **kw.
-
-        If 'waitForReturn' is True, then it will block until the function has
-        executed and return its return value.  If False, it will return None
-        immediately and the function will be executed sometime later.
-        """
-
-        if waitForReturn:
-            return_queue = Queue()
-        else:
-            return_queue = None
-
-        self._queue.put((func, args, kw, return_queue))
-
-        if return_queue is not None:
-            ret = return_queue.get(True)
-            if isinstance(ret, Exception):
-                raise ret
-            return ret
-
-    def _run(self):
-        logging.info('CollectionThread[%s]: Starting...', self.path)
-
-        try:
-            while self._running:
-                func, args, kw, return_queue = self._queue.get(True)
-
-                logging.info('CollectionThread[%s]: Running %s(*%s, **%s)', self.path, func.func_name, repr(args), repr(kw))
-                self.last_timestamp = time.time()
-
-                try:
-                    ret = func(*args, **kw)
-                except Exception, e:
-                    logging.error('CollectionThread[%s]: Unable to %s(*%s, **%s): %s',
-                        self.path, func.func_name, repr(args), repr(kw), e, exc_info=True)
-                    # we return the Exception which will be raise'd on the other end
-                    ret = e
-
-                if return_queue is not None:
-                    return_queue.put(ret)
-        except Exception, e:
-            logging.error('CollectionThread[%s]: Thread crashed! Exception: %s', e, exc_info=True)
-        finally:
-            self.wrapper.close()
-            # clean out old thread object
-            self._thread = None
-            # in case we got here via an exception
-            self._running = False
-
-            logging.info('CollectionThread[%s]: Stopped!' % self.path)
-
-    def start(self):
-        if not self._running:
-            self._running = True
-            assert self._thread is None
-            self._thread = Thread(target=self._run)
-            self._thread.start()
-
-    def stop(self):
-        def _stop():
-            self._running = False
-        self.execute(_stop, waitForReturn=False)
-
-    def stop_and_wait(self):
-        """ Tell the thread to stop and wait for it to happen. """
-        self.stop()
-        if self._thread is not None:
-            self._thread.join()
-
-class CollectionThreadPool(object):
-    def __init__(self, wrapper_class=CollectionWrapper):
-        self.wrapper_class = wrapper_class
-        self.threads = {}
-
-        self.monitor_frequency = 15
-        self.monitor_inactivity = 90
-
-        monitor = Thread(target=self._monitor_run)
-        monitor.daemon = True
-        monitor.start()
-        self._monitor_thread = monitor
-
-    # TODO: it would be awesome to have a safe way to stop inactive threads completely!
-    # TODO: we need a way to inform other code that the collection has been closed
-    def _monitor_run(self):
-        """ Monitors threads for inactivity and closes the collection on them
-        (leaves the thread itself running -- hopefully waiting peacefully with only a
-        small memory footprint!) """
-        while True:
-            cur = time.time()
-            for path, thread in self.threads.items():
-                if thread.running and thread.wrapper.opened() and thread.qempty() and cur - thread.last_timestamp >= self.monitor_inactivity:
-                    logging.info('Monitor is closing collection on inactive CollectionThread[%s]' % thread.path)
-                    def closeCollection(wrapper):
-                        wrapper.close()
-                    thread.execute(closeCollection, [thread.wrapper], waitForReturn=False)
-            time.sleep(self.monitor_frequency)
-
-    def create_thread(self, path):
-        return CollectionThread(path, wrapper_class=self.wrapper_class)
-
-    def start(self, path):
         path = os.path.realpath(path)
 
         try:
-            thread = self.threads[path]
+            col = self.collections[path]
         except KeyError:
-            thread = self.threads[path] = self.create_thread(path)
+            col = self.collections[path] = self.collection_wrapper(path, setup_new_collection)
 
-        thread.start()
+        return col
-
-        return thread
 
     def shutdown(self):
-        for thread in self.threads.values():
+        """Close all CollectionWrappers managed by this object."""
-            thread.stop()
+        for path, col in self.collections.items():
-        self.threads = {}
+            del self.collections[path]
-
+            col.close()
-# TODO: There's got to be a way to do this without initializing it ALWAYS!
-thread_pool = CollectionThreadPool()
 

AnkiServer/threading.py

@@ -0,0 +1,185 @@
+
+from __future__ import absolute_import
+
+import anki
+import anki.storage
+
+from AnkiServer.collection import CollectionWrapper, CollectionManager
+
+from threading import Thread
+from Queue import Queue
+
+import time, logging
+
+__all__ = ['ThreadingCollectionWrapper', 'ThreadingCollectionManager']
+
+class ThreadingCollectionWrapper(object):
+    """Provides the same interface as CollectionWrapper, but it creates a new Thread to 
+    interact with the collection."""
+
+    def __init__(self, path, setup_new_collection=None):
+        self.path = path
+        self.wrapper = CollectionWrapper(path, setup_new_collection)
+
+        self._queue = Queue()
+        self._thread = None
+        self._running = False
+        self.last_timestamp = time.time()
+
+        self.start()
+
+    @property
+    def running(self):
+        return self._running
+
+    def qempty(self):
+        return self._queue.empty()
+
+    def current(self):
+        from threading import current_thread
+        return current_thread() == self._thread
+
+    def execute(self, func, args=[], kw={}, waitForReturn=True):
+        """ Executes a given function on this thread with the *args and **kw.
+
+        If 'waitForReturn' is True, then it will block until the function has
+        executed and return its return value.  If False, it will return None
+        immediately and the function will be executed sometime later.
+        """
+
+        if waitForReturn:
+            return_queue = Queue()
+        else:
+            return_queue = None
+
+        self._queue.put((func, args, kw, return_queue))
+
+        if return_queue is not None:
+            ret = return_queue.get(True)
+            if isinstance(ret, Exception):
+                raise ret
+            return ret
+
+    def _run(self):
+        logging.info('CollectionThread[%s]: Starting...', self.path)
+
+        try:
+            while self._running:
+                func, args, kw, return_queue = self._queue.get(True)
+
+                if hasattr(func, 'func_name'):
+                    func_name = func.func_name
+                else:
+                    func_name = func.__class__.__name__
+
+                logging.info('CollectionThread[%s]: Running %s(*%s, **%s)', self.path, func_name, repr(args), repr(kw))
+                self.last_timestamp = time.time()
+
+                try:
+                    ret = self.wrapper.execute(func, args, kw, return_queue)
+                except Exception, e:
+                    logging.error('CollectionThread[%s]: Unable to %s(*%s, **%s): %s',
+                        self.path, func.func_name, repr(args), repr(kw), e, exc_info=True)
+                    # we return the Exception which will be raise'd on the other end
+                    ret = e
+
+                if return_queue is not None:
+                    return_queue.put(ret)
+        except Exception, e:
+            logging.error('CollectionThread[%s]: Thread crashed! Exception: %s', self.path, e, exc_info=True)
+        finally:
+            self.wrapper.close()
+            # clean out old thread object
+            self._thread = None
+            # in case we got here via an exception
+            self._running = False
+
+            logging.info('CollectionThread[%s]: Stopped!', self.path)
+
+    def start(self):
+        if not self._running:
+            self._running = True
+            assert self._thread is None
+            self._thread = Thread(target=self._run)
+            self._thread.start()
+
+    def stop(self):
+        def _stop(col):
+            self._running = False
+        self.execute(_stop, waitForReturn=False)
+
+    def stop_and_wait(self):
+        """ Tell the thread to stop and wait for it to happen. """
+        self.stop()
+        if self._thread is not None:
+            self._thread.join()
+
+    # Mimic the CollectionWrapper interface
+    def open(self):
+        pass
+    def close(self):
+        self.stop()
+    def opened(self):
+        return self.wrapper.opened()
+
+class ThreadingCollectionManager(CollectionManager):
+    """Manages a set of ThreadingCollectionWrapper objects."""
+
+    collection_wrapper = ThreadingCollectionWrapper
+
+    def __init__(self):
+        super(ThreadingCollectionManager, self).__init__()
+
+        self.monitor_frequency = 15
+        self.monitor_inactivity = 90
+
+        monitor = Thread(target=self._monitor_run)
+        monitor.daemon = True
+        monitor.start()
+        self._monitor_thread = monitor
+
+    # TODO: we should raise some error if a collection is started on a manager that has already been shutdown!
+    #       or maybe we could support being restarted?
+
+    # TODO: it would be awesome to have a safe way to stop inactive threads completely!
+    # TODO: we need a way to inform other code that the collection has been closed
+    def _monitor_run(self):
+        """ Monitors threads for inactivity and closes the collection on them
+        (leaves the thread itself running -- hopefully waiting peacefully with only a
+        small memory footprint!) """
+        while True:
+            cur = time.time()
+            for path, thread in self.collections.items():
+                if thread.running and thread.wrapper.opened() and thread.qempty() and cur - thread.last_timestamp >= self.monitor_inactivity:
+                    logging.info('Monitor is closing collection on inactive CollectionThread[%s]', thread.path)
+                    def closeCollection(wrapper):
+                        wrapper.close()
+                    thread.execute(closeCollection, waitForReturn=False)
+            time.sleep(self.monitor_frequency)
+
+    def shutdown(self):
+        # TODO: stop the monitor thread!
+
+        # This will stop all the collection threads
+        super(ThreadingCollectionManager, self).shutdown()
+
+#
+# For working with the global ThreadingCollectionManager:
+#
+
+collection_manager = None
+
+def getCollectionManager():
+    """Return the global ThreadingCollectionManager for this process."""
+    global collection_manager
+    if collection_manager is None:
+        collection_manager = ThreadingCollectionManager()
+    return collection_manager
+
+def shutdown():
+    """If the global ThreadingCollectionManager exists, shut it down."""
+    global collection_manager
+    if collection_manager is not None:
+        collection_manager.shutdown()
+        collection_manager = None
+

INSTALL.txt

@@ -12,26 +12,30 @@ Instructions for installing and running AnkiServer:
     the dependencies we need there:
 
       $ virtualenv AnkiServer.env
-      $ AnkiServer.env/bin/easy_install webob PasteDeploy PasteScript sqlalchemy simplejson MySQL-python
+      $ AnkiServer.env/bin/easy_install webob PasteDeploy PasteScript sqlalchemy simplejson
 
  3. Download and install libanki.  You can find the latest release of Anki here:
 
     http://code.google.com/p/anki/downloads/list 
 
     Look for a *.tgz file with a Summary of "Anki Source".  At the time of this writing
-    that is anki-1.0.1.tgz.
+    that is anki-2.0.11.tgz.
 
-    Download this file and extract.  Inside you will find a libanki/ directory with a
+    Download this file and extract.
-    setup.py.  You will want to run setup.py with the python executable inside our virtualenv,
-    or AnkiServer.env/bin/python, like so:
     
-      anki-1.0.1/libanki$ ../../AnkiServer.env/bin/python setup.py install
+    Then either:
+
+      a. Run the 'make install', or
+
+      b. Copy the entire directory to /usr/share/anki
 
  4. Make the egg info files (so paster can see our app):
 
       $ AnkiServer.env/bin/python setup.py egg_info
 
- 5. Then we can run AnkiServer like so:
+ 5. Copy the example.ini to production.ini and edit for your needs.
+
+ 6. Then we can run AnkiServer like so:
 
       $ AnkiServer.env/bin/paster serve development.ini
 

example.ini

@@ -11,18 +11,18 @@ next = real
 
 [app:real]
 use = egg:Paste#urlmap
-/decks = deckapp
+/collection = rest_app
-/sync = syncapp
+/sync = sync_app
 
-[app:deckapp]
+[app:rest_app]
-use = egg:AnkiServer#deckapp
+use = egg:AnkiServer#rest_app
-data_root = ./decks
+data_root = ./collections
 allowed_hosts = 127.0.0.1
 logging.config_file = logging.conf
 
-[app:syncapp]
+[app:sync_app]
-use = egg:AnkiServer#syncapp
+use = egg:AnkiServer#sync_app
-data_root = ./decks
+data_root = ./collections
 base_url = /sync/
 mysql.host = 127.0.0.1
 mysql.user = db_user

setup.py

@@ -13,7 +13,8 @@ setup(
     test_suite='nose.collector',
     entry_points="""
     [paste.app_factory]
-    syncapp = AnkiServer.sync:make_app
+    sync_app = AnkiServer.apps.sync_app:make_app
+    rest_app = AnkiServer.apps.rest_app:make_app
 
     [paste.server_runner]
     server = AnkiServer:server_runner

tests/test_rest_app.py

@@ -0,0 +1,69 @@
+
+import os
+import shutil
+import tempfile
+import unittest
+
+import AnkiServer
+from AnkiServer.apps.rest_app import CollectionHandlerGroup, DeckHandlerGroup
+
+import anki
+import anki.storage
+
+class CollectionTestBase(unittest.TestCase):
+    """Parent class for tests that need a collection set up and torn down."""
+
+    def setUp(self):
+        self.temp_dir = tempfile.mkdtemp()
+        self.collection_path = os.path.join(self.temp_dir, 'collection.anki2');
+        self.collection = anki.storage.Collection(self.collection_path)
+
+    def tearDown(self):
+        self.collection.close()
+        self.collection = None
+        shutil.rmtree(self.temp_dir)
+
+class CollectionHandlerGroupTest(CollectionTestBase):
+    def setUp(self):
+        super(CollectionHandlerGroupTest, self).setUp()
+        self.handler = CollectionHandlerGroup()
+
+    def execute(self, name, data):
+        ids = ['collection_name']
+        func = getattr(self.handler, name)
+        return func(self.collection, data, ids)
+
+    def test_list_decks(self):
+        data = {}
+        ret = self.execute('list_decks', data)
+
+        # It contains only the 'Default' deck
+        self.assertEqual(len(ret), 1)
+        self.assertEqual(ret[0]['name'], 'Default')
+
+    def test_select_deck(self):
+        data = {'deck_id': '1'}
+        ret = self.execute('select_deck', data)
+        self.assertEqual(ret, None);
+
+class DeckHandlerGroupTest(CollectionTestBase):
+    def setUp(self):
+        super(DeckHandlerGroupTest, self).setUp()
+        self.handler = DeckHandlerGroup()
+
+    def execute(self, name, data):
+        ids = ['collection_name', '1']
+        func = getattr(self.handler, name)
+        return func(self.collection, data, ids)
+
+    def test_next_card(self):
+        ret = self.execute('next_card', {})
+        self.assertEqual(ret, None)
+
+        # TODO: add a note programatically
+
+
+
+if __name__ == '__main__':
+    unittest.main()
+