From f7ae69f4e66602d9a2a06d684ca981a6f6799bf4 Mon Sep 17 00:00:00 2001
From: Karsten Lehmann <mail@kalehmann.de>
Date: Wed, 26 Aug 2020 16:22:47 +0200
Subject: [PATCH 01/38] Removed embedded anki submodule

---
 .gitmodules               | 3 ---
 src/anki-bundled          | 1 -
 src/ankisyncd/__init__.py | 3 ---
 3 files changed, 7 deletions(-)
 delete mode 100644 .gitmodules
 delete mode 160000 src/anki-bundled

diff --git a/.gitmodules b/.gitmodules
deleted file mode 100644
index 9b79105..0000000
--- a/.gitmodules
+++ /dev/null
@@ -1,3 +0,0 @@
-[submodule "anki-bundled"]
-	path = src/anki-bundled
-	url = https://github.com/dae/anki.git
diff --git a/src/anki-bundled b/src/anki-bundled
deleted file mode 160000
index cca3fcb..0000000
--- a/src/anki-bundled
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit cca3fcb2418880d0430a5c5c2e6b81ba260065b7
diff --git a/src/ankisyncd/__init__.py b/src/ankisyncd/__init__.py
index 8ad83df..e691cc3 100644
--- a/src/ankisyncd/__init__.py
+++ b/src/ankisyncd/__init__.py
@@ -1,9 +1,6 @@
 import os
 import sys
 
-sys.path.insert(0, "/usr/share/anki")
-sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), "anki-bundled"))
-
 _homepage = "https://github.com/tsudoko/anki-sync-server"
 _unknown_version = "[unknown version]"
 

From 26d16b698a5e74d01f19167f17c18d0ee49c421e Mon Sep 17 00:00:00 2001
From: Karsten Lehmann <mail@kalehmann.de>
Date: Wed, 26 Aug 2020 16:25:03 +0200
Subject: [PATCH 02/38] Removed unused import

---
 src/ankisyncd/collection.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/ankisyncd/collection.py b/src/ankisyncd/collection.py
index e32dbfe..5127f2c 100644
--- a/src/ankisyncd/collection.py
+++ b/src/ankisyncd/collection.py
@@ -1,4 +1,3 @@
-import anki
 import anki.storage
 
 import ankisyncd.media

From d0f7d05b446789bcd11542d6472ae4f54fe23d3e Mon Sep 17 00:00:00 2001
From: Karsten Lehmann <mail@kalehmann.de>
Date: Wed, 26 Aug 2020 16:44:05 +0200
Subject: [PATCH 03/38] Ported Python Syncer from Anki source code

---
 src/ankisyncd/sync.py     | 869 ++++++++++++++++++++++++++++++++++++++
 src/ankisyncd/sync_app.py |  10 +-
 2 files changed, 874 insertions(+), 5 deletions(-)
 create mode 100644 src/ankisyncd/sync.py

diff --git a/src/ankisyncd/sync.py b/src/ankisyncd/sync.py
new file mode 100644
index 0000000..2d46950
--- /dev/null
+++ b/src/ankisyncd/sync.py
@@ -0,0 +1,869 @@
+# -*- coding: utf-8 -*-
+# Copyright: Ankitects Pty Ltd and contributors
+# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+# Taken from https://github.com/ankitects/anki/blob/cca3fcb2418880d0430a5c5c2e6b81ba260065b7/anki/sync.py
+
+import io
+import gzip
+import random
+import requests
+import json
+import os
+
+from anki.db import DB, DBError
+from anki.utils import ids2str, intTime, platDesc, checksum, devMode
+from anki.consts import *
+from anki.utils import versionWithBuild
+from anki.hooks import runHook
+import anki
+from anki.lang import ngettext
+
+
+# https://github.com/ankitects/anki/blob/04b1ca75599f18eb783a8bf0bdeeeb32362f4da0/rslib/src/sync/http_client.rs#L11
+SYNC_VER = 10
+# https://github.com/ankitects/anki/blob/cca3fcb2418880d0430a5c5c2e6b81ba260065b7/anki/consts.py#L50
+SYNC_ZIP_SIZE = int(2.5*1024*1024)
+# https://github.com/ankitects/anki/blob/cca3fcb2418880d0430a5c5c2e6b81ba260065b7/anki/consts.py#L51
+SYNC_ZIP_COUNT = 25
+
+# syncing vars
+HTTP_TIMEOUT = 90
+HTTP_PROXY = None
+HTTP_BUF_SIZE = 64*1024
+
+# Incremental syncing
+##########################################################################
+
+class Syncer(object):
+    def __init__(self, col, server=None):
+        self.col = col
+        self.server = server
+
+    def sync(self):
+        "Returns 'noChanges', 'fullSync', 'success', etc"
+        self.syncMsg = ""
+        self.uname = ""
+        # if the deck has any pending changes, flush them first and bump mod
+        # time
+        self.col.save()
+
+        # step 1: login & metadata
+        runHook("sync", "login")
+        meta = self.server.meta()
+        self.col.log("rmeta", meta)
+        if not meta:
+            return "badAuth"
+        # server requested abort?
+        self.syncMsg = meta['msg']
+        if not meta['cont']:
+            return "serverAbort"
+        else:
+            # don't abort, but if 'msg' is not blank, gui should show 'msg'
+            # after sync finishes and wait for confirmation before hiding
+            pass
+        rscm = meta['scm']
+        rts = meta['ts']
+        self.rmod = meta['mod']
+        self.maxUsn = meta['usn']
+        self.uname = meta.get("uname", "")
+        self.hostNum = meta.get("hostNum")
+        meta = self.meta()
+        self.col.log("lmeta", meta)
+        self.lmod = meta['mod']
+        self.minUsn = meta['usn']
+        lscm = meta['scm']
+        lts = meta['ts']
+        if abs(rts - lts) > 300:
+            self.col.log("clock off")
+            return "clockOff"
+        if self.lmod == self.rmod:
+            self.col.log("no changes")
+            return "noChanges"
+        elif lscm != rscm:
+            self.col.log("schema diff")
+            return "fullSync"
+        self.lnewer = self.lmod > self.rmod
+        # step 1.5: check collection is valid
+        if not self.col.basicCheck():
+            self.col.log("basic check")
+            return "basicCheckFailed"
+        # step 2: startup and deletions
+        runHook("sync", "meta")
+        rrem = self.server.start(minUsn=self.minUsn, lnewer=self.lnewer)
+
+        # apply deletions to server
+        lgraves = self.removed()
+        while lgraves:
+            gchunk, lgraves = self._gravesChunk(lgraves)
+            self.server.applyGraves(chunk=gchunk)
+
+        # then apply server deletions here
+        self.remove(rrem)
+
+        # ...and small objects
+        lchg = self.changes()
+        rchg = self.server.applyChanges(changes=lchg)
+        self.mergeChanges(lchg, rchg)
+        # step 3: stream large tables from server
+        runHook("sync", "server")
+        while 1:
+            runHook("sync", "stream")
+            chunk = self.server.chunk()
+            self.col.log("server chunk", chunk)
+            self.applyChunk(chunk=chunk)
+            if chunk['done']:
+                break
+        # step 4: stream to server
+        runHook("sync", "client")
+        while 1:
+            runHook("sync", "stream")
+            chunk = self.chunk()
+            self.col.log("client chunk", chunk)
+            self.server.applyChunk(chunk=chunk)
+            if chunk['done']:
+                break
+        # step 5: sanity check
+        runHook("sync", "sanity")
+        c = self.sanityCheck()
+        ret = self.server.sanityCheck2(client=c)
+        if ret['status'] != "ok":
+            # roll back and force full sync
+            self.col.rollback()
+            self.col.modSchema(False)
+            self.col.save()
+            return "sanityCheckFailed"
+        # finalize
+        runHook("sync", "finalize")
+        mod = self.server.finish()
+        self.finish(mod)
+        return "success"
+
+    def _gravesChunk(self, graves):
+        lim = 250
+        chunk = dict(notes=[], cards=[], decks=[])
+        for cat in "notes", "cards", "decks":
+            if lim and graves[cat]:
+                chunk[cat] = graves[cat][:lim]
+                graves[cat] = graves[cat][lim:]
+                lim -= len(chunk[cat])
+
+        # anything remaining?
+        if graves['notes'] or graves['cards'] or graves['decks']:
+            return chunk, graves
+        return chunk, None
+
+    def meta(self):
+        return dict(
+            mod=self.col.mod,
+            scm=self.col.scm,
+            usn=self.col._usn,
+            ts=intTime(),
+            musn=0,
+            msg="",
+            cont=True
+        )
+
+    def changes(self):
+        "Bundle up small objects."
+        d = dict(models=self.getModels(),
+                 decks=self.getDecks(),
+                 tags=self.getTags())
+        if self.lnewer:
+            #d['conf'] = self.getConf()
+            d['crt'] = self.col.crt
+        return d
+
+    def mergeChanges(self, lchg, rchg):
+        # then the other objects
+        self.mergeModels(rchg['models'])
+        self.mergeDecks(rchg['decks'])
+        self.mergeTags(rchg['tags'])
+        if 'conf' in rchg:
+            self.mergeConf(rchg['conf'])
+        # this was left out of earlier betas
+        if 'crt' in rchg:
+            self.col.crt = rchg['crt']
+        self.prepareToChunk()
+
+    def sanityCheck(self):
+        if not self.col.basicCheck():
+            return "failed basic check"
+        for t in "cards", "notes", "revlog", "graves":
+            if self.col.db.scalar(
+                "select count() from %s where usn = -1" % t):
+                return "%s had usn = -1" % t
+        for g in self.col.decks.all():
+            if g['usn'] == -1:
+                return "deck had usn = -1"
+        for t, usn in self.col.tags.allItems():
+            if usn == -1:
+                return "tag had usn = -1"
+        found = False
+        for m in self.col.models.all():
+            if m['usn'] == -1:
+                return "model had usn = -1"
+        if found:
+            self.col.models.save()
+#        self.col.sched.reset()
+        # check for missing parent decks
+        #self.col.sched.deckDueList()
+        # return summary of deck
+        return [
+            list(self.col.sched.counts()),
+            self.col.db.scalar("select count() from cards"),
+            self.col.db.scalar("select count() from notes"),
+            self.col.db.scalar("select count() from revlog"),
+            self.col.db.scalar("select count() from graves"),
+            len(self.col.models.all()),
+            len(self.col.decks.all()),
+            len(self.col.decks.allConf()),
+        ]
+
+    def usnLim(self):
+        return "usn = -1"
+
+    def finish(self, mod=None):
+        self.col.ls = mod
+        self.col._usn = self.maxUsn + 1
+        # ensure we save the mod time even if no changes made
+        self.col.db.mod = True
+        self.col.save(mod=mod)
+        return mod
+
+    # Chunked syncing
+    ##########################################################################
+
+    def prepareToChunk(self):
+        self.tablesLeft = ["revlog", "cards", "notes"]
+        self.cursor = None
+
+    def cursorForTable(self, table):
+        lim = self.usnLim()
+        with open("/dev/stdout", "w") as f:
+            f.write(str(type(self.col.db)))
+        x = self.col.db.execute
+        d = (self.maxUsn, lim)
+        if table == "revlog":
+            return x("""
+select id, cid, %d, ease, ivl, lastIvl, factor, time, type
+from revlog where %s""" % d)
+        elif table == "cards":
+            return x("""
+select id, nid, did, ord, mod, %d, type, queue, due, ivl, factor, reps,
+lapses, left, odue, odid, flags, data from cards where %s""" % d)
+        else:
+            return x("""
+select id, guid, mid, mod, %d, tags, flds, '', '', flags, data
+from notes where %s""" % d)
+
+    def chunk(self):
+        buf = dict(done=False)
+        while self.tablesLeft:
+            curTable = self.tablesLeft.pop()
+            if not self.cursor:
+                self.cursor = self.cursorForTable(curTable)
+            buf[curTable] = self.cursor 
+        if not self.tablesLeft:
+            buf['done'] = True
+        return buf
+
+    def applyChunk(self, chunk):
+        if "revlog" in chunk:
+            self.mergeRevlog(chunk['revlog'])
+        if "cards" in chunk:
+            self.mergeCards(chunk['cards'])
+        if "notes" in chunk:
+            self.mergeNotes(chunk['notes'])
+
+    # Deletions
+    ##########################################################################
+
+    def removed(self):
+        cards = []
+        notes = []
+        decks = []
+
+        curs = self.col.db.execute(
+            "select oid, type from graves where usn = -1")
+
+        for oid, type in curs:
+            if type == REM_CARD:
+                cards.append(oid)
+            elif type == REM_NOTE:
+                notes.append(oid)
+            else:
+                decks.append(oid)
+
+        self.col.db.execute("update graves set usn=? where usn=-1",
+                             self.maxUsn)
+
+        return dict(cards=cards, notes=notes, decks=decks)
+
+    def remove(self, graves):
+        # pretend to be the server so we don't set usn = -1
+        self.col.server = True
+
+        # notes first, so we don't end up with duplicate graves
+        self.col._remNotes(graves['notes'])
+        # then cards
+        self.col.remCards(graves['cards'], notes=False)
+        # and decks
+        for oid in graves['decks']:
+            self.col.decks.rem(oid, childrenToo=False)
+
+        self.col.server = False
+
+    # Models
+    ##########################################################################
+
+    def getModels(self):
+        mods = [m for m in self.col.models.all() if m['usn'] == -1]
+        for m in mods:
+            m['usn'] = self.maxUsn
+        self.col.models.save()
+        return mods
+
+    def mergeModels(self, rchg):
+        for r in rchg:
+            l = self.col.models.get(r['id'])
+            # if missing locally or server is newer, update
+            if not l or r['mod'] > l['mod']:
+                self.col.models.update(r)
+
+    # Decks
+    ##########################################################################
+
+    def getDecks(self):
+        decks = [g for g in self.col.decks.all() if g['usn'] == -1]
+        for g in decks:
+            g['usn'] = self.maxUsn
+        dconf = [g for g in self.col.decks.allConf() if g['usn'] == -1]
+        for g in dconf:
+            g['usn'] = self.maxUsn
+        self.col.decks.save()
+        return [decks, dconf]
+
+    def mergeDecks(self, rchg):
+        for r in rchg[0]:
+            l = self.col.decks.get(r['id'], False)
+            # work around mod time being stored as string
+            if l and not isinstance(l['mod'], int):
+                l['mod'] = int(l['mod'])
+
+            # if missing locally or server is newer, update
+            if not l or r['mod'] > l['mod']:
+                self.col.decks.update(r)
+        for r in rchg[1]:
+            try:
+                l = self.col.decks.getConf(r['id'])
+            except KeyError:
+                l = None
+            # if missing locally or server is newer, update
+            if not l or r['mod'] > l['mod']:
+                self.col.decks.updateConf(r)
+
+    # Tags
+    ##########################################################################
+
+    def getTags(self):
+        tags = []
+        for t, usn in self.col.tags.allItems():
+            if usn == -1:
+                self.col.tags.tags[t] = self.maxUsn
+                tags.append(t)
+        self.col.tags.save()
+        return tags
+
+    def mergeTags(self, tags):
+        self.col.tags.register(tags, usn=self.maxUsn)
+
+    # Cards/notes/revlog
+    ##########################################################################
+
+    def mergeRevlog(self, logs):
+        self.col.db.executemany(
+            "insert or ignore into revlog values (?,?,?,?,?,?,?,?,?)",
+            logs)
+
+    def newerRows(self, data, table, modIdx):
+        ids = (r[0] for r in data)
+        lmods = {}
+        for id, mod in self.col.db.execute(
+            "select id, mod from %s where id in %s and %s" % (
+                table, ids2str(ids), self.usnLim())):
+            lmods[id] = mod
+        update = []
+        for r in data:
+            if r[0] not in lmods or lmods[r[0]] < r[modIdx]:
+                update.append(r)
+        self.col.log(table, data)
+        return update
+
+    def mergeCards(self, cards):
+        self.col.db.executemany(
+            "insert or replace into cards values "
+            "(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
+            self.newerRows(cards, "cards", 4))
+
+    def mergeNotes(self, notes):
+        rows = self.newerRows(notes, "notes", 3)
+        self.col.db.executemany(
+            "insert or replace into notes values (?,?,?,?,?,?,?,?,?,?,?)",
+            rows)
+        self.col.updateFieldCache([f[0] for f in rows])
+
+    # Col config
+    ##########################################################################
+
+    def getConf(self):
+        return self.col.conf
+
+    def mergeConf(self, conf):
+        self.col.conf = conf
+
+# Wrapper for requests that tracks upload/download progress
+##########################################################################
+
+class AnkiRequestsClient(object):
+    verify = True
+    timeout = 60
+
+    def __init__(self):
+        self.session = requests.Session()
+
+    def post(self, url, data, headers):
+        data = _MonitoringFile(data)
+        headers['User-Agent'] = self._agentName()
+        return self.session.post(
+            url, data=data, headers=headers, stream=True, timeout=self.timeout, verify=self.verify)
+
+    def get(self, url, headers=None):
+        if headers is None:
+            headers = {}
+        headers['User-Agent'] = self._agentName()
+        return self.session.get(url, stream=True, headers=headers, timeout=self.timeout, verify=self.verify)
+
+    def streamContent(self, resp):
+        resp.raise_for_status()
+
+        buf = io.BytesIO()
+        for chunk in resp.iter_content(chunk_size=HTTP_BUF_SIZE):
+            runHook("httpRecv", len(chunk))
+            buf.write(chunk)
+        return buf.getvalue()
+
+    def _agentName(self):
+        from anki import version
+        return "Anki {}".format(version)
+
+# allow user to accept invalid certs in work/school settings
+if os.environ.get("ANKI_NOVERIFYSSL"):
+    AnkiRequestsClient.verify = False
+
+    import warnings
+    warnings.filterwarnings("ignore")
+
+class _MonitoringFile(io.BufferedReader):
+    def read(self, size=-1):
+        data = io.BufferedReader.read(self, HTTP_BUF_SIZE)
+        runHook("httpSend", len(data))
+        return data
+
+# HTTP syncing tools
+##########################################################################
+
+class HttpSyncer(object):
+    def __init__(self, hkey=None, client=None, hostNum=None):
+        self.hkey = hkey
+        self.skey = checksum(str(random.random()))[:8]
+        self.client = client or AnkiRequestsClient()
+        self.postVars = {}
+        self.hostNum = hostNum
+        self.prefix = "sync/"
+
+    def syncURL(self):
+        if devMode:
+            url = "https://l1sync.ankiweb.net/"
+        else:
+            url = SYNC_BASE % (self.hostNum or "")
+        return url + self.prefix
+
+    def assertOk(self, resp):
+        # not using raise_for_status() as aqt expects this error msg
+        if resp.status_code != 200:
+            raise Exception("Unknown response code: %s" % resp.status_code)
+
+    # Posting data as a file
+    ######################################################################
+    # We don't want to post the payload as a form var, as the percent-encoding is
+    # costly. We could send it as a raw post, but more HTTP clients seem to
+    # support file uploading, so this is the more compatible choice.
+
+    def _buildPostData(self, fobj, comp):
+        BOUNDARY=b"Anki-sync-boundary"
+        bdry = b"--"+BOUNDARY
+        buf = io.BytesIO()
+        # post vars
+        self.postVars['c'] = 1 if comp else 0
+        for (key, value) in list(self.postVars.items()):
+            buf.write(bdry + b"\r\n")
+            buf.write(
+                ('Content-Disposition: form-data; name="%s"\r\n\r\n%s\r\n' %
+                (key, value)).encode("utf8"))
+        # payload as raw data or json
+        rawSize = 0
+        if fobj:
+            # header
+            buf.write(bdry + b"\r\n")
+            buf.write(b"""\
+Content-Disposition: form-data; name="data"; filename="data"\r\n\
+Content-Type: application/octet-stream\r\n\r\n""")
+            # write file into buffer, optionally compressing
+            if comp:
+                tgt = gzip.GzipFile(mode="wb", fileobj=buf, compresslevel=comp)
+            else:
+                tgt = buf
+            while 1:
+                data = fobj.read(65536)
+                if not data:
+                    if comp:
+                        tgt.close()
+                    break
+                rawSize += len(data)
+                tgt.write(data)
+            buf.write(b"\r\n")
+        buf.write(bdry + b'--\r\n')
+        size = buf.tell()
+        # connection headers
+        headers = {
+            'Content-Type': 'multipart/form-data; boundary=%s' % BOUNDARY.decode("utf8"),
+            'Content-Length': str(size),
+        }
+        buf.seek(0)
+
+        if size >= 100*1024*1024 or rawSize >= 250*1024*1024:
+            raise Exception("Collection too large to upload to AnkiWeb.")
+
+        return headers, buf
+
+    def req(self, method, fobj=None, comp=6, badAuthRaises=True):
+        headers, body = self._buildPostData(fobj, comp)
+
+        r = self.client.post(self.syncURL()+method, data=body, headers=headers)
+        if not badAuthRaises and r.status_code == 403:
+            return False
+        self.assertOk(r)
+
+        buf = self.client.streamContent(r)
+        return buf
+
+# Incremental sync over HTTP
+######################################################################
+
+class RemoteServer(HttpSyncer):
+    def __init__(self, hkey, hostNum):
+        super().__init__(self, hkey, hostNum=hostNum)
+
+    def hostKey(self, user, pw):
+        "Returns hkey or none if user/pw incorrect."
+        self.postVars = dict()
+        ret = self.req(
+            "hostKey", io.BytesIO(json.dumps(dict(u=user, p=pw)).encode("utf8")),
+            badAuthRaises=False)
+        if not ret:
+            # invalid auth
+            return
+        self.hkey = json.loads(ret.decode("utf8"))['key']
+        return self.hkey
+
+    def meta(self):
+        self.postVars = dict(
+            k=self.hkey,
+            s=self.skey,
+        )
+        ret = self.req(
+            "meta", io.BytesIO(json.dumps(dict(
+                v=SYNC_VER, cv="ankidesktop,%s,%s"%(versionWithBuild(), platDesc()))).encode("utf8")),
+            badAuthRaises=False)
+        if not ret:
+            # invalid auth
+            return
+        return json.loads(ret.decode("utf8"))
+
+    def applyGraves(self, **kw):
+        return self._run("applyGraves", kw)
+
+    def applyChanges(self, **kw):
+        return self._run("applyChanges", kw)
+
+    def start(self, **kw):
+        return self._run("start", kw)
+
+    def chunk(self, **kw):
+        return self._run("chunk", kw)
+
+    def applyChunk(self, **kw):
+        return self._run("applyChunk", kw)
+
+    def sanityCheck2(self, **kw):
+        return self._run("sanityCheck2", kw)
+
+    def finish(self, **kw):
+        return self._run("finish", kw)
+
+    def abort(self, **kw):
+        return self._run("abort", kw)
+
+    def _run(self, cmd, data):
+        return json.loads(
+            self.req(cmd, io.BytesIO(json.dumps(data).encode("utf8"))).decode("utf8"))
+
+# Full syncing
+##########################################################################
+
+class FullSyncer(HttpSyncer):
+    def __init__(self, col, hkey, client, hostNum):
+        super().__init__(self, hkey, client, hostNum=hostNum)
+        self.postVars = dict(
+            k=self.hkey,
+            v="ankidesktop,%s,%s"%(anki.version, platDesc()),
+        )
+        self.col = col
+
+    def download(self):
+        runHook("sync", "download")
+        localNotEmpty = self.col.db.scalar("select 1 from cards")
+        self.col.close()
+        cont = self.req("download")
+        tpath = self.col.path + ".tmp"
+        if cont == "upgradeRequired":
+            runHook("sync", "upgradeRequired")
+            return
+        open(tpath, "wb").write(cont)
+        # check the received file is ok
+        d = DB(tpath)
+        assert d.scalar("pragma integrity_check") == "ok"
+        remoteEmpty = not d.scalar("select 1 from cards")
+        d.close()
+        # accidental clobber?
+        if localNotEmpty and remoteEmpty:
+            os.unlink(tpath)
+            return "downloadClobber"
+        # overwrite existing collection
+        os.unlink(self.col.path)
+        os.rename(tpath, self.col.path)
+        self.col = None
+
+    def upload(self):
+        "True if upload successful."
+        runHook("sync", "upload")
+        # make sure it's ok before we try to upload
+        if self.col.db.scalar("pragma integrity_check") != "ok":
+            return False
+        if not self.col.basicCheck():
+            return False
+        # apply some adjustments, then upload
+        self.col.beforeUpload()
+        if self.req("upload", open(self.col.path, "rb")) != b"OK":
+            return False
+        return True
+
+# Media syncing
+##########################################################################
+#
+# About conflicts:
+# - to minimize data loss, if both sides are marked for sending and one
+#   side has been deleted, favour the add
+# - if added/changed on both sides, favour the server version on the
+#   assumption other syncers are in sync with the server
+#
+
+class MediaSyncer(object):
+    def __init__(self, col, server=None):
+        self.col = col
+        self.server = server
+
+    def sync(self):
+        # check if there have been any changes
+        runHook("sync", "findMedia")
+        self.col.log("findChanges")
+        try:
+            self.col.media.findChanges()
+        except DBError:
+            return "corruptMediaDB"
+
+        # begin session and check if in sync
+        lastUsn = self.col.media.lastUsn()
+        ret = self.server.begin()
+        srvUsn = ret['usn']
+        if lastUsn == srvUsn and not self.col.media.haveDirty():
+            return "noChanges"
+
+        # loop through and process changes from server
+        self.col.log("last local usn is %s"%lastUsn)
+        self.downloadCount = 0
+        while True:
+            data = self.server.mediaChanges(lastUsn=lastUsn)
+
+            self.col.log("mediaChanges resp count %d"%len(data))
+            if not data:
+                break
+
+            need = []
+            lastUsn = data[-1][1]
+            for fname, rusn, rsum in data:
+                lsum, ldirty = self.col.media.syncInfo(fname)
+                self.col.log(
+                    "check: lsum=%s rsum=%s ldirty=%d rusn=%d fname=%s"%(
+                        (lsum and lsum[0:4]),
+                        (rsum and rsum[0:4]),
+                        ldirty,
+                        rusn,
+                        fname))
+
+                if rsum:
+                    # added/changed remotely
+                    if not lsum or lsum != rsum:
+                        self.col.log("will fetch")
+                        need.append(fname)
+                    else:
+                        self.col.log("have same already")
+                    if ldirty:
+                        self.col.media.markClean([fname])
+                elif lsum:
+                    # deleted remotely
+                    if not ldirty:
+                        self.col.log("delete local")
+                        self.col.media.syncDelete(fname)
+                    else:
+                        # conflict; local add overrides remote delete
+                        self.col.log("conflict; will send")
+                else:
+                    # deleted both sides
+                    self.col.log("both sides deleted")
+                    if ldirty:
+                        self.col.media.markClean([fname])
+
+            self._downloadFiles(need)
+
+            self.col.log("update last usn to %d"%lastUsn)
+            self.col.media.setLastUsn(lastUsn) # commits
+
+        # at this point we're all up to date with the server's changes,
+        # and we need to send our own
+
+        updateConflict = False
+        toSend = self.col.media.dirtyCount()
+        while True:
+            zip, fnames = self.col.media.mediaChangesZip()
+            if not fnames:
+                break
+
+            runHook("syncMsg", ngettext(
+                "%d media change to upload", "%d media changes to upload", toSend)
+                    % toSend)
+
+            processedCnt, serverLastUsn = self.server.uploadChanges(zip)
+            self.col.media.markClean(fnames[0:processedCnt])
+
+            self.col.log("processed %d, serverUsn %d, clientUsn %d" % (
+                processedCnt, serverLastUsn, lastUsn
+            ))
+
+            if serverLastUsn - processedCnt == lastUsn:
+                self.col.log("lastUsn in sync, updating local")
+                lastUsn = serverLastUsn
+                self.col.media.setLastUsn(serverLastUsn) # commits
+            else:
+                self.col.log("concurrent update, skipping usn update")
+                # commit for markClean
+                self.col.media.db.commit()
+                updateConflict = True
+
+            toSend -= processedCnt
+
+        if updateConflict:
+            self.col.log("restart sync due to concurrent update")
+            return self.sync()
+
+        lcnt = self.col.media.mediaCount()
+        ret = self.server.mediaSanity(local=lcnt)
+        if ret == "OK":
+            return "OK"
+        else:
+            self.col.media.forceResync()
+            return ret
+
+    def _downloadFiles(self, fnames):
+        self.col.log("%d files to fetch"%len(fnames))
+        while fnames:
+            top = fnames[0:SYNC_ZIP_COUNT]
+            self.col.log("fetch %s"%top)
+            zipData = self.server.downloadFiles(files=top)
+            cnt = self.col.media.addFilesFromZip(zipData)
+            self.downloadCount += cnt
+            self.col.log("received %d files"%cnt)
+            fnames = fnames[cnt:]
+
+            n = self.downloadCount
+            runHook("syncMsg", ngettext(
+                "%d media file downloaded", "%d media files downloaded", n)
+                    % n)
+
+# Remote media syncing
+##########################################################################
+
+class RemoteMediaServer(HttpSyncer):
+    def __init__(self, col, hkey, client, hostNum):
+        self.col = col
+        super().__init__(self, hkey, client, hostNum=hostNum)
+        self.prefix = "msync/"
+
+    def begin(self):
+        self.postVars = dict(
+            k=self.hkey,
+            v="ankidesktop,%s,%s"%(anki.version, platDesc())
+        )
+        ret = self._dataOnly(self.req(
+            "begin", io.BytesIO(json.dumps(dict()).encode("utf8"))))
+        self.skey = ret['sk']
+        return ret
+
+    # args: lastUsn
+    def mediaChanges(self, **kw):
+        self.postVars = dict(
+            sk=self.skey,
+        )
+        return self._dataOnly(
+            self.req("mediaChanges", io.BytesIO(json.dumps(kw).encode("utf8"))))
+
+    # args: files
+    def downloadFiles(self, **kw):
+        return self.req("downloadFiles", io.BytesIO(json.dumps(kw).encode("utf8")))
+
+    def uploadChanges(self, zip):
+        # no compression, as we compress the zip file instead
+        return self._dataOnly(
+            self.req("uploadChanges", io.BytesIO(zip), comp=0))
+
+    # args: local
+    def mediaSanity(self, **kw):
+        return self._dataOnly(
+            self.req("mediaSanity", io.BytesIO(json.dumps(kw).encode("utf8"))))
+
+    def _dataOnly(self, resp):
+        resp = json.loads(resp.decode("utf8"))
+        if resp['err']:
+            self.col.log("error returned:%s"%resp['err'])
+            raise Exception("SyncError:%s"%resp['err'])
+        return resp['data']
+
+    # only for unit tests
+    def mediatest(self, cmd):
+        self.postVars = dict(
+            k=self.hkey,
+        )
+        return self._dataOnly(
+            self.req("newMediaTest", io.BytesIO(
+                json.dumps(dict(cmd=cmd)).encode("utf8"))))
diff --git a/src/ankisyncd/sync_app.py b/src/ankisyncd/sync_app.py
index 7cce5ae..e452898 100644
--- a/src/ankisyncd/sync_app.py
+++ b/src/ankisyncd/sync_app.py
@@ -35,24 +35,24 @@ from webob.dec import wsgify
 from webob.exc import *
 
 import anki.db
-import anki.sync
 import anki.utils
-from anki.consts import SYNC_VER, SYNC_ZIP_SIZE, SYNC_ZIP_COUNT
 from anki.consts import REM_CARD, REM_NOTE
 
 from ankisyncd.users import get_user_manager
 from ankisyncd.sessions import get_session_manager
 from ankisyncd.full_sync import get_full_sync_manager
 
+from .sync import Syncer, SYNC_VER, SYNC_ZIP_SIZE, SYNC_ZIP_COUNT
+
 logger = logging.getLogger("ankisyncd")
 
 
-class SyncCollectionHandler(anki.sync.Syncer):
+class SyncCollectionHandler(Syncer):
     operations = ['meta', 'applyChanges', 'start', 'applyGraves', 'chunk', 'applyChunk', 'sanityCheck2', 'finish']
 
     def __init__(self, col):
         # So that 'server' (the 3rd argument) can't get set
-        anki.sync.Syncer.__init__(self, col)
+        super().__init__(self, col)
 
     @staticmethod
     def _old_client(cv):
@@ -137,7 +137,7 @@ class SyncCollectionHandler(anki.sync.Syncer):
         return dict(status="ok")
 
     def finish(self, mod=None):
-        return anki.sync.Syncer.finish(self, anki.utils.intTime(1000))
+        return super().finish(self, anki.utils.intTime(1000))
 
     # This function had to be put here in its entirety because Syncer.removed()
     # doesn't use self.usnLim() (which we override in this class) in queries.

From bc889958dc443051a0bc6454a4dd8d8d4e85d1b3 Mon Sep 17 00:00:00 2001
From: Karsten Lehmann <mail@kalehmann.de>
Date: Wed, 26 Aug 2020 16:51:06 +0200
Subject: [PATCH 04/38] Added missing fields to meta endpoint

---
 src/ankisyncd/sync_app.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/src/ankisyncd/sync_app.py b/src/ankisyncd/sync_app.py
index e452898..9a41b0e 100644
--- a/src/ankisyncd/sync_app.py
+++ b/src/ankisyncd/sync_app.py
@@ -50,9 +50,10 @@ logger = logging.getLogger("ankisyncd")
 class SyncCollectionHandler(Syncer):
     operations = ['meta', 'applyChanges', 'start', 'applyGraves', 'chunk', 'applyChunk', 'sanityCheck2', 'finish']
 
-    def __init__(self, col):
+    def __init__(self, col, session):
         # So that 'server' (the 3rd argument) can't get set
         super().__init__(self, col)
+        self.session = session
 
     @staticmethod
     def _old_client(cv):
@@ -96,13 +97,15 @@ class SyncCollectionHandler(Syncer):
             self.col.media.connect()
 
         return {
-            'scm': self.col.scm,
-            'ts': anki.utils.intTime(),
             'mod': self.col.mod,
+            'scm': self.col.scm,
             'usn': self.col._usn,
+            'ts': anki.utils.intTime(),
             'musn': self.col.media.lastUsn(),
+            'uname': self.session.name,
             'msg': '',
             'cont': True,
+            'hostNum': 0,
         }
 
     def usnLim(self):
@@ -176,8 +179,9 @@ class SyncCollectionHandler(Syncer):
 class SyncMediaHandler:
     operations = ['begin', 'mediaChanges', 'mediaSanity', 'uploadChanges', 'downloadFiles']
 
-    def __init__(self, col):
+    def __init__(self, col, session):
         self.col = col
+        self.session = session
 
     def begin(self, skey):
         return {
@@ -376,7 +380,7 @@ class SyncUserSession:
             raise Exception("no handler for {}".format(operation))
 
         if getattr(self, attr) is None:
-            setattr(self, attr, handler_class(col))
+            setattr(self, attr, handler_class(col, self))
         handler = getattr(self, attr)
         # The col object may actually be new now! This happens when we close a collection
         # for inactivity and then later re-open it (creating a new Collection object).

From 3857f15c0614925a32bcfd657977d285b1dd389e Mon Sep 17 00:00:00 2001
From: Karsten Lehmann <mail@kalehmann.de>
Date: Wed, 26 Aug 2020 16:58:50 +0200
Subject: [PATCH 05/38] Read hostkey from GET or POST

This commit applies the fix from
https://github.com/tsudoko/anki-sync-server/pull/60/files

However it using a shorter version by utilizing the params attribute of
the webob request. The params attribute combines the get and post params
---
 src/ankisyncd/sync_app.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ankisyncd/sync_app.py b/src/ankisyncd/sync_app.py
index 9a41b0e..0baa830 100644
--- a/src/ankisyncd/sync_app.py
+++ b/src/ankisyncd/sync_app.py
@@ -499,7 +499,7 @@ class SyncApp:
     def __call__(self, req):
         # Get and verify the session
         try:
-            hkey = req.POST['k']
+            hkey = req.params['k']
         except KeyError:
             hkey = None
 

From b566e3259758857e74b137eb7b115af2a2523c4b Mon Sep 17 00:00:00 2001
From: Karsten Lehmann <mail@kalehmann.de>
Date: Wed, 26 Aug 2020 18:27:05 +0200
Subject: [PATCH 06/38] Removed call to load method of
 anki.collection.Collection

This method was removed in
a2b7a3084131f747fb476cc8a24f96a00c654859
---
 src/ankisyncd/full_sync.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/ankisyncd/full_sync.py b/src/ankisyncd/full_sync.py
index 9044abd..5d6a5cc 100644
--- a/src/ankisyncd/full_sync.py
+++ b/src/ankisyncd/full_sync.py
@@ -2,6 +2,7 @@
 
 import os
 from sqlite3 import dbapi2 as sqlite
+import sys
 
 import anki.db
 
@@ -28,7 +29,6 @@ class FullSyncManager:
             os.replace(temp_db_path, session.get_collection_path())
         finally:
             col.reopen()
-            col.load()
 
         return "OK"
 
@@ -39,7 +39,6 @@ class FullSyncManager:
             data = open(session.get_collection_path(), 'rb').read()
         finally:
             col.reopen()
-            col.load()
         return data
 
 

From 8358b092a3b119585286dfabfcd2ae0195cffa45 Mon Sep 17 00:00:00 2001
From: Karsten Lehmann <mail@kalehmann.de>
Date: Wed, 26 Aug 2020 21:06:57 +0200
Subject: [PATCH 07/38] Hide the media managers db

---
 src/ankisyncd/full_sync.py |  2 ++
 src/ankisyncd/media.py     | 36 +++++++++++++++++++++++-------------
 src/ankisyncd/sessions.py  |  2 +-
 src/ankisyncd/sync_app.py  | 10 +++-------
 4 files changed, 29 insertions(+), 21 deletions(-)

diff --git a/src/ankisyncd/full_sync.py b/src/ankisyncd/full_sync.py
index 5d6a5cc..dd26d68 100644
--- a/src/ankisyncd/full_sync.py
+++ b/src/ankisyncd/full_sync.py
@@ -29,6 +29,8 @@ class FullSyncManager:
             os.replace(temp_db_path, session.get_collection_path())
         finally:
             col.reopen()
+            # Reopen the media database
+            col.media.connect()
 
         return "OK"
 
diff --git a/src/ankisyncd/media.py b/src/ankisyncd/media.py
index 9c68c4c..c6627c5 100644
--- a/src/ankisyncd/media.py
+++ b/src/ankisyncd/media.py
@@ -11,18 +11,27 @@ import anki.db
 
 logger = logging.getLogger("ankisyncd.media")
 
-
-class ServerMediaManager:
+class ServerMediaManager(object):
     def __init__(self, col):
         self._dir = re.sub(r"(?i)\.(anki2)$", ".media", col.path)
         self.connect()
 
+    def addMedia(self, media_to_add):
+        self._db.executemany(
+            "INSERT OR REPLACE INTO media VALUES (?,?,?)",
+            media_to_add
+        )
+        self._db.commit()
+
+    def changes(self, lastUsn):
+        return self._db.execute("select fname,usn,csum from media order by usn desc limit ?", self.lastUsn() - lastUsn)
+        
     def connect(self):
         path = self.dir() + ".server.db"
         create = not os.path.exists(path)
-        self.db = anki.db.DB(path)
+        self._db = anki.db.DB(path)
         if create:
-            self.db.executescript(
+            self._db.executescript(
                 """CREATE TABLE media (
                        fname TEXT NOT NULL PRIMARY KEY,
                        usn INT NOT NULL,
@@ -33,35 +42,36 @@ class ServerMediaManager:
             oldpath = self.dir() + ".db2"
             if os.path.exists(oldpath):
                 logger.info("Found client media database, migrating contents")
-                self.db.execute("ATTACH ? AS old", oldpath)
-                self.db.execute(
+                self._db.execute("ATTACH ? AS old", oldpath)
+                self._db.execute(
                     "INSERT INTO media SELECT fname, lastUsn, csum FROM old.media, old.meta"
                 )
-                self.db.commit()
-                self.db.execute("DETACH old")
+                self._db.commit()
+                self._db.execute("DETACH old")
 
     def close(self):
-        self.db.close()
+        self._db.close()
 
     def dir(self):
         return self._dir
 
     def lastUsn(self):
-        return self.db.scalar("SELECT max(usn) FROM media") or 0
+        return self._db.scalar("SELECT max(usn) FROM media") or 0
 
     def mediaCount(self):
-        return self.db.scalar("SELECT count() FROM media WHERE csum IS NOT NULL")
+        return self._db.scalar("SELECT count() FROM media WHERE csum IS NOT NULL")
 
     # used only in unit tests
     def syncInfo(self, fname):
-        return self.db.first("SELECT csum, 0 FROM media WHERE fname=?", fname)
+        return self._db.first("SELECT csum, 0 FROM media WHERE fname=?", fname)
 
     def syncDelete(self, fname):
         fpath = os.path.join(self.dir(), fname)
         if os.path.exists(fpath):
             os.remove(fpath)
-        self.db.execute(
+        self._db.execute(
             "UPDATE media SET csum = NULL, usn = ? WHERE fname = ?",
             self.lastUsn() + 1,
             fname,
         )
+        self._db.commit()
diff --git a/src/ankisyncd/sessions.py b/src/ankisyncd/sessions.py
index 2e09ab6..7c609db 100644
--- a/src/ankisyncd/sessions.py
+++ b/src/ankisyncd/sessions.py
@@ -32,7 +32,7 @@ class SqliteSessionManager(SimpleSessionManager):
     everytime the SyncApp is restarted."""
 
     def __init__(self, session_db_path):
-        SimpleSessionManager.__init__(self)
+        super().__init__()
 
         self.session_db_path = os.path.realpath(session_db_path)
         self._ensure_schema_up_to_date()
diff --git a/src/ankisyncd/sync_app.py b/src/ankisyncd/sync_app.py
index 0baa830..fad24e4 100644
--- a/src/ankisyncd/sync_app.py
+++ b/src/ankisyncd/sync_app.py
@@ -93,8 +93,7 @@ class SyncCollectionHandler(Syncer):
             return {"cont": False, "msg": "Your client doesn't support the v{} scheduler.".format(self.col.schedVer())}
 
         # Make sure the media database is open!
-        if self.col.media.db is None:
-            self.col.media.connect()
+        self.col.media.connect()
 
         return {
             'mod': self.col.mod,
@@ -267,9 +266,7 @@ class SyncMediaHandler:
             self._remove_media_files(media_to_remove)
 
         if media_to_add:
-            self.col.media.db.executemany(
-                "INSERT OR REPLACE INTO media VALUES (?,?,?)", media_to_add)
-            self.col.media.db.commit()
+            self.col.media.addMedia(media_to_add)
 
         assert self.col.media.lastUsn() == oldUsn + processed_count  # TODO: move to some unit test
         return processed_count
@@ -325,10 +322,9 @@ class SyncMediaHandler:
     def mediaChanges(self, lastUsn):
         result = []
         server_lastUsn = self.col.media.lastUsn()
-        fname = csum = None
 
         if lastUsn < server_lastUsn or lastUsn == 0:
-            for fname,usn,csum, in self.col.media.db.execute("select fname,usn,csum from media order by usn desc limit ?", server_lastUsn - lastUsn):
+            for fname,usn,csum, in self.col.media.changes(lastUsn):
                 result.append([fname, usn, csum])
 
         # anki assumes server_lastUsn == result[-1][1]

From c97a096e8a36809aa3a7daab33e85d217a33bb29 Mon Sep 17 00:00:00 2001
From: Karsten Lehmann <mail@kalehmann.de>
Date: Fri, 28 Aug 2020 00:01:36 +0200
Subject: [PATCH 08/38] Made sure to use ConfigManager in Syncer

---
 src/ankisyncd/sync.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/ankisyncd/sync.py b/src/ankisyncd/sync.py
index 2d46950..900973f 100644
--- a/src/ankisyncd/sync.py
+++ b/src/ankisyncd/sync.py
@@ -14,6 +14,7 @@ import os
 from anki.db import DB, DBError
 from anki.utils import ids2str, intTime, platDesc, checksum, devMode
 from anki.consts import *
+from anki.config import ConfigManager
 from anki.utils import versionWithBuild
 from anki.hooks import runHook
 import anki
@@ -205,7 +206,7 @@ class Syncer(object):
                 return "model had usn = -1"
         if found:
             self.col.models.save()
-#        self.col.sched.reset()
+        self.col.sched.reset()
         # check for missing parent decks
         #self.col.sched.deckDueList()
         # return summary of deck
@@ -420,7 +421,10 @@ from notes where %s""" % d)
         return self.col.conf
 
     def mergeConf(self, conf):
-        self.col.conf = conf
+        newConf = ConfigManager(self.col)
+        for key, value in conf.items():
+            newConf[key] = value
+        self.col.conf = newConf
 
 # Wrapper for requests that tracks upload/download progress
 ##########################################################################

From 93d37d6ab6ee7ebbd98ed665889334dcf317c09d Mon Sep 17 00:00:00 2001
From: Karsten Lehmann <mail@kalehmann.de>
Date: Fri, 28 Aug 2020 00:02:10 +0200
Subject: [PATCH 09/38] fix chunk in sync

---
 src/ankisyncd/sync.py | 28 +++++++++++-----------------
 1 file changed, 11 insertions(+), 17 deletions(-)

diff --git a/src/ankisyncd/sync.py b/src/ankisyncd/sync.py
index 900973f..8cd3c3f 100644
--- a/src/ankisyncd/sync.py
+++ b/src/ankisyncd/sync.py
@@ -239,32 +239,26 @@ class Syncer(object):
         self.tablesLeft = ["revlog", "cards", "notes"]
         self.cursor = None
 
-    def cursorForTable(self, table):
+    def queryTable(self, table):
         lim = self.usnLim()
-        with open("/dev/stdout", "w") as f:
-            f.write(str(type(self.col.db)))
-        x = self.col.db.execute
-        d = (self.maxUsn, lim)
         if table == "revlog":
-            return x("""
-select id, cid, %d, ease, ivl, lastIvl, factor, time, type
-from revlog where %s""" % d)
+            return self.col.db.execute("""
+select id, cid, ?, ease, ivl, lastIvl, factor, time, type
+from revlog where ?""", self.maxUsn, lim)
         elif table == "cards":
-            return x("""
-select id, nid, did, ord, mod, %d, type, queue, due, ivl, factor, reps,
-lapses, left, odue, odid, flags, data from cards where %s""" % d)
+            return self.col.db.execute("""
+select id, nid, did, ord, mod, ?, type, queue, due, ivl, factor, reps,
+lapses, left, odue, odid, flags, data from cards where ?""", self.maxUsn, lim)
         else:
-            return x("""
-select id, guid, mid, mod, %d, tags, flds, '', '', flags, data
-from notes where %s""" % d)
+            return self.col.db.execute("""
+select id, guid, mid, mod, ?, tags, flds, '', '', flags, data
+from notes where ?""", self.maxUsn, lim)
 
     def chunk(self):
         buf = dict(done=False)
         while self.tablesLeft:
             curTable = self.tablesLeft.pop()
-            if not self.cursor:
-                self.cursor = self.cursorForTable(curTable)
-            buf[curTable] = self.cursor 
+            buf[curTable] = self.queryTable(curTable)
         if not self.tablesLeft:
             buf['done'] = True
         return buf

From e18e86e80904485738d6296a24ea08d079f3ddcc Mon Sep 17 00:00:00 2001
From: Karsten Lehmann <mail@kalehmann.de>
Date: Fri, 28 Aug 2020 00:03:11 +0200
Subject: [PATCH 10/38] Log sanity check errors

---
 src/ankisyncd/sync_app.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/ankisyncd/sync_app.py b/src/ankisyncd/sync_app.py
index fad24e4..bce786d 100644
--- a/src/ankisyncd/sync_app.py
+++ b/src/ankisyncd/sync_app.py
@@ -132,14 +132,18 @@ class SyncCollectionHandler(Syncer):
         self.mergeChanges(lchg, self.rchg)
         return lchg
 
-    def sanityCheck2(self, client):
+    def sanityCheck2(self, client, full=None):
         server = self.sanityCheck()
         if client != server:
+            logger.info(
+                f"sanity check failed with server: {server} client: {client}"
+            )
+
             return dict(status="bad", c=client, s=server)
         return dict(status="ok")
 
     def finish(self, mod=None):
-        return super().finish(self, anki.utils.intTime(1000))
+        return super().finish(anki.utils.intTime(1000))
 
     # This function had to be put here in its entirety because Syncer.removed()
     # doesn't use self.usnLim() (which we override in this class) in queries.
@@ -295,7 +299,6 @@ class SyncMediaHandler:
         for filename in filenames:
             try:
                 self.col.media.syncDelete(filename)
-                self.col.media.db.commit()
             except OSError as err:
                 logger.error("Error when removing file '%s' from media dir: "
                               "%s" % (filename, str(err)))

From 7eff3815a48d28b3a2965188e593a40f21db2278 Mon Sep 17 00:00:00 2001
From: Karsten Lehmann <mail@kalehmann.de>
Date: Fri, 28 Aug 2020 00:34:07 +0200
Subject: [PATCH 11/38] Always downgrade the database before sync

This prevents the missing collation unicase error on the client
---
 src/ankisyncd/full_sync.py | 57 ++++++++++++++++++++++++++++++--------
 1 file changed, 46 insertions(+), 11 deletions(-)

diff --git a/src/ankisyncd/full_sync.py b/src/ankisyncd/full_sync.py
index dd26d68..9e7c3cc 100644
--- a/src/ankisyncd/full_sync.py
+++ b/src/ankisyncd/full_sync.py
@@ -1,13 +1,36 @@
 # -*- coding: utf-8 -*-
 
+import logging
 import os
 from sqlite3 import dbapi2 as sqlite
+import shutil
 import sys
+from webob.exc import HTTPBadRequest
 
-import anki.db
+from anki.db import DB
+from anki.collection import Collection
 
-class FullSyncManager:
-    def upload(self, col, data, session):
+logger = logging.getLogger("ankisyncd.media")
+logger.setLevel(1)
+
+class FullSyncManager(object):
+    def test_db(self, db: DB):
+        """
+        :param anki.db.DB db: the database uploaded from the client.
+        """
+        if db.scalar("pragma integrity_check") != "ok":
+            raise HTTPBadRequest(
+                "Integrity check failed for uploaded collection database file."
+            )
+
+    def upload(self, col: Collection, data: bytes, session) -> str:
+        """
+        Uploads a sqlite database from the client to the sync server.
+
+        :param anki.collection.Collectio col:
+        :param bytes data: The binary sqlite database from the client.
+        :param .sync_app.SyncUserSession session: The current session.
+        """
         # Verify integrity of the received database file before replacing our
         # existing db.
         temp_db_path = session.get_collection_path() + ".tmp"
@@ -15,10 +38,8 @@ class FullSyncManager:
             f.write(data)
 
         try:
-            with anki.db.DB(temp_db_path) as test_db:
-                if test_db.scalar("pragma integrity_check") != "ok":
-                    raise HTTPBadRequest("Integrity check failed for uploaded "
-                                         "collection database file.")
+            with DB(temp_db_path) as test_db:
+                self.test_db(test_db)
         except sqlite.Error as e:
             raise HTTPBadRequest("Uploaded collection database file is "
                                  "corrupt.")
@@ -26,7 +47,7 @@ class FullSyncManager:
         # Overwrite existing db.
         col.close()
         try:
-            os.replace(temp_db_path, session.get_collection_path())
+            shutil.copyfile(temp_db_path, session.get_collection_path())
         finally:
             col.reopen()
             # Reopen the media database
@@ -34,13 +55,27 @@ class FullSyncManager:
 
         return "OK"
 
+    def download(self, col: Collection, session) -> bytes:
+        """Download the binary database.
 
-    def download(self, col, session):
-        col.close()
+        Performs a downgrade to database schema 11 before sending the database
+        to the client.
+
+        :param anki.collection.Collection col:
+        :param .sync_app.SyncUserSession session:
+
+        :return bytes: the binary sqlite3 database
+        """
+        col.close(downgrade=True)
+        db_path = session.get_collection_path()
         try:
-            data = open(session.get_collection_path(), 'rb').read()
+            with open(db_path, 'rb') as tmp:
+                data = tmp.read()
         finally:
             col.reopen()
+            # Reopen the media database
+            col.media.connect()
+
         return data
 
 

From 4c09c1e248ebef36d517adff75bf4cdb4a95047f Mon Sep 17 00:00:00 2001
From: Karsten Lehmann <mail@kalehmann.de>
Date: Fri, 28 Aug 2020 12:27:15 +0200
Subject: [PATCH 12/38] fix sanity check

---
 src/ankisyncd/sync.py     | 14 ++++++++------
 src/ankisyncd/sync_app.py |  2 +-
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/src/ankisyncd/sync.py b/src/ankisyncd/sync.py
index 8cd3c3f..f5b97ca 100644
--- a/src/ankisyncd/sync.py
+++ b/src/ankisyncd/sync.py
@@ -187,7 +187,7 @@ class Syncer(object):
             self.col.crt = rchg['crt']
         self.prepareToChunk()
 
-    def sanityCheck(self):
+    def sanityCheck(self, full):
         if not self.col.basicCheck():
             return "failed basic check"
         for t in "cards", "notes", "revlog", "graves":
@@ -244,21 +244,24 @@ class Syncer(object):
         if table == "revlog":
             return self.col.db.execute("""
 select id, cid, ?, ease, ivl, lastIvl, factor, time, type
-from revlog where ?""", self.maxUsn, lim)
+from revlog where %s""" % lim, self.maxUsn)
         elif table == "cards":
             return self.col.db.execute("""
 select id, nid, did, ord, mod, ?, type, queue, due, ivl, factor, reps,
-lapses, left, odue, odid, flags, data from cards where ?""", self.maxUsn, lim)
+lapses, left, odue, odid, flags, data from cards where %s""" % lim, self.maxUsn)
         else:
             return self.col.db.execute("""
 select id, guid, mid, mod, ?, tags, flds, '', '', flags, data
-from notes where ?""", self.maxUsn, lim)
+from notes where %s""" % lim, self.maxUsn)
 
     def chunk(self):
         buf = dict(done=False)
         while self.tablesLeft:
             curTable = self.tablesLeft.pop()
             buf[curTable] = self.queryTable(curTable)
+            self.col.db.execute(
+                f"update {curTable} set usn=? where usn=-1", self.maxUsn
+            )
         if not self.tablesLeft:
             buf['done'] = True
         return buf
@@ -417,8 +420,7 @@ from notes where ?""", self.maxUsn, lim)
     def mergeConf(self, conf):
         newConf = ConfigManager(self.col)
         for key, value in conf.items():
-            newConf[key] = value
-        self.col.conf = newConf
+            self.col.set_config(key, value)
 
 # Wrapper for requests that tracks upload/download progress
 ##########################################################################
diff --git a/src/ankisyncd/sync_app.py b/src/ankisyncd/sync_app.py
index bce786d..638e6d6 100644
--- a/src/ankisyncd/sync_app.py
+++ b/src/ankisyncd/sync_app.py
@@ -133,7 +133,7 @@ class SyncCollectionHandler(Syncer):
         return lchg
 
     def sanityCheck2(self, client, full=None):
-        server = self.sanityCheck()
+        server = self.sanityCheck(full)
         if client != server:
             logger.info(
                 f"sanity check failed with server: {server} client: {client}"

From 537bbe89c95f94e79b355093f1c40ddd6e874d46 Mon Sep 17 00:00:00 2001
From: Karsten Lehmann <mail@kalehmann.de>
Date: Fri, 28 Aug 2020 12:39:51 +0200
Subject: [PATCH 13/38] updated readme to reflect latest changes

---
 README.md | 85 ++++++++++++++++++++-----------------------------------
 1 file changed, 31 insertions(+), 54 deletions(-)

diff --git a/README.md b/README.md
index 0714b57..7aa73fc 100644
--- a/README.md
+++ b/README.md
@@ -39,22 +39,8 @@ It supports Python 3 and Anki 2.1.
 Installing
 ----------
 
-0. Install Anki. The currently supported version range is 2.1.1〜2.1.11, with the
-   exception of 2.1.9<sup id="readme-fn-01b">[1](#readme-fn-01)</sup>. (Keep in
-   mind this range only applies to the Anki used by the server, clients can be
-   as old as 2.0.27 and still work.) Running the server with other versions might
-   work as long as they're not 2.0.x, but things might break, so do it at your
-   own risk. If for some reason you can't get the supported Anki version easily
-   on your system, you can use `anki-bundled` from this repo:
-
-        $ git submodule update --init
-        $ cd anki-bundled
-        $ pip install -r requirements.txt
-
-   Keep in mind `pyaudio`, a dependency of Anki, requires development headers for
-   Python 3 and PortAudio to be present before running `pip`. If you can't or
-   don't want to install these, you can try [patching Anki](#running-ankisyncd-without-pyaudio).
-
+0. Install the current version of Anki.
+   
 1. Install the dependencies:
 
         $ pip install webob
@@ -65,22 +51,28 @@ Installing
 
         $ ./ankisyncctl.py adduser <username>
 
-4. Run ankisyncd:
+4. Setup a proxy to unchunk the requests.
+Webob does not support the header "Transfer-Encoding: chunked" used by Anki and
+therefore ankisyncd sees chunked requests as empty. To solve this problem setup
+Nginx (or any other webserver of your choice) and configure it to "unchunk" the
+requests to ankisyncd:
+
+    server {
+        listen      27701;
+        server_name default;
+
+        location / {
+            proxy_http_version 1.0;
+            proxy_pass         http://ankisyncd:27701/;
+        }
+    }
+
+5. Run ankisyncd:
 
         $ python -m ankisyncd
 
 ---
 
-<span id="readme-fn-01"></span>
-1. 2.1.9 is not supported due to [commit `95ccbfdd3679`][] introducing the
-   dependency on the `aqt` module, which depends on PyQt5. The server should
-   still work fine if you have PyQt5 installed. This has been fixed in
-   [commit `a389b8b4a0e2`][], which is a part of the 2.1.10 release.
-[↑](#readme-fn-01b)
-
-[commit `95ccbfdd3679`]: https://github.com/dae/anki/commit/95ccbfdd3679dd46f22847c539c7fddb8fa904ea
-[commit `a389b8b4a0e2`]: https://github.com/dae/anki/commit/a389b8b4a0e209023c4533a7ee335096a704079c
-
 Installing (Docker)
 -------------------
 
@@ -89,6 +81,18 @@ Follow [these instructions](https://github.com/kuklinistvan/docker-anki-sync-ser
 Setting up Anki
 ---------------
 
+### Anki 2.1.28 and above
+
+Create a new directory in [the add-ons folder][addons21] (name it something
+like ankisyncd), create a file named `__init__.py` containing the code below
+and put it in the `ankisyncd` directory.
+
+    import os
+
+    addr = "http://127.0.0.1:27701/" # put your server address here
+    os.environ["SYNC_ENDPOINT"] = addr + "sync/"
+    os.environ["SYNC_ENDPOINT_MEDIA"] = addr + "msync/"
+
 ### Anki 2.1
 
 Create a new directory in [the add-ons folder][addons21] (name it something
@@ -125,8 +129,6 @@ Unless you have set up a reverse proxy to handle encrypted connections, use
 whatever you have specified in `ankisyncd.conf` (or, if using a reverse proxy,
 whatever port you configured to accept the front-end connection).
 
-**Do not use trailing slashes.**
-
 Even though the AnkiDroid interface will request an email address, this is not
 required; it will simply be the username you configured with `ankisyncctl.py
 adduser`.
@@ -138,31 +140,6 @@ Running `ankisyncd` without `pyaudio`
 want to install PortAudio, you can edit some files in the `anki-bundled`
 directory to exclude `pyaudio`:
 
-### Anki ≥2.1.9
-
-Just remove "pyaudio" from requirements.txt and you're done. This change has
-been introduced in [commit `ca710ab3f1c1`][].
-
-[commit `ca710ab3f1c1`]: https://github.com/dae/anki/commit/ca710ab3f1c1174469a3b48f1257c0fc0ce624bf
-
-### Older versions
-
-First go to `anki-bundled`, then follow one of the instructions below. They all
-do the same thing, you can pick whichever one you're most comfortable with.
-
-Manual version: remove every line past "# Packaged commands" in anki/sound.py,
-remove every line starting with "pyaudio" in requirements.txt
-
-`ed` version:
-
-    $ echo '/# Packaged commands/,$d;w' | tr ';' '\n' | ed anki/sound.py
-    $ echo '/^pyaudio/d;w' | tr ';' '\n' | ed requirements.txt
-
-`sed -i` version:
-
-    $ sed -i '/# Packaged commands/,$d' anki/sound.py
-    $ sed -i '/^pyaudio/d' requirements.txt
-
 ENVVAR configuration overrides
 ------------------------------
 

From 471e3aead4abaa9f5e29b509207def18619dabe8 Mon Sep 17 00:00:00 2001
From: Karsten Lehmann <mail@kalehmann.de>
Date: Fri, 28 Aug 2020 17:05:28 +0200
Subject: [PATCH 14/38] Removed haveDirty check from sync code

The dirty field does not exist in the media table anymore.
---
 src/ankisyncd/sync.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ankisyncd/sync.py b/src/ankisyncd/sync.py
index f5b97ca..1e8fd06 100644
--- a/src/ankisyncd/sync.py
+++ b/src/ankisyncd/sync.py
@@ -697,7 +697,7 @@ class MediaSyncer(object):
         lastUsn = self.col.media.lastUsn()
         ret = self.server.begin()
         srvUsn = ret['usn']
-        if lastUsn == srvUsn and not self.col.media.haveDirty():
+        if lastUsn == srvUsn:
             return "noChanges"
 
         # loop through and process changes from server

From c6f82e20cd9c9fcebc6f3f2b85cee5f84adb5edc Mon Sep 17 00:00:00 2001
From: Karsten Lehmann <mail@kalehmann.de>
Date: Fri, 28 Aug 2020 19:58:10 +0200
Subject: [PATCH 15/38] Removed unused class MediaSyncer

---
 src/ankisyncd/sync.py | 142 ------------------------------------------
 1 file changed, 142 deletions(-)

diff --git a/src/ankisyncd/sync.py b/src/ankisyncd/sync.py
index 1e8fd06..3eda801 100644
--- a/src/ankisyncd/sync.py
+++ b/src/ankisyncd/sync.py
@@ -669,148 +669,6 @@ class FullSyncer(HttpSyncer):
             return False
         return True
 
-# Media syncing
-##########################################################################
-#
-# About conflicts:
-# - to minimize data loss, if both sides are marked for sending and one
-#   side has been deleted, favour the add
-# - if added/changed on both sides, favour the server version on the
-#   assumption other syncers are in sync with the server
-#
-
-class MediaSyncer(object):
-    def __init__(self, col, server=None):
-        self.col = col
-        self.server = server
-
-    def sync(self):
-        # check if there have been any changes
-        runHook("sync", "findMedia")
-        self.col.log("findChanges")
-        try:
-            self.col.media.findChanges()
-        except DBError:
-            return "corruptMediaDB"
-
-        # begin session and check if in sync
-        lastUsn = self.col.media.lastUsn()
-        ret = self.server.begin()
-        srvUsn = ret['usn']
-        if lastUsn == srvUsn:
-            return "noChanges"
-
-        # loop through and process changes from server
-        self.col.log("last local usn is %s"%lastUsn)
-        self.downloadCount = 0
-        while True:
-            data = self.server.mediaChanges(lastUsn=lastUsn)
-
-            self.col.log("mediaChanges resp count %d"%len(data))
-            if not data:
-                break
-
-            need = []
-            lastUsn = data[-1][1]
-            for fname, rusn, rsum in data:
-                lsum, ldirty = self.col.media.syncInfo(fname)
-                self.col.log(
-                    "check: lsum=%s rsum=%s ldirty=%d rusn=%d fname=%s"%(
-                        (lsum and lsum[0:4]),
-                        (rsum and rsum[0:4]),
-                        ldirty,
-                        rusn,
-                        fname))
-
-                if rsum:
-                    # added/changed remotely
-                    if not lsum or lsum != rsum:
-                        self.col.log("will fetch")
-                        need.append(fname)
-                    else:
-                        self.col.log("have same already")
-                    if ldirty:
-                        self.col.media.markClean([fname])
-                elif lsum:
-                    # deleted remotely
-                    if not ldirty:
-                        self.col.log("delete local")
-                        self.col.media.syncDelete(fname)
-                    else:
-                        # conflict; local add overrides remote delete
-                        self.col.log("conflict; will send")
-                else:
-                    # deleted both sides
-                    self.col.log("both sides deleted")
-                    if ldirty:
-                        self.col.media.markClean([fname])
-
-            self._downloadFiles(need)
-
-            self.col.log("update last usn to %d"%lastUsn)
-            self.col.media.setLastUsn(lastUsn) # commits
-
-        # at this point we're all up to date with the server's changes,
-        # and we need to send our own
-
-        updateConflict = False
-        toSend = self.col.media.dirtyCount()
-        while True:
-            zip, fnames = self.col.media.mediaChangesZip()
-            if not fnames:
-                break
-
-            runHook("syncMsg", ngettext(
-                "%d media change to upload", "%d media changes to upload", toSend)
-                    % toSend)
-
-            processedCnt, serverLastUsn = self.server.uploadChanges(zip)
-            self.col.media.markClean(fnames[0:processedCnt])
-
-            self.col.log("processed %d, serverUsn %d, clientUsn %d" % (
-                processedCnt, serverLastUsn, lastUsn
-            ))
-
-            if serverLastUsn - processedCnt == lastUsn:
-                self.col.log("lastUsn in sync, updating local")
-                lastUsn = serverLastUsn
-                self.col.media.setLastUsn(serverLastUsn) # commits
-            else:
-                self.col.log("concurrent update, skipping usn update")
-                # commit for markClean
-                self.col.media.db.commit()
-                updateConflict = True
-
-            toSend -= processedCnt
-
-        if updateConflict:
-            self.col.log("restart sync due to concurrent update")
-            return self.sync()
-
-        lcnt = self.col.media.mediaCount()
-        ret = self.server.mediaSanity(local=lcnt)
-        if ret == "OK":
-            return "OK"
-        else:
-            self.col.media.forceResync()
-            return ret
-
-    def _downloadFiles(self, fnames):
-        self.col.log("%d files to fetch"%len(fnames))
-        while fnames:
-            top = fnames[0:SYNC_ZIP_COUNT]
-            self.col.log("fetch %s"%top)
-            zipData = self.server.downloadFiles(files=top)
-            cnt = self.col.media.addFilesFromZip(zipData)
-            self.downloadCount += cnt
-            self.col.log("received %d files"%cnt)
-            fnames = fnames[cnt:]
-
-            n = self.downloadCount
-            runHook("syncMsg", ngettext(
-                "%d media file downloaded", "%d media files downloaded", n)
-                    % n)
-
 # Remote media syncing
 ##########################################################################
 

From 9d67943c11ca0c46aad5024d9e16313d513c32e0 Mon Sep 17 00:00:00 2001
From: Karsten Lehmann <mail@kalehmann.de>
Date: Fri, 28 Aug 2020 20:02:18 +0200
Subject: [PATCH 16/38] Marked test to fail because of missing _logChanges
 Method

---
 tests/test_media.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/test_media.py b/tests/test_media.py
index fc67dd4..5e9f34f 100644
--- a/tests/test_media.py
+++ b/tests/test_media.py
@@ -15,6 +15,9 @@ class ServerMediaManagerTest(unittest.TestCase):
         cls.colutils.clean_up()
         cls.colutils = None
 
+    # This test is currently expected to fail because the _logChanges
+    # method of the media manager does not exist anymore.
+    @unittest.expectedFailure
     def test_upgrade(self):
         col = self.colutils.create_empty_col()
         cm = col.media

From 94da88a3b2b3c20ea997d440b80eec2206b840ff Mon Sep 17 00:00:00 2001
From: Karsten Lehmann <mail@kalehmann.de>
Date: Fri, 28 Aug 2020 20:03:26 +0200
Subject: [PATCH 17/38] Updated media test to work with latest changes

---
 tests/test_media.py | 26 +++++++++++++++++---------
 1 file changed, 17 insertions(+), 9 deletions(-)

diff --git a/tests/test_media.py b/tests/test_media.py
index 5e9f34f..a7aacd9 100644
--- a/tests/test_media.py
+++ b/tests/test_media.py
@@ -1,5 +1,6 @@
 import os.path
 import unittest
+from unittest.mock import MagicMock
 
 import ankisyncd.media
 import helpers.collection_utils
@@ -44,19 +45,26 @@ class ServerMediaManagerTest(unittest.TestCase):
             list(cm.db.execute("SELECT fname, csum FROM media")),
         )
         self.assertEqual(cm.lastUsn(), sm.lastUsn())
-        self.assertEqual(list(sm.db.execute("SELECT usn FROM media")), [(161,), (161,)])
+        self.assertEqual(
+            list(sm.db.execute("SELECT usn FROM media")),
+            [(161,), (161,)]
+        )
 
     def test_mediaChanges_lastUsn_order(self):
         col = self.colutils.create_empty_col()
         col.media = ankisyncd.media.ServerMediaManager(col)
-        mh = ankisyncd.sync_app.SyncMediaHandler(col)
-        mh.col.media.db.execute("""
-            INSERT INTO media (fname, usn, csum)
-            VALUES
+        session = MagicMock()
+        session.name = 'test'
+        mh = ankisyncd.sync_app.SyncMediaHandler(col, session)
+        mh.col.media.addMedia(
+            (
                 ('fileA', 101, '53059abba1a72c7aff34a3eaf7fef10ed65541ce'),
-                ('fileB', 100, 'a5ae546046d09559399c80fa7076fb10f1ce4bcd')
-        """)
-
+                ('fileB', 100, 'a5ae546046d09559399c80fa7076fb10f1ce4bcd'),
+            )
+        )
         # anki assumes mh.col.media.lastUsn() == mh.mediaChanges()['data'][-1][1]
         # ref: anki/sync.py:720 (commit cca3fcb2418880d0430a5c5c2e6b81ba260065b7)
-        self.assertEqual(mh.mediaChanges(lastUsn=99)['data'][-1][1], mh.col.media.lastUsn())
+        self.assertEqual(
+            mh.mediaChanges(lastUsn=99)['data'][-1][1],
+            mh.col.media.lastUsn()
+        )

From 89dcfd6ecd3625039d96c0bcf26a5d67d862caa7 Mon Sep 17 00:00:00 2001
From: Karsten Lehmann <mail@kalehmann.de>
Date: Fri, 28 Aug 2020 20:04:18 +0200
Subject: [PATCH 18/38] ServerMediaManager extends MediaManager

---
 src/ankisyncd/media.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/ankisyncd/media.py b/src/ankisyncd/media.py
index c6627c5..47341f4 100644
--- a/src/ankisyncd/media.py
+++ b/src/ankisyncd/media.py
@@ -8,11 +8,13 @@ import os
 import os.path
 
 import anki.db
+from anki.media import MediaManager
 
 logger = logging.getLogger("ankisyncd.media")
 
-class ServerMediaManager(object):
-    def __init__(self, col):
+class ServerMediaManager(MediaManager):
+    def __init__(self, col, server=True):
+        super().__init__(col, server)
         self._dir = re.sub(r"(?i)\.(anki2)$", ".media", col.path)
         self.connect()
 

From 75c9267ecc6889708dbf3721abf8726a825363b4 Mon Sep 17 00:00:00 2001
From: Karsten Lehmann <mail@kalehmann.de>
Date: Fri, 28 Aug 2020 20:04:57 +0200
Subject: [PATCH 19/38] Fix parent initialization of SyncCollectionHandler

---
 src/ankisyncd/sync_app.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ankisyncd/sync_app.py b/src/ankisyncd/sync_app.py
index 638e6d6..3cbe3aa 100644
--- a/src/ankisyncd/sync_app.py
+++ b/src/ankisyncd/sync_app.py
@@ -52,7 +52,7 @@ class SyncCollectionHandler(Syncer):
 
     def __init__(self, col, session):
         # So that 'server' (the 3rd argument) can't get set
-        super().__init__(self, col)
+        super().__init__(col)
         self.session = session
 
     @staticmethod

From 2c1e5936b3920118973ddb766beb78c933e6e7d2 Mon Sep 17 00:00:00 2001
From: Karsten Lehmann <mail@kalehmann.de>
Date: Fri, 28 Aug 2020 20:05:32 +0200
Subject: [PATCH 20/38] Removed test because of deprecated MediaSyncer

---
 tests/test_web_media.py | 435 ----------------------------------------
 1 file changed, 435 deletions(-)
 delete mode 100644 tests/test_web_media.py

diff --git a/tests/test_web_media.py b/tests/test_web_media.py
deleted file mode 100644
index 0e2c787..0000000
--- a/tests/test_web_media.py
+++ /dev/null
@@ -1,435 +0,0 @@
-# -*- coding: utf-8 -*-
-import tempfile
-import filecmp
-import sqlite3
-import os
-import shutil
-
-import helpers.file_utils
-import helpers.server_utils
-import helpers.db_utils
-import anki.utils
-from anki.sync import MediaSyncer
-from helpers.mock_servers import MockRemoteMediaServer
-from helpers.monkey_patches import monkeypatch_mediamanager, unpatch_mediamanager
-from sync_app_functional_test_base import SyncAppFunctionalTestBase
-
-
-class SyncAppFunctionalMediaTest(SyncAppFunctionalTestBase):
-    def setUp(self):
-        SyncAppFunctionalTestBase.setUp(self)
-
-        monkeypatch_mediamanager()
-        self.tempdir = tempfile.mkdtemp(prefix=self.__class__.__name__)
-        self.hkey = self.mock_remote_server.hostKey("testuser", "testpassword")
-        client_collection = self.colutils.create_empty_col()
-        self.client_syncer = self.create_client_syncer(client_collection,
-                                                       self.hkey,
-                                                       self.server_test_app)
-
-    def tearDown(self):
-        self.hkey = None
-        self.client_syncer = None
-        unpatch_mediamanager()
-        SyncAppFunctionalTestBase.tearDown(self)
-
-    @staticmethod
-    def create_client_syncer(collection, hkey, server_test_app):
-        mock_remote_server = MockRemoteMediaServer(col=collection,
-                                                   hkey=hkey,
-                                                   server_test_app=server_test_app)
-        media_syncer = MediaSyncer(col=collection,
-                                   server=mock_remote_server)
-        return media_syncer
-
-    @staticmethod
-    def file_checksum(fname):
-        with open(fname, "rb") as f:
-            return anki.utils.checksum(f.read())
-
-    def media_dbs_differ(self, left_db_path, right_db_path, compare_timestamps=False):
-        """
-        Compares two media sqlite database files for equality. mtime and dirMod
-        timestamps are not considered when comparing.
-
-        :param left_db_path: path to the left db file
-        :param right_db_path: path to the right db file
-        :param compare_timestamps: flag determining if timestamp values
-                                   (media.mtime and meta.dirMod) are included
-                                   in the comparison
-        :return: True if the specified databases differ, False else
-        """
-
-        if not os.path.isfile(right_db_path):
-            raise IOError("file '" + left_db_path + "' does not exist")
-        elif not os.path.isfile(right_db_path):
-            raise IOError("file '" + right_db_path + "' does not exist")
-
-        # Create temporary copies of the files to act on.
-        newleft = os.path.join(self.tempdir, left_db_path) + ".tmp"
-        shutil.copyfile(left_db_path, newleft)
-        left_db_path = newleft
-
-        newright = os.path.join(self.tempdir, left_db_path) + ".tmp"
-        shutil.copyfile(right_db_path, newright)
-        right_db_path = newright
-
-        if not compare_timestamps:
-            # Set all timestamps that are not NULL to 0.
-            for dbPath in [left_db_path, right_db_path]:
-                connection = sqlite3.connect(dbPath)
-
-                connection.execute("""UPDATE media SET mtime=0
-                                      WHERE mtime IS NOT NULL""")
-
-                connection.execute("""UPDATE meta SET dirMod=0
-                                      WHERE rowid=1""")
-                connection.commit()
-                connection.close()
-
-        return helpers.db_utils.diff(left_db_path, right_db_path)
-
-    def test_sync_empty_media_dbs(self):
-        # With both the client and the server having no media to sync,
-        # syncing should change nothing.
-        self.assertEqual('noChanges', self.client_syncer.sync())
-        self.assertEqual('noChanges', self.client_syncer.sync())
-
-    def test_sync_file_from_server(self):
-        """
-        Adds a file on the server. After syncing, client and server should have
-        the identical file in their media directories and media databases.
-        """
-        client = self.client_syncer
-        server = helpers.server_utils.get_syncer_for_hkey(self.server_app,
-                                                      self.hkey,
-                                                      'media')
-
-        # Create a test file.
-        temp_file_path = helpers.file_utils.create_named_file("foo.jpg", "hello")
-
-        # Add the test file to the server's collection.
-        helpers.server_utils.add_files_to_server_mediadb(server.col.media, [temp_file_path])
-
-        # Syncing should work.
-        self.assertEqual(client.sync(), 'OK')
-
-        # The test file should be present in the server's and in the client's
-        # media directory.
-        self.assertTrue(
-            filecmp.cmp(os.path.join(client.col.media.dir(), "foo.jpg"),
-                        os.path.join(server.col.media.dir(), "foo.jpg")))
-
-        # Further syncing should do nothing.
-        self.assertEqual(client.sync(), 'noChanges')
-
-    def test_sync_file_from_client(self):
-        """
-        Adds a file on the client. After syncing, client and server should have
-        the identical file in their media directories and media databases.
-        """
-        join = os.path.join
-        client = self.client_syncer
-        server = helpers.server_utils.get_syncer_for_hkey(self.server_app,
-                                                      self.hkey,
-                                                      'media')
-
-        # Create a test file.
-        temp_file_path = helpers.file_utils.create_named_file("foo.jpg", "hello")
-
-        # Add the test file to the client's media collection.
-        helpers.server_utils.add_files_to_client_mediadb(client.col.media,
-                                                  [temp_file_path],
-                                                  update_db=True)
-
-        # Syncing should work.
-        self.assertEqual(client.sync(), 'OK')
-
-        # The same file should be present in both the client's and the server's
-        # media directory.
-        self.assertTrue(filecmp.cmp(join(client.col.media.dir(), "foo.jpg"),
-                                    join(server.col.media.dir(), "foo.jpg")))
-
-        # Further syncing should do nothing.
-        self.assertEqual(client.sync(), 'noChanges')
-
-        # The media data of client and server should be identical.
-        self.assertEqual(
-            list(client.col.media.db.execute("SELECT fname, csum FROM media")),
-            list(server.col.media.db.execute("SELECT fname, csum FROM media"))
-        )
-        self.assertEqual(client.col.media.lastUsn(), server.col.media.lastUsn())
-
-    def test_sync_different_files(self):
-        """
-        Adds a file on the client and a file with different name and content on
-        the server. After syncing, both client and server should have both
-        files in their media directories and databases.
-        """
-        join = os.path.join
-        isfile = os.path.isfile
-        client = self.client_syncer
-        server = helpers.server_utils.get_syncer_for_hkey(self.server_app,
-                                                      self.hkey,
-                                                      'media')
-
-        # Create two files and add one to the server and one to the client.
-        file_for_client = helpers.file_utils.create_named_file("foo.jpg", "hello")
-        file_for_server = helpers.file_utils.create_named_file("bar.jpg", "goodbye")
-
-        helpers.server_utils.add_files_to_client_mediadb(client.col.media,
-                                                  [file_for_client],
-                                                  update_db=True)
-        helpers.server_utils.add_files_to_server_mediadb(server.col.media, [file_for_server])
-
-        # Syncing should work.
-        self.assertEqual(client.sync(), 'OK')
-
-        # Both files should be present in the client's and in the server's
-        # media directories.
-        self.assertTrue(isfile(join(client.col.media.dir(), "foo.jpg")))
-        self.assertTrue(isfile(join(server.col.media.dir(), "foo.jpg")))
-        self.assertTrue(filecmp.cmp(
-            join(client.col.media.dir(), "foo.jpg"),
-            join(server.col.media.dir(), "foo.jpg"))
-        )
-        self.assertTrue(isfile(join(client.col.media.dir(), "bar.jpg")))
-        self.assertTrue(isfile(join(server.col.media.dir(), "bar.jpg")))
-        self.assertTrue(filecmp.cmp(
-            join(client.col.media.dir(), "bar.jpg"),
-            join(server.col.media.dir(), "bar.jpg"))
-        )
-
-        # Further syncing should change nothing.
-        self.assertEqual(client.sync(), 'noChanges')
-
-    def test_sync_different_contents(self):
-        """
-        Adds a file to the client and a file with identical name but different
-        contents to the server. After syncing, both client and server should
-        have the server's version of the file in their media directories and
-        databases.
-        """
-        join = os.path.join
-        isfile = os.path.isfile
-        client = self.client_syncer
-        server = helpers.server_utils.get_syncer_for_hkey(self.server_app,
-                                                      self.hkey,
-                                                      'media')
-
-        # Create two files with identical names but different contents and
-        # checksums. Add one to the server and one to the client.
-        file_for_client = helpers.file_utils.create_named_file("foo.jpg", "hello")
-        file_for_server = helpers.file_utils.create_named_file("foo.jpg", "goodbye")
-
-        helpers.server_utils.add_files_to_client_mediadb(client.col.media,
-                                                  [file_for_client],
-                                                  update_db=True)
-        helpers.server_utils.add_files_to_server_mediadb(server.col.media, [file_for_server])
-
-        # Syncing should work.
-        self.assertEqual(client.sync(), 'OK')
-
-        # A version of the file should be present in both the client's and the
-        # server's media directory.
-        self.assertTrue(isfile(join(client.col.media.dir(), "foo.jpg")))
-        self.assertEqual(os.listdir(client.col.media.dir()), ['foo.jpg'])
-        self.assertTrue(isfile(join(server.col.media.dir(), "foo.jpg")))
-        self.assertEqual(os.listdir(server.col.media.dir()), ['foo.jpg'])
-        self.assertEqual(client.sync(), 'noChanges')
-
-        # Both files should have the contents of the server's version.
-        _checksum = client.col.media._checksum
-        self.assertEqual(_checksum(join(client.col.media.dir(), "foo.jpg")),
-                         _checksum(file_for_server))
-        self.assertEqual(_checksum(join(server.col.media.dir(), "foo.jpg")),
-                         _checksum(file_for_server))
-
-    def test_sync_add_and_delete_on_client(self):
-        """
-        Adds a file on the client. After syncing, the client and server should
-        both have the file. Then removes the file from the client's directory
-        and marks it as deleted in its database. After syncing again, the
-        server should have removed its version of the file from its media dir
-        and marked it as deleted in its db.
-        """
-        join = os.path.join
-        isfile = os.path.isfile
-        client = self.client_syncer
-        server = helpers.server_utils.get_syncer_for_hkey(self.server_app,
-                                                      self.hkey,
-                                                      'media')
-
-        # Create a test file.
-        temp_file_path = helpers.file_utils.create_named_file("foo.jpg", "hello")
-
-        # Add the test file to client's media collection.
-        helpers.server_utils.add_files_to_client_mediadb(client.col.media,
-                                                  [temp_file_path],
-                                                  update_db=True)
-
-        # Syncing client should work.
-        self.assertEqual(client.sync(), 'OK')
-
-        # The same file should be present in both client's and the server's
-        # media directory.
-        self.assertTrue(filecmp.cmp(join(client.col.media.dir(), "foo.jpg"),
-                                    join(server.col.media.dir(), "foo.jpg")))
-
-        # Syncing client again should do nothing.
-        self.assertEqual(client.sync(), 'noChanges')
-
-        # Remove files from client's media dir and write changes to its db.
-        os.remove(join(client.col.media.dir(), "foo.jpg"))
-
-        # TODO: client.col.media.findChanges() doesn't work here - why?
-        client.col.media._logChanges()
-        self.assertEqual(client.col.media.syncInfo("foo.jpg"), (None, 1))
-        self.assertFalse(isfile(join(client.col.media.dir(), "foo.jpg")))
-
-        # Syncing client again should work.
-        self.assertEqual(client.sync(), 'OK')
-
-        # server should have picked up the removal from client.
-        self.assertEqual(server.col.media.syncInfo("foo.jpg"), (None, 0))
-        self.assertFalse(isfile(join(server.col.media.dir(), "foo.jpg")))
-
-        # Syncing client again should do nothing.
-        self.assertEqual(client.sync(), 'noChanges')
-
-    def test_sync_compare_database_to_expected(self):
-        """
-        Adds a test image file to the client's media directory. After syncing,
-        the server's database should, except for timestamps, be identical to a
-        database containing the expected data.
-        """
-        client = self.client_syncer
-
-        # Add a test image file to the client's media collection but don't
-        # update its media db since the desktop client updates that, using
-        # findChanges(), only during syncs.
-        support_file = helpers.file_utils.get_asset_path('blue.jpg')
-        self.assertTrue(os.path.isfile(support_file))
-        helpers.server_utils.add_files_to_client_mediadb(client.col.media,
-                                                  [support_file],
-                                                  update_db=False)
-
-        # Syncing should work.
-        self.assertEqual(client.sync(), "OK")
-
-        # Create temporary db file with expected results.
-        chksum = client.col.media._checksum(support_file)
-        sql = ("""
-            CREATE TABLE meta (dirMod int, lastUsn int);
-
-            INSERT INTO `meta` (dirMod, lastUsn) VALUES (123456789,1);
-
-            CREATE TABLE media (
-              fname text not null primary key,
-              csum text,
-              mtime int not null,
-               dirty int not null
-            );
-
-            INSERT INTO `media` (fname, csum, mtime, dirty) VALUES (
-               'blue.jpg',
-               '%s',
-               1441483037,
-               0
-            );
-
-            CREATE INDEX idx_media_dirty on media (dirty);
-        """ % chksum)
-
-        _, dbpath = tempfile.mkstemp(suffix=".anki2")
-        helpers.db_utils.from_sql(dbpath, sql)
-
-        # Except for timestamps, the client's db after sync should be identical
-        # to the expected data.
-        self.assertFalse(self.media_dbs_differ(
-            client.col.media.db._path,
-            dbpath
-        ))
-        os.unlink(dbpath)
-
-    def test_sync_mediaChanges(self):
-        client = self.client_syncer
-        client2 = self.create_client_syncer(self.colutils.create_empty_col(), self.hkey, self.server_test_app)
-        server = helpers.server_utils.get_syncer_for_hkey(self.server_app, self.hkey, 'media')
-        self.assertEqual(server.mediaChanges(lastUsn=client.col.media.lastUsn())['data'], [])
-
-        helpers.server_utils.add_files_to_client_mediadb(client.col.media, [
-            helpers.file_utils.create_named_file("a", "lastUsn a"),
-            helpers.file_utils.create_named_file("b", "lastUsn b"),
-            helpers.file_utils.create_named_file("c", "lastUsn c"),
-        ], update_db=True)
-        self.assertEqual(client.sync(), "OK")
-        self.assertEqual(server.mediaChanges(lastUsn=client.col.media.lastUsn())['data'], [])
-
-        self.assertEqual(client2.sync(), "OK")
-        os.remove(os.path.join(client2.col.media.dir(), "c"))
-        client2.col.media._logChanges()
-        self.assertEqual(client2.sync(), "OK")
-        self.assertEqual(server.mediaChanges(lastUsn=client.col.media.lastUsn())['data'], [['c', 4, None]])
-        self.assertEqual(client.sync(), "OK")
-        self.assertEqual(server.mediaChanges(lastUsn=client.col.media.lastUsn())['data'], [])
-
-        helpers.server_utils.add_files_to_client_mediadb(client.col.media, [
-            helpers.file_utils.create_named_file("d", "lastUsn d"),
-        ], update_db=True)
-        client.col.media._logChanges()
-        self.assertEqual(client.sync(), "OK")
-
-        self.assertEqual(server.mediaChanges(lastUsn=client2.col.media.lastUsn())['data'], [['d', 5, self.file_checksum(os.path.join(server.col.media.dir(), "d"))]])
-
-        self.assertEqual(client2.sync(), "OK")
-        self.assertEqual(server.mediaChanges(lastUsn=client2.col.media.lastUsn())['data'], [])
-
-        dpath = os.path.join(client.col.media.dir(), "d")
-        with open(dpath, "a") as f:
-                f.write("\nsome change")
-        # files with the same mtime and name are considered equivalent by anki.media.MediaManager._changes
-        os.utime(dpath, (315529200, 315529200))
-        client.col.media._logChanges()
-        self.assertEqual(client.sync(), "OK")
-        self.assertEqual(server.mediaChanges(lastUsn=client2.col.media.lastUsn())['data'], [['d', 6, self.file_checksum(os.path.join(server.col.media.dir(), "d"))]])
-        self.assertEqual(client2.sync(), "OK")
-        self.assertEqual(server.mediaChanges(lastUsn=client2.col.media.lastUsn())['data'], [])
-
-    def test_sync_rename(self):
-        """
-        Adds 3 media files to the client's media directory, syncs and then
-        renames them and syncs again. After syncing, both the client and the
-        server should only have the renamed files.
-        """
-        client = self.client_syncer
-        client2 = self.create_client_syncer(self.colutils.create_empty_col(), self.hkey, self.server_test_app)
-        server = helpers.server_utils.get_syncer_for_hkey(self.server_app, self.hkey, 'media')
-        self.assertEqual(server.mediaChanges(lastUsn=client.col.media.lastUsn())['data'], [])
-
-        helpers.server_utils.add_files_to_client_mediadb(client.col.media, [
-            helpers.file_utils.create_named_file("a.wav", "lastUsn a"),
-            helpers.file_utils.create_named_file("b.wav", "lastUsn b"),
-            helpers.file_utils.create_named_file("c.wav", "lastUsn c"),
-        ], update_db=True)
-        self.assertEqual(client.sync(), "OK")
-
-        for fname in os.listdir(client.col.media.dir()):
-            os.rename(
-                os.path.join(client.col.media.dir(), fname),
-                os.path.join(client.col.media.dir(), fname[:1] + ".mp3")
-            )
-        client.col.media._logChanges()
-        self.assertEqual(client.sync(), "OK")
-        self.assertEqual(
-             set(os.listdir(server.col.media.dir())),
-             {"a.mp3", "b.mp3", "c.mp3"},
-        )
-        self.assertEqual(
-             set(os.listdir(client.col.media.dir())),
-             set(os.listdir(server.col.media.dir())),
-        )
-        self.assertEqual(
-            list(client.col.media.db.execute("SELECT fname, csum FROM media ORDER BY fname")),
-            list(server.col.media.db.execute("SELECT fname, csum FROM media ORDER BY fname")),
-        )

From c5bce6282fc8f3477edc6e4309e506cb456a3c07 Mon Sep 17 00:00:00 2001
From: Karsten Lehmann <mail@kalehmann.de>
Date: Fri, 28 Aug 2020 20:06:15 +0200
Subject: [PATCH 21/38] Adapted sync app test to latest changes

---
 tests/test_sync_app.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/tests/test_sync_app.py b/tests/test_sync_app.py
index 8e3ff89..d49adec 100644
--- a/tests/test_sync_app.py
+++ b/tests/test_sync_app.py
@@ -3,9 +3,9 @@ import os
 import sqlite3
 import tempfile
 import unittest
+from unittest.mock import MagicMock, Mock
 
-from anki.consts import SYNC_VER
-
+from ankisyncd.sync import SYNC_VER
 from ankisyncd.sync_app import SyncCollectionHandler
 from ankisyncd.sync_app import SyncUserSession
 
@@ -14,8 +14,13 @@ from collection_test_base import CollectionTestBase
 
 class SyncCollectionHandlerTest(CollectionTestBase):
     def setUp(self):
-        CollectionTestBase.setUp(self)
-        self.syncCollectionHandler = SyncCollectionHandler(self.collection)
+        super().setUp()
+        self.session = MagicMock()
+        self.session.name = 'test'
+        self.syncCollectionHandler = SyncCollectionHandler(
+            self.collection,
+            self.session
+        )
 
     def tearDown(self):
         CollectionTestBase.tearDown(self)
@@ -60,6 +65,7 @@ class SyncCollectionHandlerTest(CollectionTestBase):
         self.assertTrue((type(meta['ts']) == int) and meta['ts'] > 0)
         self.assertEqual(meta['mod'], self.collection.mod)
         self.assertEqual(meta['usn'], self.collection._usn)
+        self.assertEqual(meta['uname'], self.session.name)
         self.assertEqual(meta['musn'], self.collection.media.lastUsn())
         self.assertEqual(meta['msg'], '')
         self.assertEqual(meta['cont'], True)

From 5f17eb7db9c798922f864d6b80a1bc7feb168ebb Mon Sep 17 00:00:00 2001
From: Karsten Lehmann <mail@kalehmann.de>
Date: Fri, 28 Aug 2020 20:07:22 +0200
Subject: [PATCH 22/38] server_utils test helper works with non-expose media db

---
 tests/helpers/server_utils.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/helpers/server_utils.py b/tests/helpers/server_utils.py
index fed41ac..45e6b76 100644
--- a/tests/helpers/server_utils.py
+++ b/tests/helpers/server_utils.py
@@ -86,5 +86,6 @@ def add_files_to_server_mediadb(media, filepaths):
 
             with open(os.path.join(media.dir(), fname), 'wb') as f:
                 f.write(data)
-            media.db.execute("INSERT INTO media VALUES (?, ?, ?)", fname, media.lastUsn() + 1, csum)
-            media.db.commit()
+            media.addMedia(
+                ((fname, media.lastUsn() + 1, csum),)
+            )

From e2e756dcda2a08442a9dd37d71336ae04e99968c Mon Sep 17 00:00:00 2001
From: Karsten Lehmann <mail@kalehmann.de>
Date: Fri, 28 Aug 2020 20:08:23 +0200
Subject: [PATCH 23/38] Removed references to unused methods

---
 tests/helpers/monkey_patches.py | 19 +------------------
 1 file changed, 1 insertion(+), 18 deletions(-)

diff --git a/tests/helpers/monkey_patches.py b/tests/helpers/monkey_patches.py
index 6a9792e..e65fe80 100644
--- a/tests/helpers/monkey_patches.py
+++ b/tests/helpers/monkey_patches.py
@@ -2,7 +2,7 @@
 import os
 import sqlite3 as sqlite
 from anki.media import MediaManager
-from anki.storage import DB
+from anki.db import DB
 
 mediamanager_orig_funcs = {
     "findChanges": None,
@@ -26,10 +26,6 @@ def monkeypatch_mediamanager():
 
     def make_cwd_safe(original_func):
         mediamanager_orig_funcs["findChanges"] = MediaManager.findChanges
-        mediamanager_orig_funcs["mediaChangesZip"] = MediaManager.mediaChangesZip
-        mediamanager_orig_funcs["addFilesFromZip"] = MediaManager.addFilesFromZip
-        mediamanager_orig_funcs["syncDelete"] = MediaManager.syncDelete
-        mediamanager_orig_funcs["_logChanges"] = MediaManager._logChanges
 
         def wrapper(instance, *args):
             old_cwd = os.getcwd()
@@ -42,27 +38,14 @@ def monkeypatch_mediamanager():
         return wrapper
 
     MediaManager.findChanges = make_cwd_safe(MediaManager.findChanges)
-    MediaManager.mediaChangesZip = make_cwd_safe(MediaManager.mediaChangesZip)
-    MediaManager.addFilesFromZip = make_cwd_safe(MediaManager.addFilesFromZip)
-    MediaManager.syncDelete = make_cwd_safe(MediaManager.syncDelete)
-    MediaManager._logChanges = make_cwd_safe(MediaManager._logChanges)
 
 
 def unpatch_mediamanager():
     """Undoes monkey patches to Anki's MediaManager."""
 
     MediaManager.findChanges = mediamanager_orig_funcs["findChanges"]
-    MediaManager.mediaChangesZip = mediamanager_orig_funcs["mediaChangesZip"]
-    MediaManager.addFilesFromZip = mediamanager_orig_funcs["addFilesFromZip"]
-    MediaManager.syncDelete = mediamanager_orig_funcs["syncDelete"]
-    MediaManager._logChanges = mediamanager_orig_funcs["_logChanges"]
 
     mediamanager_orig_funcs["findChanges"] = None
-    mediamanager_orig_funcs["mediaChangesZip"] = None
-    mediamanager_orig_funcs["mediaChangesZip"] = None
-    mediamanager_orig_funcs["mediaChangesZip"] = None
-    mediamanager_orig_funcs["_logChanges"] = None
-
 
 def monkeypatch_db():
     """

From 0ef99f3524b5fd389a4b588b6ddf8fe657e95cd0 Mon Sep 17 00:00:00 2001
From: Karsten Lehmann <mail@kalehmann.de>
Date: Fri, 28 Aug 2020 20:08:54 +0200
Subject: [PATCH 24/38] mock_servers test helper works with new syncer code

---
 tests/helpers/mock_servers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/helpers/mock_servers.py b/tests/helpers/mock_servers.py
index 38442d3..fdd3a18 100644
--- a/tests/helpers/mock_servers.py
+++ b/tests/helpers/mock_servers.py
@@ -3,7 +3,7 @@ import io
 import logging
 import types
 
-from anki.sync import HttpSyncer, RemoteServer, RemoteMediaServer
+from ankisyncd.sync import HttpSyncer, RemoteServer, RemoteMediaServer
 
 
 class MockServerConnection:

From 60f12cf0a04128a5f9099056c22db49c0b9ce491 Mon Sep 17 00:00:00 2001
From: Karsten Lehmann <mail@kalehmann.de>
Date: Fri, 28 Aug 2020 20:09:14 +0200
Subject: [PATCH 25/38] file_utils test helper works with new syncer code

---
 tests/helpers/file_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/helpers/file_utils.py b/tests/helpers/file_utils.py
index ad99b5d..28dece3 100644
--- a/tests/helpers/file_utils.py
+++ b/tests/helpers/file_utils.py
@@ -10,7 +10,7 @@ import tempfile
 import unicodedata
 import zipfile
 
-from anki.consts import SYNC_ZIP_SIZE
+from ankisyncd.sync import SYNC_ZIP_SIZE
 
 
 def create_named_file(filename, file_contents=None):

From 9831159653fb4c5784bb9db27db595f78e0530a1 Mon Sep 17 00:00:00 2001
From: Karsten Lehmann <mail@kalehmann.de>
Date: Fri, 28 Aug 2020 20:09:59 +0200
Subject: [PATCH 26/38] Close whole collection instead of just the database

---
 tests/helpers/collection_utils.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/helpers/collection_utils.py b/tests/helpers/collection_utils.py
index 10fcaf5..b8e1231 100644
--- a/tests/helpers/collection_utils.py
+++ b/tests/helpers/collection_utils.py
@@ -5,7 +5,6 @@ import tempfile
 
 from anki import Collection
 
-
 class CollectionUtils:
     """
     Provides utility methods for creating, inspecting and manipulating anki
@@ -26,7 +25,7 @@ class CollectionUtils:
 
         file_path = os.path.join(self.tempdir, "collection.anki2")
         master_col = Collection(file_path)
-        master_col.db.close()
+        master_col.close()
         self.master_db_path = file_path
 
     def __enter__(self):

From b0d57d3a02f91ffa40d31747caeb7dce27ce29cc Mon Sep 17 00:00:00 2001
From: Karsten Lehmann <mail@kalehmann.de>
Date: Fri, 28 Aug 2020 20:10:26 +0200
Subject: [PATCH 27/38] Use wrapped collection in tests

That way we make sure, our ServerMediaManager is used instead
of the MediaManager
---
 tests/collection_test_base.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tests/collection_test_base.py b/tests/collection_test_base.py
index 03d05b0..b1da4fd 100644
--- a/tests/collection_test_base.py
+++ b/tests/collection_test_base.py
@@ -8,6 +8,8 @@ import shutil
 import anki
 import anki.storage
 
+from ankisyncd.collection import CollectionManager
+
 
 class CollectionTestBase(unittest.TestCase):
     """Parent class for tests that need a collection set up and torn down."""
@@ -15,7 +17,9 @@ class CollectionTestBase(unittest.TestCase):
     def setUp(self):
         self.temp_dir = tempfile.mkdtemp()
         self.collection_path = os.path.join(self.temp_dir, 'collection.anki2');
-        self.collection = anki.storage.Collection(self.collection_path)
+        cm = CollectionManager({})
+        collectionWrapper = cm.get_collection(self.collection_path)
+        self.collection = collectionWrapper._get_collection()
         self.mock_app = MagicMock()
 
     def tearDown(self):

From 75c1ea09947dffab4788d0198520d4925d5ee2dd Mon Sep 17 00:00:00 2001
From: Karsten Lehmann <mail@kalehmann.de>
Date: Fri, 28 Aug 2020 20:33:08 +0200
Subject: [PATCH 28/38] Added emacs temporary files to gitignore

---
 .gitignore | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.gitignore b/.gitignore
index 9061ee6..c3241a4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -95,6 +95,10 @@ share/python-wheels/
 *.egg
 MANIFEST
 
+# Emacs temporary files
+*#*#
+*.#*
+
 # PyInstaller
 #  Usually these files are written by a python script from a template
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.

From 9da0eb77735c9efc81ae5d3955eecb8c25068f0d Mon Sep 17 00:00:00 2001
From: Karsten Lehmann <mail@kalehmann.de>
Date: Fri, 28 Aug 2020 20:34:22 +0200
Subject: [PATCH 29/38] Added anki to dependencies

---
 poetry.lock              | 327 +++++++++++++++++++++++++++++++--------
 pyproject.toml           |   1 +
 src/requirements-dev.txt |  27 ++--
 src/requirements.txt     |   5 +
 4 files changed, 285 insertions(+), 75 deletions(-)

diff --git a/poetry.lock b/poetry.lock
index 5c9f4dc..3810d88 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,3 +1,32 @@
+[[package]]
+category = "main"
+description = "Anki's library code"
+name = "anki"
+optional = false
+python-versions = ">=3.7"
+version = "2.1.32"
+
+[package.dependencies]
+ankirspy = "2.1.32"
+beautifulsoup4 = "*"
+decorator = "*"
+distro = "*"
+orjson = "*"
+protobuf = "*"
+psutil = "*"
+
+[package.dependencies.requests]
+extras = ["socks"]
+version = "*"
+
+[[package]]
+category = "main"
+description = "Anki's Rust library code Python bindings"
+name = "ankirspy"
+optional = false
+python-versions = "*"
+version = "2.1.32"
+
 [[package]]
 category = "dev"
 description = "Disable App Nap on OS X 10.9"
@@ -7,19 +36,35 @@ optional = false
 python-versions = "*"
 version = "0.1.0"
 
+[[package]]
+category = "dev"
+description = "The secure Argon2 password hashing algorithm."
+name = "argon2-cffi"
+optional = false
+python-versions = "*"
+version = "20.1.0"
+
+[package.dependencies]
+cffi = ">=1.0.0"
+six = "*"
+
+[package.extras]
+dev = ["coverage (>=5.0.2)", "hypothesis", "pytest", "sphinx", "wheel", "pre-commit"]
+docs = ["sphinx"]
+tests = ["coverage (>=5.0.2)", "hypothesis", "pytest"]
+
 [[package]]
 category = "dev"
 description = "Classes Without Boilerplate"
 name = "attrs"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
-version = "19.3.0"
+version = "20.1.0"
 
 [package.extras]
-azure-pipelines = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "pytest-azurepipelines"]
-dev = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "sphinx", "pre-commit"]
-docs = ["sphinx", "zope.interface"]
-tests = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"]
+dev = ["coverage (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "sphinx", "sphinx-rtd-theme", "pre-commit"]
+docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"]
+tests = ["coverage (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"]
 
 [[package]]
 category = "dev"
@@ -65,6 +110,17 @@ optional = false
 python-versions = "*"
 version = "2020.6.20"
 
+[[package]]
+category = "dev"
+description = "Foreign Function Interface for Python calling C code."
+name = "cffi"
+optional = false
+python-versions = "*"
+version = "1.14.2"
+
+[package.dependencies]
+pycparser = "*"
+
 [[package]]
 category = "main"
 description = "Universal encoding detector for Python 2 and 3"
@@ -177,8 +233,8 @@ category = "dev"
 description = "IPython: Productive Interactive Computing"
 name = "ipython"
 optional = false
-python-versions = ">=3.6"
-version = "7.16.1"
+python-versions = ">=3.7"
+version = "7.17.0"
 
 [package.dependencies]
 appnope = "*"
@@ -326,7 +382,7 @@ description = "Jupyter protocol implementation and client libraries"
 name = "jupyter-client"
 optional = false
 python-versions = ">=3.5"
-version = "6.1.6"
+version = "6.1.7"
 
 [package.dependencies]
 jupyter-core = ">=4.6.0"
@@ -336,7 +392,7 @@ tornado = ">=4.1"
 traitlets = "*"
 
 [package.extras]
-test = ["async-generator", "ipykernel", "ipython", "mock", "pytest", "pytest-asyncio", "pytest-timeout"]
+test = ["ipykernel", "ipython", "mock", "pytest", "pytest-asyncio", "async-generator", "pytest-timeout"]
 
 [[package]]
 category = "dev"
@@ -374,7 +430,7 @@ description = "The JupyterLab notebook server extension."
 name = "jupyterlab"
 optional = false
 python-versions = ">=3.5"
-version = "2.2.2"
+version = "2.2.6"
 
 [package.dependencies]
 jinja2 = ">=2.10"
@@ -410,7 +466,7 @@ description = "Python LiveReload is an awesome tool for web developers"
 name = "livereload"
 optional = false
 python-versions = "*"
-version = "2.6.2"
+version = "2.6.3"
 
 [package.dependencies]
 six = "*"
@@ -565,10 +621,11 @@ description = "A web-based notebook environment for interactive computing"
 name = "notebook"
 optional = false
 python-versions = ">=3.5"
-version = "6.0.3"
+version = "6.1.3"
 
 [package.dependencies]
 Send2Trash = "*"
+argon2-cffi = "*"
 ipykernel = "*"
 ipython-genutils = "*"
 jinja2 = "*"
@@ -578,12 +635,22 @@ nbconvert = "*"
 nbformat = "*"
 prometheus-client = "*"
 pyzmq = ">=17"
-terminado = ">=0.8.1"
+terminado = ">=0.8.3"
 tornado = ">=5.0"
 traitlets = ">=4.2.1"
 
 [package.extras]
-test = ["nose", "coverage", "requests", "nose-warnings-filters", "nbval", "nose-exclude", "selenium", "pytest", "pytest-cov", "nose-exclude"]
+docs = ["sphinx", "nbsphinx", "sphinxcontrib-github-alt"]
+test = ["nose", "coverage", "requests", "nose-warnings-filters", "nbval", "nose-exclude", "selenium", "pytest", "pytest-cov", "requests-unixsocket"]
+
+[[package]]
+category = "main"
+description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy"
+marker = "platform_machine == \"x86_64\""
+name = "orjson"
+optional = false
+python-versions = ">=3.6"
+version = "3.3.1"
 
 [[package]]
 category = "dev"
@@ -653,11 +720,23 @@ description = "Library for building powerful interactive command lines in Python
 name = "prompt-toolkit"
 optional = false
 python-versions = ">=3.6.1"
-version = "3.0.5"
+version = "3.0.6"
 
 [package.dependencies]
 wcwidth = "*"
 
+[[package]]
+category = "main"
+description = "Protocol Buffers"
+name = "protobuf"
+optional = false
+python-versions = "*"
+version = "3.13.0"
+
+[package.dependencies]
+setuptools = "*"
+six = ">=1.9"
+
 [[package]]
 category = "main"
 description = "Cross-platform lib for process and system monitoring in Python."
@@ -686,6 +765,14 @@ optional = false
 python-versions = "*"
 version = "0.2.11"
 
+[[package]]
+category = "dev"
+description = "C parser in Python"
+name = "pycparser"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+version = "2.20"
+
 [[package]]
 category = "dev"
 description = "Pygments is a syntax highlighting package written in Python."
@@ -756,7 +843,7 @@ description = "Python bindings for 0MQ"
 name = "pyzmq"
 optional = false
 python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*"
-version = "19.0.1"
+version = "19.0.2"
 
 [[package]]
 category = "dev"
@@ -764,7 +851,7 @@ description = "Jupyter Qt console"
 name = "qtconsole"
 optional = false
 python-versions = "*"
-version = "4.7.5"
+version = "4.7.6"
 
 [package.dependencies]
 ipykernel = ">=4.1"
@@ -824,7 +911,7 @@ python-versions = "*"
 version = "1.5.0"
 
 [[package]]
-category = "dev"
+category = "main"
 description = "Python 2 and 3 compatibility utilities"
 name = "six"
 optional = false
@@ -878,7 +965,7 @@ marker = "python_version > \"2.7\""
 name = "tqdm"
 optional = false
 python-versions = ">=2.6, !=3.0.*, !=3.1.*"
-version = "4.48.0"
+version = "4.48.2"
 
 [package.extras]
 dev = ["py-make (>=0.1.0)", "twine", "argopt", "pydoc-markdown"]
@@ -965,17 +1052,47 @@ docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"]
 testing = ["jaraco.itertools", "func-timeout"]
 
 [metadata]
-content-hash = "256b39b0726f0028059bd4d3a895cfe5a0676284c57a7615e6178734caa70227"
+content-hash = "85d03342e458196cc35e890733a1dd3c48a504cda333b46114dd57c58b42c9b6"
+lock-version = "1.0"
 python-versions = "^3.7"
 
 [metadata.files]
+anki = [
+    {file = "anki-2.1.32-py3-none-any.whl", hash = "sha256:97cfc292876196572b3d037ab218e3c9014ec7b31744c82e9847a45e796e3fdd"},
+]
+ankirspy = [
+    {file = "ankirspy-2.1.32-cp37-cp37m-macosx_10_7_x86_64.whl", hash = "sha256:6cd446155ee56f2557ecee6cfa42857ef44f4e5322a9fd5a06ff25a3bffc6980"},
+    {file = "ankirspy-2.1.32-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:59cf16a23f7afabfe011302ae47833c13d57fcbfd7bbf9e2ff78c52cbffea106"},
+    {file = "ankirspy-2.1.32-cp37-none-win_amd64.whl", hash = "sha256:e5d133cda5a849a5734cd12d3e7d29f34907116e97712d70c895232cbba9a802"},
+    {file = "ankirspy-2.1.32-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:8358846c61b575b163fb12bfcb28ba12d44611606f04eef7230f374f9c31c2a4"},
+    {file = "ankirspy-2.1.32-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:ce71ae0e9695246cc58bd6c51b3ca5d8958a32fa3cee77843eb1ed95a35739ff"},
+    {file = "ankirspy-2.1.32-cp38-none-win_amd64.whl", hash = "sha256:1962126aaf72b678bde10bebb5108f988d6888be35870c46ec2e14af7fedee1e"},
+]
 appnope = [
     {file = "appnope-0.1.0-py2.py3-none-any.whl", hash = "sha256:5b26757dc6f79a3b7dc9fab95359328d5747fcb2409d331ea66d0272b90ab2a0"},
     {file = "appnope-0.1.0.tar.gz", hash = "sha256:8b995ffe925347a2138d7ac0fe77155e4311a0ea6d6da4f5128fe4b3cbe5ed71"},
 ]
+argon2-cffi = [
+    {file = "argon2-cffi-20.1.0.tar.gz", hash = "sha256:d8029b2d3e4b4cea770e9e5a0104dd8fa185c1724a0f01528ae4826a6d25f97d"},
+    {file = "argon2_cffi-20.1.0-cp27-cp27m-macosx_10_6_intel.whl", hash = "sha256:6ea92c980586931a816d61e4faf6c192b4abce89aa767ff6581e6ddc985ed003"},
+    {file = "argon2_cffi-20.1.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:05a8ac07c7026542377e38389638a8a1e9b78f1cd8439cd7493b39f08dd75fbf"},
+    {file = "argon2_cffi-20.1.0-cp27-cp27m-win32.whl", hash = "sha256:0bf066bc049332489bb2d75f69216416329d9dc65deee127152caeb16e5ce7d5"},
+    {file = "argon2_cffi-20.1.0-cp27-cp27m-win_amd64.whl", hash = "sha256:57358570592c46c420300ec94f2ff3b32cbccd10d38bdc12dc6979c4a8484fbc"},
+    {file = "argon2_cffi-20.1.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:7d455c802727710e9dfa69b74ccaab04568386ca17b0ad36350b622cd34606fe"},
+    {file = "argon2_cffi-20.1.0-cp35-abi3-manylinux1_x86_64.whl", hash = "sha256:b160416adc0f012fb1f12588a5e6954889510f82f698e23ed4f4fa57f12a0647"},
+    {file = "argon2_cffi-20.1.0-cp35-cp35m-win32.whl", hash = "sha256:9bee3212ba4f560af397b6d7146848c32a800652301843df06b9e8f68f0f7361"},
+    {file = "argon2_cffi-20.1.0-cp35-cp35m-win_amd64.whl", hash = "sha256:392c3c2ef91d12da510cfb6f9bae52512a4552573a9e27600bdb800e05905d2b"},
+    {file = "argon2_cffi-20.1.0-cp36-cp36m-win32.whl", hash = "sha256:ba7209b608945b889457f949cc04c8e762bed4fe3fec88ae9a6b7765ae82e496"},
+    {file = "argon2_cffi-20.1.0-cp36-cp36m-win_amd64.whl", hash = "sha256:da7f0445b71db6d3a72462e04f36544b0de871289b0bc8a7cc87c0f5ec7079fa"},
+    {file = "argon2_cffi-20.1.0-cp37-abi3-macosx_10_6_intel.whl", hash = "sha256:cc0e028b209a5483b6846053d5fd7165f460a1f14774d79e632e75e7ae64b82b"},
+    {file = "argon2_cffi-20.1.0-cp37-cp37m-win32.whl", hash = "sha256:18dee20e25e4be86680b178b35ccfc5d495ebd5792cd00781548d50880fee5c5"},
+    {file = "argon2_cffi-20.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:6678bb047373f52bcff02db8afab0d2a77d83bde61cfecea7c5c62e2335cb203"},
+    {file = "argon2_cffi-20.1.0-cp38-cp38-win32.whl", hash = "sha256:77e909cc756ef81d6abb60524d259d959bab384832f0c651ed7dcb6e5ccdbb78"},
+    {file = "argon2_cffi-20.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:9dfd5197852530294ecb5795c97a823839258dfd5eb9420233c7cfedec2058f2"},
+]
 attrs = [
-    {file = "attrs-19.3.0-py2.py3-none-any.whl", hash = "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c"},
-    {file = "attrs-19.3.0.tar.gz", hash = "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"},
+    {file = "attrs-20.1.0-py2.py3-none-any.whl", hash = "sha256:2867b7b9f8326499ab5b0e2d12801fa5c98842d2cbd22b35112ae04bf85b4dff"},
+    {file = "attrs-20.1.0.tar.gz", hash = "sha256:0ef97238856430dcf9228e07f316aefc17e8939fc8507e18c6501b761ef1a42a"},
 ]
 backcall = [
     {file = "backcall-0.2.0-py2.py3-none-any.whl", hash = "sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255"},
@@ -994,6 +1111,36 @@ certifi = [
     {file = "certifi-2020.6.20-py2.py3-none-any.whl", hash = "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41"},
     {file = "certifi-2020.6.20.tar.gz", hash = "sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3"},
 ]
+cffi = [
+    {file = "cffi-1.14.2-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:da9d3c506f43e220336433dffe643fbfa40096d408cb9b7f2477892f369d5f82"},
+    {file = "cffi-1.14.2-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:23e44937d7695c27c66a54d793dd4b45889a81b35c0751ba91040fe825ec59c4"},
+    {file = "cffi-1.14.2-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:0da50dcbccd7cb7e6c741ab7912b2eff48e85af217d72b57f80ebc616257125e"},
+    {file = "cffi-1.14.2-cp27-cp27m-win32.whl", hash = "sha256:76ada88d62eb24de7051c5157a1a78fd853cca9b91c0713c2e973e4196271d0c"},
+    {file = "cffi-1.14.2-cp27-cp27m-win_amd64.whl", hash = "sha256:15a5f59a4808f82d8ec7364cbace851df591c2d43bc76bcbe5c4543a7ddd1bf1"},
+    {file = "cffi-1.14.2-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:e4082d832e36e7f9b2278bc774886ca8207346b99f278e54c9de4834f17232f7"},
+    {file = "cffi-1.14.2-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:57214fa5430399dffd54f4be37b56fe22cedb2b98862550d43cc085fb698dc2c"},
+    {file = "cffi-1.14.2-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:6843db0343e12e3f52cc58430ad559d850a53684f5b352540ca3f1bc56df0731"},
+    {file = "cffi-1.14.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:577791f948d34d569acb2d1add5831731c59d5a0c50a6d9f629ae1cefd9ca4a0"},
+    {file = "cffi-1.14.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:8662aabfeab00cea149a3d1c2999b0731e70c6b5bac596d95d13f643e76d3d4e"},
+    {file = "cffi-1.14.2-cp35-cp35m-win32.whl", hash = "sha256:837398c2ec00228679513802e3744d1e8e3cb1204aa6ad408b6aff081e99a487"},
+    {file = "cffi-1.14.2-cp35-cp35m-win_amd64.whl", hash = "sha256:bf44a9a0141a082e89c90e8d785b212a872db793a0080c20f6ae6e2a0ebf82ad"},
+    {file = "cffi-1.14.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:29c4688ace466a365b85a51dcc5e3c853c1d283f293dfcc12f7a77e498f160d2"},
+    {file = "cffi-1.14.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:99cc66b33c418cd579c0f03b77b94263c305c389cb0c6972dac420f24b3bf123"},
+    {file = "cffi-1.14.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:65867d63f0fd1b500fa343d7798fa64e9e681b594e0a07dc934c13e76ee28fb1"},
+    {file = "cffi-1.14.2-cp36-cp36m-win32.whl", hash = "sha256:f5033952def24172e60493b68717792e3aebb387a8d186c43c020d9363ee7281"},
+    {file = "cffi-1.14.2-cp36-cp36m-win_amd64.whl", hash = "sha256:7057613efefd36cacabbdbcef010e0a9c20a88fc07eb3e616019ea1692fa5df4"},
+    {file = "cffi-1.14.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:6539314d84c4d36f28d73adc1b45e9f4ee2a89cdc7e5d2b0a6dbacba31906798"},
+    {file = "cffi-1.14.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:672b539db20fef6b03d6f7a14b5825d57c98e4026401fce838849f8de73fe4d4"},
+    {file = "cffi-1.14.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:95e9094162fa712f18b4f60896e34b621df99147c2cee216cfa8f022294e8e9f"},
+    {file = "cffi-1.14.2-cp37-cp37m-win32.whl", hash = "sha256:b9aa9d8818c2e917fa2c105ad538e222a5bce59777133840b93134022a7ce650"},
+    {file = "cffi-1.14.2-cp37-cp37m-win_amd64.whl", hash = "sha256:e4b9b7af398c32e408c00eb4e0d33ced2f9121fd9fb978e6c1b57edd014a7d15"},
+    {file = "cffi-1.14.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e613514a82539fc48291d01933951a13ae93b6b444a88782480be32245ed4afa"},
+    {file = "cffi-1.14.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:9b219511d8b64d3fa14261963933be34028ea0e57455baf6781fe399c2c3206c"},
+    {file = "cffi-1.14.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:c0b48b98d79cf795b0916c57bebbc6d16bb43b9fc9b8c9f57f4cf05881904c75"},
+    {file = "cffi-1.14.2-cp38-cp38-win32.whl", hash = "sha256:15419020b0e812b40d96ec9d369b2bc8109cc3295eac6e013d3261343580cc7e"},
+    {file = "cffi-1.14.2-cp38-cp38-win_amd64.whl", hash = "sha256:12a453e03124069b6896107ee133ae3ab04c624bb10683e1ed1c1663df17c13c"},
+    {file = "cffi-1.14.2.tar.gz", hash = "sha256:ae8f34d50af2c2154035984b8b5fc5d9ed63f32fe615646ab435b05b132ca91b"},
+]
 chardet = [
     {file = "chardet-3.0.4-py2.py3-none-any.whl", hash = "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"},
     {file = "chardet-3.0.4.tar.gz", hash = "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae"},
@@ -1038,8 +1185,8 @@ ipykernel = [
     {file = "ipykernel-5.3.4.tar.gz", hash = "sha256:9b2652af1607986a1b231c62302d070bc0534f564c393a5d9d130db9abbbe89d"},
 ]
 ipython = [
-    {file = "ipython-7.16.1-py3-none-any.whl", hash = "sha256:2dbcc8c27ca7d3cfe4fcdff7f45b27f9a8d3edfa70ff8024a71c7a8eb5f09d64"},
-    {file = "ipython-7.16.1.tar.gz", hash = "sha256:9f4fcb31d3b2c533333893b9172264e4821c1ac91839500f31bd43f2c59b3ccf"},
+    {file = "ipython-7.17.0-py3-none-any.whl", hash = "sha256:5a8f159ca8b22b9a0a1f2a28befe5ad2b703339afb58c2ffe0d7c8d7a3af5999"},
+    {file = "ipython-7.17.0.tar.gz", hash = "sha256:b70974aaa2674b05eb86a910c02ed09956a33f2dd6c71afc60f0b128a77e7f28"},
 ]
 ipython-genutils = [
     {file = "ipython_genutils-0.2.0-py2.py3-none-any.whl", hash = "sha256:72dd37233799e619666c9f639a9da83c34013a73e8bbc79a7a6348d93c61fab8"},
@@ -1075,8 +1222,8 @@ jupyter = [
     {file = "jupyter-1.0.0.zip", hash = "sha256:3e1f86076bbb7c8c207829390305a2b1fe836d471ed54be66a3b8c41e7f46cc7"},
 ]
 jupyter-client = [
-    {file = "jupyter_client-6.1.6-py3-none-any.whl", hash = "sha256:7ad9aa91505786420d77edc5f9fb170d51050c007338ba8d196f603223fd3b3a"},
-    {file = "jupyter_client-6.1.6.tar.gz", hash = "sha256:b360f8d4638bc577a4656e93f86298db755f915098dc763f6fc05da0c5d7a595"},
+    {file = "jupyter_client-6.1.7-py3-none-any.whl", hash = "sha256:c958d24d6eacb975c1acebb68ac9077da61b5f5c040f22f6849928ad7393b950"},
+    {file = "jupyter_client-6.1.7.tar.gz", hash = "sha256:49e390b36fe4b4226724704ea28d9fb903f1a3601b6882ce3105221cd09377a1"},
 ]
 jupyter-console = [
     {file = "jupyter_console-6.1.0-py2.py3-none-any.whl", hash = "sha256:b392155112ec86a329df03b225749a0fa903aa80811e8eda55796a40b5e470d8"},
@@ -1087,15 +1234,15 @@ jupyter-core = [
     {file = "jupyter_core-4.6.3.tar.gz", hash = "sha256:394fd5dd787e7c8861741880bdf8a00ce39f95de5d18e579c74b882522219e7e"},
 ]
 jupyterlab = [
-    {file = "jupyterlab-2.2.2-py3-none-any.whl", hash = "sha256:d0d743ea75b8eee20a18b96ccef24f76ee009bafb2617f3f330698fe3a00026e"},
-    {file = "jupyterlab-2.2.2.tar.gz", hash = "sha256:8aa9bc4b5020e7b9ec6e006d516d48bddf7d2528680af65840464ee722d59db3"},
+    {file = "jupyterlab-2.2.6-py3-none-any.whl", hash = "sha256:ae557386633fcb74359f436f2b87788a451260a07f2f14a1880fca8f4a9f64de"},
+    {file = "jupyterlab-2.2.6.tar.gz", hash = "sha256:6554b022d2cd120100e165ec537c6511d70de7f89e253b3c667ea28f2a9263ff"},
 ]
 jupyterlab-server = [
     {file = "jupyterlab_server-1.2.0-py3-none-any.whl", hash = "sha256:55d256077bf13e5bc9e8fbd5aac51bef82f6315111cec6b712b9a5ededbba924"},
     {file = "jupyterlab_server-1.2.0.tar.gz", hash = "sha256:5431d9dde96659364b7cc877693d5d21e7b80cea7ae3959ecc2b87518e5f5d8c"},
 ]
 livereload = [
-    {file = "livereload-2.6.2.tar.gz", hash = "sha256:d1eddcb5c5eb8d2ca1fa1f750e580da624c0f7fcb734aa5780dc81b7dcbd89be"},
+    {file = "livereload-2.6.3.tar.gz", hash = "sha256:776f2f865e59fde56490a56bcc6773b6917366bce0c267c60ee8aaf1a0959869"},
 ]
 lunr = [
     {file = "lunr-0.5.8-py2.py3-none-any.whl", hash = "sha256:aab3f489c4d4fab4c1294a257a30fec397db56f0a50273218ccc3efdbf01d6ca"},
@@ -1133,6 +1280,11 @@ markupsafe = [
     {file = "MarkupSafe-1.1.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6"},
     {file = "MarkupSafe-1.1.1-cp37-cp37m-win32.whl", hash = "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2"},
     {file = "MarkupSafe-1.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c"},
+    {file = "MarkupSafe-1.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15"},
+    {file = "MarkupSafe-1.1.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:cdb132fc825c38e1aeec2c8aa9338310d29d337bebbd7baa06889d09a60a1fa2"},
+    {file = "MarkupSafe-1.1.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42"},
+    {file = "MarkupSafe-1.1.1-cp38-cp38-win32.whl", hash = "sha256:596510de112c685489095da617b5bcbbac7dd6384aeebeda4df6025d0256a81b"},
+    {file = "MarkupSafe-1.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be"},
     {file = "MarkupSafe-1.1.1.tar.gz", hash = "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b"},
 ]
 mistune = [
@@ -1155,8 +1307,28 @@ nltk = [
     {file = "nltk-3.5.zip", hash = "sha256:845365449cd8c5f9731f7cb9f8bd6fd0767553b9d53af9eb1b3abf7700936b35"},
 ]
 notebook = [
-    {file = "notebook-6.0.3-py3-none-any.whl", hash = "sha256:3edc616c684214292994a3af05eaea4cc043f6b4247d830f3a2f209fa7639a80"},
-    {file = "notebook-6.0.3.tar.gz", hash = "sha256:47a9092975c9e7965ada00b9a20f0cf637d001db60d241d479f53c0be117ad48"},
+    {file = "notebook-6.1.3-py3-none-any.whl", hash = "sha256:964cc40cff68e473f3778aef9266e867f7703cb4aebdfd250f334efe02f64c86"},
+    {file = "notebook-6.1.3.tar.gz", hash = "sha256:9990d51b9931a31e681635899aeb198b4c4b41586a9e87fbfaaed1a71d0a05b6"},
+]
+orjson = [
+    {file = "orjson-3.3.1-cp36-cp36m-macosx_10_7_x86_64.whl", hash = "sha256:0f33d28083819579976669f54ca79675d8e95fd5d75e7db21b798354ed8dd15b"},
+    {file = "orjson-3.3.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4c290f1c0b6665d60181ee2f0ef631640d04ead2002ca4eadce4991ea5d6a4ed"},
+    {file = "orjson-3.3.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:bf542f372162533550e86003d48664ab5fc1b44fb2b88923b9794cc8db6f0cf0"},
+    {file = "orjson-3.3.1-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:28e6116ebd2082357bb9c66a76a3a1dc6aa4de0754801ac10b9903d31b752a1b"},
+    {file = "orjson-3.3.1-cp36-none-win_amd64.whl", hash = "sha256:c4ac5a1d1767733708fd9b45cbbab3f8871af57b54b707a2dc6fddb47e51a81a"},
+    {file = "orjson-3.3.1-cp37-cp37m-macosx_10_7_x86_64.whl", hash = "sha256:0f11fd620b74fbdcf29021b3a9c36fb6e13efcdd63cbacc292d0786b54b4b2e8"},
+    {file = "orjson-3.3.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:e455c5b42a023f4777526c623d2e9ae415084de5130f93aefe689ea482de5f67"},
+    {file = "orjson-3.3.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:8c90083c67653d88b132820719e604250f26ba04229efe3149bf82ba2a08f8cf"},
+    {file = "orjson-3.3.1-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:bc23eed41167b4454cddd51f72a7ee4163c33565c509bb9469adf56384b1cce2"},
+    {file = "orjson-3.3.1-cp37-none-win_amd64.whl", hash = "sha256:3bff4765281da6fa8ddbbe692e5061f950d11aabdfe64837fb53ead4756e9af6"},
+    {file = "orjson-3.3.1-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:1e19907c1ccf82976c2d111f3914a2c0697720b91908e8ef02405e4dc21c662a"},
+    {file = "orjson-3.3.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:aa8332a3ee0fa03a331bea4f28cdcc4d363b53af2ea41630d7eb580422514a1f"},
+    {file = "orjson-3.3.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:4ab9536c3776136303ab9e6432691d970e6aa5d27dbc2b5e0ca0d0db3e12f1c4"},
+    {file = "orjson-3.3.1-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:28dc7e1f89440a68c1ccb937f6f0ae40fa3875de84f747262c00bc18aa25c5ec"},
+    {file = "orjson-3.3.1-cp38-none-win_amd64.whl", hash = "sha256:fa4d5d734e76d9f21a94444fbf1de7eea185b355b324d38c8a7456ce63c3bbeb"},
+    {file = "orjson-3.3.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:b0533d6719b781db7563c478672d91faeac9ea810f30f16ebb5e917c4451b098"},
+    {file = "orjson-3.3.1-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:a7d634eb69083ca5a49baf412625604813f9e3365cb869f445c388d15fe60122"},
+    {file = "orjson-3.3.1.tar.gz", hash = "sha256:149d6a2bc71514826979b9d053f3df0c2397a99e2b87213ba71605a1626d662c"},
 ]
 packaging = [
     {file = "packaging-20.4-py2.py3-none-any.whl", hash = "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181"},
@@ -1182,8 +1354,28 @@ prometheus-client = [
     {file = "prometheus_client-0.8.0.tar.gz", hash = "sha256:c6e6b706833a6bd1fd51711299edee907857be10ece535126a158f911ee80915"},
 ]
 prompt-toolkit = [
-    {file = "prompt_toolkit-3.0.5-py3-none-any.whl", hash = "sha256:df7e9e63aea609b1da3a65641ceaf5bc7d05e0a04de5bd45d05dbeffbabf9e04"},
-    {file = "prompt_toolkit-3.0.5.tar.gz", hash = "sha256:563d1a4140b63ff9dd587bda9557cffb2fe73650205ab6f4383092fb882e7dc8"},
+    {file = "prompt_toolkit-3.0.6-py3-none-any.whl", hash = "sha256:683397077a64cd1f750b71c05afcfc6612a7300cb6932666531e5a54f38ea564"},
+    {file = "prompt_toolkit-3.0.6.tar.gz", hash = "sha256:7630ab85a23302839a0f26b31cc24f518e6155dea1ed395ea61b42c45941b6a6"},
+]
+protobuf = [
+    {file = "protobuf-3.13.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:9c2e63c1743cba12737169c447374fab3dfeb18111a460a8c1a000e35836b18c"},
+    {file = "protobuf-3.13.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:1e834076dfef9e585815757a2c7e4560c7ccc5962b9d09f831214c693a91b463"},
+    {file = "protobuf-3.13.0-cp35-cp35m-macosx_10_9_intel.whl", hash = "sha256:df3932e1834a64b46ebc262e951cd82c3cf0fa936a154f0a42231140d8237060"},
+    {file = "protobuf-3.13.0-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:8c35bcbed1c0d29b127c886790e9d37e845ffc2725cc1db4bd06d70f4e8359f4"},
+    {file = "protobuf-3.13.0-cp35-cp35m-win32.whl", hash = "sha256:339c3a003e3c797bc84499fa32e0aac83c768e67b3de4a5d7a5a9aa3b0da634c"},
+    {file = "protobuf-3.13.0-cp35-cp35m-win_amd64.whl", hash = "sha256:361acd76f0ad38c6e38f14d08775514fbd241316cce08deb2ce914c7dfa1184a"},
+    {file = "protobuf-3.13.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:9edfdc679a3669988ec55a989ff62449f670dfa7018df6ad7f04e8dbacb10630"},
+    {file = "protobuf-3.13.0-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:5db9d3e12b6ede5e601b8d8684a7f9d90581882925c96acf8495957b4f1b204b"},
+    {file = "protobuf-3.13.0-cp36-cp36m-win32.whl", hash = "sha256:c8abd7605185836f6f11f97b21200f8a864f9cb078a193fe3c9e235711d3ff1e"},
+    {file = "protobuf-3.13.0-cp36-cp36m-win_amd64.whl", hash = "sha256:4d1174c9ed303070ad59553f435846a2f877598f59f9afc1b89757bdf846f2a7"},
+    {file = "protobuf-3.13.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0bba42f439bf45c0f600c3c5993666fcb88e8441d011fad80a11df6f324eef33"},
+    {file = "protobuf-3.13.0-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:c0c5ab9c4b1eac0a9b838f1e46038c3175a95b0f2d944385884af72876bd6bc7"},
+    {file = "protobuf-3.13.0-cp37-cp37m-win32.whl", hash = "sha256:f68eb9d03c7d84bd01c790948320b768de8559761897763731294e3bc316decb"},
+    {file = "protobuf-3.13.0-cp37-cp37m-win_amd64.whl", hash = "sha256:91c2d897da84c62816e2f473ece60ebfeab024a16c1751aaf31100127ccd93ec"},
+    {file = "protobuf-3.13.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3dee442884a18c16d023e52e32dd34a8930a889e511af493f6dc7d4d9bf12e4f"},
+    {file = "protobuf-3.13.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:e7662437ca1e0c51b93cadb988f9b353fa6b8013c0385d63a70c8a77d84da5f9"},
+    {file = "protobuf-3.13.0-py2.py3-none-any.whl", hash = "sha256:d69697acac76d9f250ab745b46c725edf3e98ac24763990b24d58c16c642947a"},
+    {file = "protobuf-3.13.0.tar.gz", hash = "sha256:6a82e0c8bb2bf58f606040cc5814e07715b2094caeba281e2e7d0b0e2e397db5"},
 ]
 psutil = [
     {file = "psutil-5.7.2-cp27-none-win32.whl", hash = "sha256:f2018461733b23f308c298653c8903d32aaad7873d25e1d228765e91ae42c3f2"},
@@ -1213,6 +1405,10 @@ pyaudio = [
     {file = "PyAudio-0.2.11-cp36-cp36m-win_amd64.whl", hash = "sha256:2a19bdb8ec1445b4f3e4b7b109e0e4cec1fd1f1ce588592aeb6db0b58d4fb3b0"},
     {file = "PyAudio-0.2.11.tar.gz", hash = "sha256:93bfde30e0b64e63a46f2fd77e85c41fd51182a4a3413d9edfaf9ffaa26efb74"},
 ]
+pycparser = [
+    {file = "pycparser-2.20-py2.py3-none-any.whl", hash = "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705"},
+    {file = "pycparser-2.20.tar.gz", hash = "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0"},
+]
 pygments = [
     {file = "Pygments-2.6.1-py3-none-any.whl", hash = "sha256:ff7a40b4860b727ab48fad6360eb351cc1b33cbf9b15a0f689ca5353e9463324"},
     {file = "Pygments-2.6.1.tar.gz", hash = "sha256:647344a061c249a3b74e230c739f434d7ea4d8b1d5f3721bc0f3558049b38f44"},
@@ -1268,38 +1464,38 @@ pyyaml = [
     {file = "PyYAML-5.3.1.tar.gz", hash = "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d"},
 ]
 pyzmq = [
-    {file = "pyzmq-19.0.1-cp27-cp27m-macosx_10_9_intel.whl", hash = "sha256:58688a2dfa044fad608a8e70ba8d019d0b872ec2acd75b7b5e37da8905605891"},
-    {file = "pyzmq-19.0.1-cp27-cp27m-win32.whl", hash = "sha256:87c78f6936e2654397ca2979c1d323ee4a889eef536cc77a938c6b5be33351a7"},
-    {file = "pyzmq-19.0.1-cp27-cp27m-win_amd64.whl", hash = "sha256:97b6255ae77328d0e80593681826a0479cb7bac0ba8251b4dd882f5145a2293a"},
-    {file = "pyzmq-19.0.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:15b4cb21118f4589c4db8be4ac12b21c8b4d0d42b3ee435d47f686c32fe2e91f"},
-    {file = "pyzmq-19.0.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:931339ac2000d12fe212e64f98ce291e81a7ec6c73b125f17cf08415b753c087"},
-    {file = "pyzmq-19.0.1-cp35-cp35m-macosx_10_9_intel.whl", hash = "sha256:2a88b8fabd9cc35bd59194a7723f3122166811ece8b74018147a4ed8489e6421"},
-    {file = "pyzmq-19.0.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:bafd651b557dd81d89bd5f9c678872f3e7b7255c1c751b78d520df2caac80230"},
-    {file = "pyzmq-19.0.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:8952f6ba6ae598e792703f3134af5a01af8f5c7cf07e9a148f05a12b02412cea"},
-    {file = "pyzmq-19.0.1-cp35-cp35m-win32.whl", hash = "sha256:54aa24fd60c4262286fc64ca632f9e747c7cc3a3a1144827490e1dc9b8a3a960"},
-    {file = "pyzmq-19.0.1-cp35-cp35m-win_amd64.whl", hash = "sha256:dcbc3f30c11c60d709c30a213dc56e88ac016fe76ac6768e64717bd976072566"},
-    {file = "pyzmq-19.0.1-cp36-cp36m-macosx_10_9_intel.whl", hash = "sha256:6ca519309703e95d55965735a667809bbb65f52beda2fdb6312385d3e7a6d234"},
-    {file = "pyzmq-19.0.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:4ee0bfd82077a3ff11c985369529b12853a4064320523f8e5079b630f9551448"},
-    {file = "pyzmq-19.0.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:ba6f24431b569aec674ede49cad197cad59571c12deed6ad8e3c596da8288217"},
-    {file = "pyzmq-19.0.1-cp36-cp36m-win32.whl", hash = "sha256:956775444d01331c7eb412c5fb9bb62130dfaac77e09f32764ea1865234e2ca9"},
-    {file = "pyzmq-19.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:b08780e3a55215873b3b8e6e7ca8987f14c902a24b6ac081b344fd430d6ca7cd"},
-    {file = "pyzmq-19.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:21f7d91f3536f480cb2c10d0756bfa717927090b7fb863e6323f766e5461ee1c"},
-    {file = "pyzmq-19.0.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:bfff5ffff051f5aa47ba3b379d87bd051c3196b0c8a603e8b7ed68a6b4f217ec"},
-    {file = "pyzmq-19.0.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:07fb8fe6826a229dada876956590135871de60dbc7de5a18c3bcce2ed1f03c98"},
-    {file = "pyzmq-19.0.1-cp37-cp37m-win32.whl", hash = "sha256:342fb8a1dddc569bc361387782e8088071593e7eaf3e3ecf7d6bd4976edff112"},
-    {file = "pyzmq-19.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:faee2604f279d31312bc455f3d024f160b6168b9c1dde22bf62d8c88a4deca8e"},
-    {file = "pyzmq-19.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:5b9d21fc56c8aacd2e6d14738021a9d64f3f69b30578a99325a728e38a349f85"},
-    {file = "pyzmq-19.0.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:af0c02cf49f4f9eedf38edb4f3b6bb621d83026e7e5d76eb5526cc5333782fd6"},
-    {file = "pyzmq-19.0.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:5f1f2eb22aab606f808163eb1d537ac9a0ba4283fbeb7a62eb48d9103cf015c2"},
-    {file = "pyzmq-19.0.1-cp38-cp38-win32.whl", hash = "sha256:f9d7e742fb0196992477415bb34366c12e9bb9a0699b8b3f221ff93b213d7bec"},
-    {file = "pyzmq-19.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:5b99c2ae8089ef50223c28bac57510c163bfdff158c9e90764f812b94e69a0e6"},
-    {file = "pyzmq-19.0.1-pp27-pypy_73-macosx_10_9_x86_64.whl", hash = "sha256:cf5d689ba9513b9753959164cf500079383bc18859f58bf8ce06d8d4bef2b054"},
-    {file = "pyzmq-19.0.1-pp36-pypy36_pp73-macosx_10_9_x86_64.whl", hash = "sha256:aaa8b40b676576fd7806839a5de8e6d5d1b74981e6376d862af6c117af2a3c10"},
-    {file = "pyzmq-19.0.1.tar.gz", hash = "sha256:13a5638ab24d628a6ade8f794195e1a1acd573496c3b85af2f1183603b7bf5e0"},
+    {file = "pyzmq-19.0.2-cp27-cp27m-macosx_10_9_intel.whl", hash = "sha256:59f1e54627483dcf61c663941d94c4af9bf4163aec334171686cdaee67974fe5"},
+    {file = "pyzmq-19.0.2-cp27-cp27m-win32.whl", hash = "sha256:c36ffe1e5aa35a1af6a96640d723d0d211c5f48841735c2aa8d034204e87eb87"},
+    {file = "pyzmq-19.0.2-cp27-cp27m-win_amd64.whl", hash = "sha256:0a422fc290d03958899743db091f8154958410fc76ce7ee0ceb66150f72c2c97"},
+    {file = "pyzmq-19.0.2-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:c20dd60b9428f532bc59f2ef6d3b1029a28fc790d408af82f871a7db03e722ff"},
+    {file = "pyzmq-19.0.2-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:d46fb17f5693244de83e434648b3dbb4f4b0fec88415d6cbab1c1452b6f2ae17"},
+    {file = "pyzmq-19.0.2-cp35-cp35m-macosx_10_9_intel.whl", hash = "sha256:f1a25a61495b6f7bb986accc5b597a3541d9bd3ef0016f50be16dbb32025b302"},
+    {file = "pyzmq-19.0.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:ab0d01148d13854de716786ca73701012e07dff4dfbbd68c4e06d8888743526e"},
+    {file = "pyzmq-19.0.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:720d2b6083498a9281eaee3f2927486e9fe02cd16d13a844f2e95217f243efea"},
+    {file = "pyzmq-19.0.2-cp35-cp35m-win32.whl", hash = "sha256:29d51279060d0a70f551663bc592418bcad7f4be4eea7b324f6dd81de05cb4c1"},
+    {file = "pyzmq-19.0.2-cp35-cp35m-win_amd64.whl", hash = "sha256:5120c64646e75f6db20cc16b9a94203926ead5d633de9feba4f137004241221d"},
+    {file = "pyzmq-19.0.2-cp36-cp36m-macosx_10_9_intel.whl", hash = "sha256:8a6ada5a3f719bf46a04ba38595073df8d6b067316c011180102ba2a1925f5b5"},
+    {file = "pyzmq-19.0.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:fa411b1d8f371d3a49d31b0789eb6da2537dadbb2aef74a43aa99a78195c3f76"},
+    {file = "pyzmq-19.0.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:00dca814469436455399660247d74045172955459c0bd49b54a540ce4d652185"},
+    {file = "pyzmq-19.0.2-cp36-cp36m-win32.whl", hash = "sha256:046b92e860914e39612e84fa760fc3f16054d268c11e0e25dcb011fb1bc6a075"},
+    {file = "pyzmq-19.0.2-cp36-cp36m-win_amd64.whl", hash = "sha256:99cc0e339a731c6a34109e5c4072aaa06d8e32c0b93dc2c2d90345dd45fa196c"},
+    {file = "pyzmq-19.0.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e36f12f503511d72d9bdfae11cadbadca22ff632ff67c1b5459f69756a029c19"},
+    {file = "pyzmq-19.0.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:c40fbb2b9933369e994b837ee72193d6a4c35dfb9a7c573257ef7ff28961272c"},
+    {file = "pyzmq-19.0.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:5d9fc809aa8d636e757e4ced2302569d6e60e9b9c26114a83f0d9d6519c40493"},
+    {file = "pyzmq-19.0.2-cp37-cp37m-win32.whl", hash = "sha256:3fa6debf4bf9412e59353defad1f8035a1e68b66095a94ead8f7a61ae90b2675"},
+    {file = "pyzmq-19.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:73483a2caaa0264ac717af33d6fb3f143d8379e60a422730ee8d010526ce1913"},
+    {file = "pyzmq-19.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:36ab114021c0cab1a423fe6689355e8f813979f2c750968833b318c1fa10a0fd"},
+    {file = "pyzmq-19.0.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:8b66b94fe6243d2d1d89bca336b2424399aac57932858b9a30309803ffc28112"},
+    {file = "pyzmq-19.0.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:654d3e06a4edc566b416c10293064732516cf8871a4522e0a2ba00cc2a2e600c"},
+    {file = "pyzmq-19.0.2-cp38-cp38-win32.whl", hash = "sha256:276ad604bffd70992a386a84bea34883e696a6b22e7378053e5d3227321d9702"},
+    {file = "pyzmq-19.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:09d24a80ccb8cbda1af6ed8eb26b005b6743e58e9290566d2a6841f4e31fa8e0"},
+    {file = "pyzmq-19.0.2-pp27-pypy_73-macosx_10_9_x86_64.whl", hash = "sha256:c1a31cd42905b405530e92bdb70a8a56f048c8a371728b8acf9d746ecd4482c0"},
+    {file = "pyzmq-19.0.2-pp36-pypy36_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a7e7f930039ee0c4c26e4dfee015f20bd6919cd8b97c9cd7afbde2923a5167b6"},
+    {file = "pyzmq-19.0.2.tar.gz", hash = "sha256:296540a065c8c21b26d63e3cea2d1d57902373b16e4256afe46422691903a438"},
 ]
 qtconsole = [
-    {file = "qtconsole-4.7.5-py2.py3-none-any.whl", hash = "sha256:4f43d0b049eacb7d723772847f0c465feccce0ccb398871a6e146001a22bad23"},
-    {file = "qtconsole-4.7.5.tar.gz", hash = "sha256:f5cb275d30fc8085e2d1d18bc363e5ba0ce6e559bf37d7d6727b773134298754"},
+    {file = "qtconsole-4.7.6-py2.py3-none-any.whl", hash = "sha256:570b9e1dd4f9b727699b0ed04c6943d9d32d5a2085aa69d82d814e039bbcf74b"},
+    {file = "qtconsole-4.7.6.tar.gz", hash = "sha256:6c24397c19a49a5cf69582c931db4b0f6b00a78530a2bfd122936f2ebfae2fef"},
 ]
 qtpy = [
     {file = "QtPy-1.9.0-py2.py3-none-any.whl", hash = "sha256:fa0b8363b363e89b2a6f49eddc162a04c0699ae95e109a6be3bb145a913190ea"},
@@ -1323,6 +1519,7 @@ regex = [
     {file = "regex-2020.7.14-cp38-cp38-manylinux1_i686.whl", hash = "sha256:5ea81ea3dbd6767873c611687141ec7b06ed8bab43f68fad5b7be184a920dc99"},
     {file = "regex-2020.7.14-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:bbb332d45b32df41200380fff14712cb6093b61bd142272a10b16778c418e98e"},
     {file = "regex-2020.7.14-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:c11d6033115dc4887c456565303f540c44197f4fc1a2bfb192224a301534888e"},
+    {file = "regex-2020.7.14-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:75aaa27aa521a182824d89e5ab0a1d16ca207318a6b65042b046053cfc8ed07a"},
     {file = "regex-2020.7.14-cp38-cp38-win32.whl", hash = "sha256:d6cff2276e502b86a25fd10c2a96973fdb45c7a977dca2138d661417f3728341"},
     {file = "regex-2020.7.14-cp38-cp38-win_amd64.whl", hash = "sha256:7a2dd66d2d4df34fa82c9dc85657c5e019b87932019947faece7983f2089a840"},
     {file = "regex-2020.7.14.tar.gz", hash = "sha256:3a3af27a8d23143c49a3420efe5b3f8cf1a48c6fc8bc6856b03f638abc1833bb"},
@@ -1363,8 +1560,8 @@ tornado = [
     {file = "tornado-6.0.4.tar.gz", hash = "sha256:0fe2d45ba43b00a41cd73f8be321a44936dc1aba233dee979f17a042b83eb6dc"},
 ]
 tqdm = [
-    {file = "tqdm-4.48.0-py2.py3-none-any.whl", hash = "sha256:fcb7cb5b729b60a27f300b15c1ffd4744f080fb483b88f31dc8654b082cc8ea5"},
-    {file = "tqdm-4.48.0.tar.gz", hash = "sha256:6baa75a88582b1db6d34ce4690da5501d2a1cb65c34664840a456b2c9f794d29"},
+    {file = "tqdm-4.48.2-py2.py3-none-any.whl", hash = "sha256:1a336d2b829be50e46b84668691e0a2719f26c97c62846298dd5ae2937e4d5cf"},
+    {file = "tqdm-4.48.2.tar.gz", hash = "sha256:564d632ea2b9cb52979f7956e093e831c28d441c11751682f84c86fc46e4fd21"},
 ]
 traitlets = [
     {file = "traitlets-4.3.3-py2.py3-none-any.whl", hash = "sha256:70b4c6a1d9019d7b4f6846832288f86998aa3b9207c6821f3578a6a6a467fe44"},
diff --git a/pyproject.toml b/pyproject.toml
index a5ecd48..f776a08 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -6,6 +6,7 @@ authors = ["Vikash Kothary <kothary.vikash@gmail.com>"]
 
 [tool.poetry.dependencies]
 python = "^3.7"
+anki = "^2.1.32"
 beautifulsoup4 = "^4.9.1"
 requests = "^2.24.0"
 markdown = "^3.2.2"
diff --git a/src/requirements-dev.txt b/src/requirements-dev.txt
index 44b3e2b..49dca27 100644
--- a/src/requirements-dev.txt
+++ b/src/requirements-dev.txt
@@ -1,9 +1,13 @@
+anki==2.1.32
+ankirspy==2.1.32
 appnope==0.1.0; sys_platform == "darwin" or platform_system == "Darwin" or python_version >= "3.3" and sys_platform == "darwin"
-attrs==19.3.0
+argon2-cffi==20.1.0
+attrs==20.1.0
 backcall==0.2.0
 beautifulsoup4==4.9.1
 bleach==3.1.5
 certifi==2020.6.20
+cffi==1.14.2
 chardet==3.0.4
 click==7.1.2
 colorama==0.4.3; python_version >= "3.3" and sys_platform == "win32" or sys_platform == "win32"
@@ -15,7 +19,7 @@ future==0.18.2
 idna==2.10
 importlib-metadata==1.7.0; python_version < "3.8"
 ipykernel==5.3.4
-ipython==7.16.1
+ipython==7.17.0
 ipython-genutils==0.2.0
 ipywidgets==7.5.1
 jedi==0.17.2
@@ -24,12 +28,12 @@ joblib==0.16.0; python_version > "2.7"
 json5==0.9.5
 jsonschema==3.2.0
 jupyter==1.0.0
-jupyter-client==6.1.6
+jupyter-client==6.1.7
 jupyter-console==6.1.0
 jupyter-core==4.6.3
-jupyterlab==2.2.2
+jupyterlab==2.2.6
 jupyterlab-server==1.2.0
-livereload==2.6.2
+livereload==2.6.3
 lunr==0.5.8
 markdown==3.2.2
 markupsafe==1.1.1
@@ -38,17 +42,20 @@ mkdocs==1.1.2
 nbconvert==5.6.1
 nbformat==5.0.7
 nltk==3.5; python_version > "2.7"
-notebook==6.0.3
+notebook==6.1.3
+orjson==3.3.1; platform_machine == "x86_64"
 packaging==20.4
 pandocfilters==1.4.2
 parso==0.7.1
 pexpect==4.8.0; python_version >= "3.3" and sys_platform != "win32" or sys_platform != "win32"
 pickleshare==0.7.5
 prometheus-client==0.8.0
-prompt-toolkit==3.0.5
+prompt-toolkit==3.0.6
+protobuf==3.13.0
 psutil==5.7.2
 ptyprocess==0.6.0; sys_platform != "win32" or os_name != "nt" or python_version >= "3.3" and sys_platform != "win32"
 pyaudio==0.2.11
+pycparser==2.20
 pygments==2.6.1
 pyparsing==2.4.7
 pyrsistent==0.16.0
@@ -56,8 +63,8 @@ python-dateutil==2.8.1
 pywin32==228; sys_platform == "win32"
 pywinpty==0.5.7; os_name == "nt"
 pyyaml==5.3.1
-pyzmq==19.0.1
-qtconsole==4.7.5
+pyzmq==19.0.2
+qtconsole==4.7.6
 qtpy==1.9.0
 regex==2020.7.14; python_version > "2.7"
 requests==2.24.0
@@ -67,7 +74,7 @@ soupsieve==1.9.6
 terminado==0.8.3
 testpath==0.4.4
 tornado==6.0.4
-tqdm==4.48.0; python_version > "2.7"
+tqdm==4.48.2; python_version > "2.7"
 traitlets==4.3.3
 urllib3==1.25.10
 wcwidth==0.2.5
diff --git a/src/requirements.txt b/src/requirements.txt
index f66853b..3a1dc74 100644
--- a/src/requirements.txt
+++ b/src/requirements.txt
@@ -1,3 +1,5 @@
+anki==2.1.32
+ankirspy==2.1.32
 beautifulsoup4==4.9.1
 certifi==2020.6.20
 chardet==3.0.4
@@ -6,10 +8,13 @@ distro==1.5.0
 idna==2.10
 importlib-metadata==1.7.0; python_version < "3.8"
 markdown==3.2.2
+orjson==3.3.1; platform_machine == "x86_64"
+protobuf==3.13.0
 psutil==5.7.2
 pyaudio==0.2.11
 requests==2.24.0
 send2trash==1.5.0
+six==1.15.0
 soupsieve==1.9.6
 urllib3==1.25.10
 webob==1.8.6

From 70bfaa171970dffd1d0cfb9ac774fda474785338 Mon Sep 17 00:00:00 2001
From: Karsten Lehmann <mail@kalehmann.de>
Date: Mon, 31 Aug 2020 16:50:53 +0200
Subject: [PATCH 30/38] Further clarified configuration for Nginx proxy

---
 README.md | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index 7aa73fc..06c8c84 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,5 @@
 ankisyncd
-=========
+11;rgb:2323/2727/2929=========
 
 [![Documentation Status](https://readthedocs.org/projects/anki-sync-server/badge/?version=latest)](https://anki-sync-server.readthedocs.io/?badge=latest)
 [![Gitter](https://badges.gitter.im/ankicommunity/community.svg)](https://gitter.im/ankicommunity/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
@@ -52,20 +52,31 @@ Installing
         $ ./ankisyncctl.py adduser <username>
 
 4. Setup a proxy to unchunk the requests.
-Webob does not support the header "Transfer-Encoding: chunked" used by Anki and
-therefore ankisyncd sees chunked requests as empty. To solve this problem setup
-Nginx (or any other webserver of your choice) and configure it to "unchunk" the
-requests to ankisyncd:
 
+    Webob does not support the header "Transfer-Encoding: chunked" used by Anki
+    and therefore ankisyncd sees chunked requests as empty. To solve this problem
+    setup Nginx (or any other webserver of your choice) and configure it to
+    "unchunk" the requests for ankisyncd.
+
+    For example, if you use Nginx  on the same machine as ankisyncd, you first
+    have to change the port in `ankisyncd.conf` to something other than `27701`.
+    Then configure Nginx to listen on port `27701` and forward the unchunked
+    requests to ankisyncd.
+
+    An example configuration with ankisyncd running on the same machine as Nginx
+    and listening on port `27702` may look like:
+
+    ```
     server {
         listen      27701;
         server_name default;
 
         location / {
             proxy_http_version 1.0;
-            proxy_pass         http://ankisyncd:27701/;
+            proxy_pass         http://localhost:27702/;
         }
     }
+    ```
 
 5. Run ankisyncd:
 

From 6f29fce600f2dcdc496eab4cfc5f3dc566e0861c Mon Sep 17 00:00:00 2001
From: Karsten Lehmann <mail@kalehmann.de>
Date: Tue, 1 Sep 2020 23:53:59 +0200
Subject: [PATCH 31/38] replaced relative with absolute import

---
 src/ankisyncd/sync_app.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/ankisyncd/sync_app.py b/src/ankisyncd/sync_app.py
index 3cbe3aa..90e70a2 100644
--- a/src/ankisyncd/sync_app.py
+++ b/src/ankisyncd/sync_app.py
@@ -38,11 +38,10 @@ import anki.db
 import anki.utils
 from anki.consts import REM_CARD, REM_NOTE
 
-from ankisyncd.users import get_user_manager
-from ankisyncd.sessions import get_session_manager
 from ankisyncd.full_sync import get_full_sync_manager
-
-from .sync import Syncer, SYNC_VER, SYNC_ZIP_SIZE, SYNC_ZIP_COUNT
+from ankisyncd.sessions import get_session_manager
+from ankisyncd.sync import Syncer, SYNC_VER, SYNC_ZIP_SIZE, SYNC_ZIP_COUNT
+from ankisyncd.users import get_user_manager
 
 logger = logging.getLogger("ankisyncd")
 

From 3ec37ec80f4a7147b908c3fc104596fcfc19c45c Mon Sep 17 00:00:00 2001
From: Karsten Lehmann <mail@kalehmann.de>
Date: Tue, 1 Sep 2020 23:56:44 +0200
Subject: [PATCH 32/38] Removed artifact from README

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 06c8c84..c33876f 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,5 @@
 ankisyncd
-11;rgb:2323/2727/2929=========
+=========
 
 [![Documentation Status](https://readthedocs.org/projects/anki-sync-server/badge/?version=latest)](https://anki-sync-server.readthedocs.io/?badge=latest)
 [![Gitter](https://badges.gitter.im/ankicommunity/community.svg)](https://gitter.im/ankicommunity/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)

From 65e9bbf747c3d5d9af428f216b160e492588b607 Mon Sep 17 00:00:00 2001
From: Karsten Lehmann <mail@kalehmann.de>
Date: Wed, 2 Sep 2020 00:03:58 +0200
Subject: [PATCH 33/38] Updated docs about installation of requirements

---
 README.md | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/README.md b/README.md
index c33876f..1c8e75c 100644
--- a/README.md
+++ b/README.md
@@ -38,12 +38,10 @@ It supports Python 3 and Anki 2.1.
 
 Installing
 ----------
-
-0. Install the current version of Anki.
    
 1. Install the dependencies:
 
-        $ pip install webob
+        $ pip install -r src/requirements.txt
 
 2. Modify ankisyncd.conf according to your needs
 

From 7deda95d77b488b26242b64aafa5ebd709fa1e24 Mon Sep 17 00:00:00 2001
From: Karsten Lehmann <mail@kalehmann.de>
Date: Wed, 2 Sep 2020 00:07:57 +0200
Subject: [PATCH 34/38] Removed whole section about ankisyncd without pyaudio
 from the README

---
 README.md | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/README.md b/README.md
index 1c8e75c..61dd905 100644
--- a/README.md
+++ b/README.md
@@ -29,9 +29,6 @@ It supports Python 3 and Anki 2.1.
    - [Anki 2.1](#anki-21)
    - [Anki 2.0](#anki-20)
    - [AnkiDroid](#ankidroid)
- - [Running `ankisyncd` without `pyaudio`](#running-ankisyncd-without-pyaudio)
-   - [Anki ≥2.1.9](#anki-219)
-   - [Older versions](#older-versions)
  - [ENVVAR configuration overrides](#envvar-configuration-overrides)
  - [Support for other database backends](#support-for-other-database-backends)
 </details>
@@ -142,13 +139,6 @@ Even though the AnkiDroid interface will request an email address, this is not
 required; it will simply be the username you configured with `ankisyncctl.py
 adduser`.
 
-Running `ankisyncd` without `pyaudio`
--------------------------------------
-
-`ankisyncd` doesn't use the audio recording feature of Anki, so if you don't
-want to install PortAudio, you can edit some files in the `anki-bundled`
-directory to exclude `pyaudio`:
-
 ENVVAR configuration overrides
 ------------------------------
 

From 614f209f98d54fa0c1eb6724fa39a4440a54cf16 Mon Sep 17 00:00:00 2001
From: Karsten Lehmann <mail@kalehmann.de>
Date: Wed, 2 Sep 2020 18:30:56 +0200
Subject: [PATCH 35/38] Removed unused post- and prehooks

---
 src/ankisyncd/sync_app.py | 53 ---------------------------------------
 1 file changed, 53 deletions(-)

diff --git a/src/ankisyncd/sync_app.py b/src/ankisyncd/sync_app.py
index 90e70a2..d28bd6a 100644
--- a/src/ankisyncd/sync_app.py
+++ b/src/ankisyncd/sync_app.py
@@ -396,9 +396,6 @@ class SyncApp:
         self.base_media_url  = config['base_media_url']
         self.setup_new_collection = None
 
-        self.prehooks = {}
-        self.posthooks = {}
-
         self.user_manager = get_user_manager(config)
         self.session_manager = get_session_manager(config)
         self.full_sync_manager = get_full_sync_manager(config)
@@ -410,39 +407,6 @@ class SyncApp:
         if not self.base_media_url.endswith('/'):
             self.base_media_url += '/'
 
-    # backwards compat
-    @property
-    def hook_pre_sync(self):
-        return self.prehooks.get("start")
-
-    @hook_pre_sync.setter
-    def hook_pre_sync(self, value):
-        self.prehooks['start'] = value
-
-    @property
-    def hook_post_sync(self):
-        return self.posthooks.get("finish")
-
-    @hook_post_sync.setter
-    def hook_post_sync(self, value):
-        self.posthooks['finish'] = value
-
-    @property
-    def hook_upload(self):
-        return self.prehooks.get("upload")
-
-    @hook_upload.setter
-    def hook_upload(self, value):
-        self.prehooks['upload'] = value
-
-    @property
-    def hook_download(self):
-        return self.posthooks.get("download")
-
-    @hook_download.setter
-    def hook_download(self, value):
-        self.posthooks['download'] = value
-
     def generateHostKey(self, username):
         """Generates a new host key to be used by the given username to identify their session.
         This values is random."""
@@ -549,39 +513,22 @@ class SyncApp:
 
                     self.session_manager.save(hkey, session)
                     session = self.session_manager.load(hkey, self.create_session)
-
                 thread = session.get_thread()
-
-                if url in self.prehooks:
-                    thread.execute(self.prehooks[url], [session])
-
                 result = self._execute_handler_method_in_thread(url, data, session)
-
                 # If it's a complex data type, we convert it to JSON
                 if type(result) not in (str, bytes, Response):
                     result = json.dumps(result)
 
-                if url in self.posthooks:
-                    thread.execute(self.posthooks[url], [session])
-
                 return result
 
             elif url == 'upload':
                 thread = session.get_thread()
-                if url in self.prehooks:
-                    thread.execute(self.prehooks[url], [session])
                 result = thread.execute(self.operation_upload, [data['data'], session])
-                if url in self.posthooks:
-                    thread.execute(self.posthooks[url], [session])
                 return result
 
             elif url == 'download':
                 thread = session.get_thread()
-                if url in self.prehooks:
-                    thread.execute(self.prehooks[url], [session])
                 result = thread.execute(self.operation_download, [session])
-                if url in self.posthooks:
-                    thread.execute(self.posthooks[url], [session])
                 return result
 
             # This was one of our operations but it didn't get handled... Oops!

From 0d8f3c6eeaa1296e8bbe3065f62c3279d2570c20 Mon Sep 17 00:00:00 2001
From: Karsten Lehmann <mail@kalehmann.de>
Date: Wed, 2 Sep 2020 18:33:48 +0200
Subject: [PATCH 36/38] Removed runHook calls from sync code

These hooks were only used on the client.
---
 src/ankisyncd/sync.py | 19 +++----------------
 1 file changed, 3 insertions(+), 16 deletions(-)

diff --git a/src/ankisyncd/sync.py b/src/ankisyncd/sync.py
index 3eda801..183bcc4 100644
--- a/src/ankisyncd/sync.py
+++ b/src/ankisyncd/sync.py
@@ -16,7 +16,6 @@ from anki.utils import ids2str, intTime, platDesc, checksum, devMode
 from anki.consts import *
 from anki.config import ConfigManager
 from anki.utils import versionWithBuild
-from anki.hooks import runHook
 import anki
 from anki.lang import ngettext
 
@@ -50,7 +49,6 @@ class Syncer(object):
         self.col.save()
 
         # step 1: login & metadata
-        runHook("sync", "login")
         meta = self.server.meta()
         self.col.log("rmeta", meta)
         if not meta:
@@ -90,7 +88,6 @@ class Syncer(object):
             self.col.log("basic check")
             return "basicCheckFailed"
         # step 2: startup and deletions
-        runHook("sync", "meta")
         rrem = self.server.start(minUsn=self.minUsn, lnewer=self.lnewer)
 
         # apply deletions to server
@@ -107,25 +104,20 @@ class Syncer(object):
         rchg = self.server.applyChanges(changes=lchg)
         self.mergeChanges(lchg, rchg)
         # step 3: stream large tables from server
-        runHook("sync", "server")
-        while 1:
-            runHook("sync", "stream")
+        while True:
             chunk = self.server.chunk()
             self.col.log("server chunk", chunk)
             self.applyChunk(chunk=chunk)
             if chunk['done']:
                 break
         # step 4: stream to server
-        runHook("sync", "client")
-        while 1:
-            runHook("sync", "stream")
+        while True:
             chunk = self.chunk()
             self.col.log("client chunk", chunk)
             self.server.applyChunk(chunk=chunk)
             if chunk['done']:
                 break
         # step 5: sanity check
-        runHook("sync", "sanity")
         c = self.sanityCheck()
         ret = self.server.sanityCheck2(client=c)
         if ret['status'] != "ok":
@@ -135,7 +127,6 @@ class Syncer(object):
             self.col.save()
             return "sanityCheckFailed"
         # finalize
-        runHook("sync", "finalize")
         mod = self.server.finish()
         self.finish(mod)
         return "success"
@@ -449,7 +440,6 @@ class AnkiRequestsClient(object):
 
         buf = io.BytesIO()
         for chunk in resp.iter_content(chunk_size=HTTP_BUF_SIZE):
-            runHook("httpRecv", len(chunk))
             buf.write(chunk)
         return buf.getvalue()
 
@@ -467,7 +457,7 @@ if os.environ.get("ANKI_NOVERIFYSSL"):
 class _MonitoringFile(io.BufferedReader):
     def read(self, size=-1):
         data = io.BufferedReader.read(self, HTTP_BUF_SIZE)
-        runHook("httpSend", len(data))
+
         return data
 
 # HTTP syncing tools
@@ -632,13 +622,11 @@ class FullSyncer(HttpSyncer):
         self.col = col
 
     def download(self):
-        runHook("sync", "download")
         localNotEmpty = self.col.db.scalar("select 1 from cards")
         self.col.close()
         cont = self.req("download")
         tpath = self.col.path + ".tmp"
         if cont == "upgradeRequired":
-            runHook("sync", "upgradeRequired")
             return
         open(tpath, "wb").write(cont)
         # check the received file is ok
@@ -657,7 +645,6 @@ class FullSyncer(HttpSyncer):
 
     def upload(self):
         "True if upload successful."
-        runHook("sync", "upload")
         # make sure it's ok before we try to upload
         if self.col.db.scalar("pragma integrity_check") != "ok":
             return False

From e9e06454169c72c82070a1d5032c9149dd3ddce1 Mon Sep 17 00:00:00 2001
From: Karsten Lehmann <mail@kalehmann.de>
Date: Wed, 2 Sep 2020 18:39:50 +0200
Subject: [PATCH 37/38] Removed unused sync method

---
 src/ankisyncd/sync.py | 105 ------------------------------------------
 1 file changed, 105 deletions(-)

diff --git a/src/ankisyncd/sync.py b/src/ankisyncd/sync.py
index 183bcc4..66be132 100644
--- a/src/ankisyncd/sync.py
+++ b/src/ankisyncd/sync.py
@@ -40,111 +40,6 @@ class Syncer(object):
         self.col = col
         self.server = server
 
-    def sync(self):
-        "Returns 'noChanges', 'fullSync', 'success', etc"
-        self.syncMsg = ""
-        self.uname = ""
-        # if the deck has any pending changes, flush them first and bump mod
-        # time
-        self.col.save()
-
-        # step 1: login & metadata
-        meta = self.server.meta()
-        self.col.log("rmeta", meta)
-        if not meta:
-            return "badAuth"
-        # server requested abort?
-        self.syncMsg = meta['msg']
-        if not meta['cont']:
-            return "serverAbort"
-        else:
-            # don't abort, but if 'msg' is not blank, gui should show 'msg'
-            # after sync finishes and wait for confirmation before hiding
-            pass
-        rscm = meta['scm']
-        rts = meta['ts']
-        self.rmod = meta['mod']
-        self.maxUsn = meta['usn']
-        self.uname = meta.get("uname", "")
-        self.hostNum = meta.get("hostNum")
-        meta = self.meta()
-        self.col.log("lmeta", meta)
-        self.lmod = meta['mod']
-        self.minUsn = meta['usn']
-        lscm = meta['scm']
-        lts = meta['ts']
-        if abs(rts - lts) > 300:
-            self.col.log("clock off")
-            return "clockOff"
-        if self.lmod == self.rmod:
-            self.col.log("no changes")
-            return "noChanges"
-        elif lscm != rscm:
-            self.col.log("schema diff")
-            return "fullSync"
-        self.lnewer = self.lmod > self.rmod
-        # step 1.5: check collection is valid
-        if not self.col.basicCheck():
-            self.col.log("basic check")
-            return "basicCheckFailed"
-        # step 2: startup and deletions
-        rrem = self.server.start(minUsn=self.minUsn, lnewer=self.lnewer)
-
-        # apply deletions to server
-        lgraves = self.removed()
-        while lgraves:
-            gchunk, lgraves = self._gravesChunk(lgraves)
-            self.server.applyGraves(chunk=gchunk)
-
-        # then apply server deletions here
-        self.remove(rrem)
-
-        # ...and small objects
-        lchg = self.changes()
-        rchg = self.server.applyChanges(changes=lchg)
-        self.mergeChanges(lchg, rchg)
-        # step 3: stream large tables from server
-        while True:
-            chunk = self.server.chunk()
-            self.col.log("server chunk", chunk)
-            self.applyChunk(chunk=chunk)
-            if chunk['done']:
-                break
-        # step 4: stream to server
-        while True:
-            chunk = self.chunk()
-            self.col.log("client chunk", chunk)
-            self.server.applyChunk(chunk=chunk)
-            if chunk['done']:
-                break
-        # step 5: sanity check
-        c = self.sanityCheck()
-        ret = self.server.sanityCheck2(client=c)
-        if ret['status'] != "ok":
-            # roll back and force full sync
-            self.col.rollback()
-            self.col.modSchema(False)
-            self.col.save()
-            return "sanityCheckFailed"
-        # finalize
-        mod = self.server.finish()
-        self.finish(mod)
-        return "success"
-
-    def _gravesChunk(self, graves):
-        lim = 250
-        chunk = dict(notes=[], cards=[], decks=[])
-        for cat in "notes", "cards", "decks":
-            if lim and graves[cat]:
-                chunk[cat] = graves[cat][:lim]
-                graves[cat] = graves[cat][lim:]
-                lim -= len(chunk[cat])
-
-        # anything remaining?
-        if graves['notes'] or graves['cards'] or graves['decks']:
-            return chunk, graves
-        return chunk, None
-
     def meta(self):
         return dict(
             mod=self.col.mod,

From f51005032a24f42c7b3fc138273b0f1b0147dd35 Mon Sep 17 00:00:00 2001
From: Karsten Lehmann <mail@kalehmann.de>
Date: Wed, 2 Sep 2020 18:43:15 +0200
Subject: [PATCH 38/38] Removed explicit object parent from FullSyncManager

---
 src/ankisyncd/full_sync.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ankisyncd/full_sync.py b/src/ankisyncd/full_sync.py
index 9e7c3cc..a6c9b9d 100644
--- a/src/ankisyncd/full_sync.py
+++ b/src/ankisyncd/full_sync.py
@@ -13,7 +13,7 @@ from anki.collection import Collection
 logger = logging.getLogger("ankisyncd.media")
 logger.setLevel(1)
 
-class FullSyncManager(object):
+class FullSyncManager:
     def test_db(self, db: DB):
         """
         :param anki.db.DB db: the database uploaded from the client.