sync for pc and android

src/ankisyncd/ankisyncd.conf

@@ -0,0 +1,20 @@
+[sync_app]
+# change to 127.0.0.1 if you don't want the server to be accessible from the internet
+host = 0.0.0.0
+port = 27701
+data_root = ./collections
+base_url = /sync/
+base_media_url = /msync/
+auth_db_path = ./auth.db
+# optional, for session persistence between restarts
+session_db_path = ./session.db
+
+# optional, for overriding the default managers and wrappers
+# # must inherit from ankisyncd.full_sync.FullSyncManager, e.g,
+# full_sync_manager = great_stuff.postgres.PostgresFullSyncManager
+# # must inherit from ankisyncd.session.SimpleSessionManager, e.g,
+# session_manager = great_stuff.postgres.PostgresSessionManager
+# # must inherit from ankisyncd.users.SimpleUserManager, e.g,
+# user_manager = great_stuff.postgres.PostgresUserManager
+# # must inherit from ankisyncd.collection.CollectionWrapper, e.g,
+# collection_wrapper = great_stuff.postgres.PostgresCollectionWrapper

src/ankisyncd/sync.py

@@ -22,7 +22,7 @@ from anki.lang import ngettext
 
 
 # https://github.com/ankitects/anki/blob/04b1ca75599f18eb783a8bf0bdeeeb32362f4da0/rslib/src/sync/http_client.rs#L11
-SYNC_VER = 10
+SYNC_VER = 11
 # https://github.com/ankitects/anki/blob/cca3fcb2418880d0430a5c5c2e6b81ba260065b7/anki/consts.py#L50
 SYNC_ZIP_SIZE = int(2.5 * 1024 * 1024)
 # https://github.com/ankitects/anki/blob/cca3fcb2418880d0430a5c5c2e6b81ba260065b7/anki/consts.py#L51

src/ankisyncd/sync_app.py

@@ -26,6 +26,8 @@ import time
 import unicodedata
 import zipfile
 import types
+import pyzstd
+import pdb
 from webob import Response
 from webob.exc import *
 import urllib.parse
@@ -41,7 +43,11 @@ from ankisyncd.users import get_user_manager
 
 logger = logging.getLogger("ankisyncd")
 
-
+from enum import Enum
+# 继承枚举类
+class ClientType(Enum):
+    PC  = 1
+    Android   = 2
 class SyncCollectionHandler(Syncer):
     operations = [
         "meta",
@@ -463,7 +469,47 @@ class Requests(object):
     @POST.setter
     def POST(self, value):
         self._request_items_dict = value
-
+    def wrap_body(self, body, env):
+        if "APP_CLIENT_TYPE" not in env:
+            return body
+        if env["APP_CLIENT_TYPE"] == ClientType.PC:
+            return pyzstd.compress(body)
+        # if env["APP_CLIENT_TYPE"] == ClientType.Android:
+        #     return body
+        return body
+    def parseAndroid(self, body, env):
+        env["APP_CLIENT_TYPE"] = ClientType.Android
+        request_items_dict = {}
+        pattern = b'--Anki-sync-boundary\r\nContent-Disposition: form-data; name="(.*?)"\
+(?:; filename=".*?"\r\nContent-Type: application/octet-stream)*\r\n\r\n(.*?|[\s\S]*?)\r\n(?:--Anki-sync-boundary--\r\n$)*'
+        res = re.findall(pattern , body)
+        for k in res:
+            if len(k) < 2:
+                logger.error(f"error pattern match: {k}")
+                continue
+            v = k[1] if k[0] == b"data" else k[1].decode()
+            request_items_dict[k[0].decode()] = v
+        if "data" in request_items_dict and "c" in request_items_dict \
+            and int(request_items_dict["c"]):
+            data = request_items_dict["data"] 
+            with gzip.GzipFile(mode="rb", fileobj=io.BytesIO(data)) as gz:
+                data = gz.read()
+            request_items_dict["data"]  = data
+        if "data" not in request_items_dict:
+            #pdb.set_trace()
+            pass
+        return request_items_dict
+    def parsePC(self, body, env):
+        env["APP_CLIENT_TYPE"] = ClientType.PC
+        request_items_dict = {}
+        body = pyzstd.decompress(body)
+        request_items_dict["data"] = body
+        http_anki_sync = env.get("HTTP_ANKI_SYNC", "")
+        if http_anki_sync != "":
+            anki_sync = json.loads(http_anki_sync)
+            for k in anki_sync.keys():
+                request_items_dict[k] = anki_sync[k]
+        return request_items_dict
     @property
     def parse(self):
         """Return a MultiDict containing all the variables from a form
@@ -472,98 +518,22 @@ class Requests(object):
         env = self.environ
         query_string = env["QUERY_STRING"]
         content_len = env.get("CONTENT_LENGTH", "0")
-        input = env.get("wsgi.input")
-        length = 0 if content_len == "" else int(content_len)
-        body = b""
         request_items_dict = {}
-        if length == 0:
+        input = env.get("wsgi.input")
         if input is None:
             return request_items_dict
-            if env.get("HTTP_TRANSFER_ENCODING", "0") == "chunked":
+        length = 0 if content_len == "" else int(content_len)
-                # readlines and read(no argument) will block
+        length = length if length > 0 else int(input.readline(), 16)
-                # convert byte str to number base 16
+        body = input.read(length)
-                leng = int(input.readline(), 16)
-                c = 0
-                bdry = b""
-                data = []
-                data_other = []
-                while leng > 0:
-                    c += 1
-                    dt = input.read(leng + 2)
-                    if c == 1:
-                        bdry = dt
-                    elif c >= 3:
-                        # data
-                        data_other.append(dt)
-                    leng = int(input.readline(), 16)
-                data_other = [item for item in data_other if item != b"\r\n\r\n"]
-                for item in data_other:
-                    if bdry in item:
-                        break
-                    # only strip \r\n if there are extra \n
-                    # eg b'?V\xc1\x8f>\xf9\xb1\n\r\n'
-                    data.append(item[:-2])
-                request_items_dict["data"] = b"".join(data)
-                others = data_other[len(data) :]
-                boundary = others[0]
-                others = b"".join(others).split(boundary.strip())
-                others.pop()
-                others.pop(0)
-                for i in others:
-                    i = i.splitlines()
-                    key = re.findall(b'name="(.*?)"', i[2], flags=re.M)[0].decode(
-                        "utf-8"
-                    )
-                    v = i[-1].decode("utf-8")
-                    request_items_dict[key] = v
-                return request_items_dict
-
-            if query_string != "":
-                # GET method
-                body = query_string
-                request_items_dict = urllib.parse.parse_qs(body)
-                for k, v in request_items_dict.items():
-                    request_items_dict[k] = "".join(v)
-                return request_items_dict
-
-        else:
-            body = env["wsgi.input"].read(length)
-
         if body is None or body == b"":
             return request_items_dict
-            # process body to dict
+        if 'application/octet-stream' in env.get("CONTENT_TYPE",""):
-        repeat = body.splitlines()[0]
+            request_items_dict = self.parsePC(body, env)
-        items = re.split(repeat, body)
-        # del first ,last item
-        items.pop()
-        items.pop(0)
-        for item in items:
-            if b'name="data"' in item:
-                data_field = None
-                # remove \r\n
-                if b"application/octet-stream" in item:
-                    # Ankidroid case
-                    item = re.sub(
-                        b'Content-Disposition: form-data; name="data"; filename="data"',
-                        b"",
-                        item,
-                    )
-                    item = re.sub(b"Content-Type: application/octet-stream", b"", item)
-                    data_field = item.strip()
         else:
-                    # PKzip file stream and others
+            request_items_dict = self.parseAndroid(body, env)
-                    item = re.sub(
+        length = request_items_dict if len(body) < 10240 else len(body) 
-                        b'Content-Disposition: form-data; name="data"; filename="data"',
+        logger.info(f">>>>>::request body or size: {length}")
-                        b"",
+        self.request_items_dict = request_items_dict
-                        item,
-                    )
-                    data_field = item.strip()
-                request_items_dict["data"] = data_field
-                continue
-            item = re.sub(b"\r\n", b"", item, flags=re.MULTILINE)
-            key = re.findall(b'name="(.*?)"', item)[0].decode("utf-8")
-            v = item[item.rfind(b'"') + 1 :].decode("utf-8")
-            request_items_dict[key] = v
         return request_items_dict
 
 
@@ -583,8 +553,16 @@ class chunked(object):
             b,
         )
         w = self.__wrapped__(*args, **kwargs)
-        resp = Response(w)
+        if type(w) is str:
-        return resp(environ, start_response)
+            w = w.encode()
+        length = w if len(w) < 10240 else len(w)
+        body = b.wrap_body(w, environ)
+        logger.info(f"<<<<<::response body or size: {length} compress size{len(body)}")
+        resp = Response(body, "200 OK" ,[
+                    ("anki-original-size", str(len(body))),
+                ])
+        r = resp(environ, start_response)
+        return r
 
     def __get__(self, instance, cls):
         if instance is None:
@@ -639,19 +617,6 @@ class SyncApp:
         return SyncUserSession(
             username, user_path, self.collection_manager, self.setup_new_collection
         )
-
-    def _decode_data(self, data, compression=0):
-        if compression:
-            with gzip.GzipFile(mode="rb", fileobj=io.BytesIO(data)) as gz:
-                data = gz.read()
-
-        try:
-            data = json.loads(data.decode())
-        except (ValueError, UnicodeDecodeError):
-            data = {"data": data}
-
-        return data
-
     def operation_hostKey(self, username, password):
         if not self.user_manager.authenticate(username, password):
             return
@@ -697,22 +662,15 @@ class SyncApp:
             except KeyError:
                 skey = None
 
-        try:
-            compression = int(req.POST["c"])
-        except KeyError:
-            compression = 0
-
         try:
             data = req.POST["data"]
-            data = self._decode_data(data, compression)
+            data = json.loads(data.decode())
         except KeyError:
             data = {}
-
         if req.path.startswith(self.base_url):
             url = req.path[len(self.base_url) :]
             if url not in self.valid_urls:
                 raise HTTPNotFound()
-
             if url == "hostKey":
                 result = self.operation_hostKey(data.get("u"), data.get("p"))
                 if result:
@@ -733,10 +691,11 @@ class SyncApp:
                         session.version = data["v"]
                     if "cv" in data:
                         session.client_version = data["cv"]
-
                     self.session_manager.save(hkey, session)
                     session = self.session_manager.load(hkey, self.create_session)
                 thread = session.get_thread()
+                if "_pad" in data:
+                    del data["_pad"]
                 result = self._execute_handler_method_in_thread(url, data, session)
                 # If it's a complex data type, we convert it to JSON
                 if type(result) not in (str, bytes, Response):
@@ -766,10 +725,10 @@ class SyncApp:
 
             if url not in self.valid_urls:
                 raise HTTPNotFound()
-
             if url == "begin":
                 data["skey"] = session.skey
-
+            if "v" in data:
+                del data["v"]
             result = self._execute_handler_method_in_thread(url, data, session)
 
             # If it's a complex data type, we convert it to JSON
@@ -792,7 +751,6 @@ class SyncApp:
             # Retrieve the correct handler method.
             handler = session.get_handler_for_operation(method_name, col)
             handler_method = getattr(handler, method_name)
-
             res = handler_method(**keyword_args)
 
             col.save()