diff --git a/glibc-2.18.tar.gz b/glibc-2.18.tar.gz new file mode 100644 index 0000000..96c727b Binary files /dev/null and b/glibc-2.18.tar.gz differ diff --git a/src/ankisyncd/ankisyncd.conf b/src/ankisyncd/ankisyncd.conf new file mode 100644 index 0000000..dea356f --- /dev/null +++ b/src/ankisyncd/ankisyncd.conf @@ -0,0 +1,20 @@ +[sync_app] +# change to 127.0.0.1 if you don't want the server to be accessible from the internet +host = 0.0.0.0 +port = 27701 +data_root = ./collections +base_url = /sync/ +base_media_url = /msync/ +auth_db_path = ./auth.db +# optional, for session persistence between restarts +session_db_path = ./session.db + +# optional, for overriding the default managers and wrappers +# # must inherit from ankisyncd.full_sync.FullSyncManager, e.g, +# full_sync_manager = great_stuff.postgres.PostgresFullSyncManager +# # must inherit from ankisyncd.session.SimpleSessionManager, e.g, +# session_manager = great_stuff.postgres.PostgresSessionManager +# # must inherit from ankisyncd.users.SimpleUserManager, e.g, +# user_manager = great_stuff.postgres.PostgresUserManager +# # must inherit from ankisyncd.collection.CollectionWrapper, e.g, +# collection_wrapper = great_stuff.postgres.PostgresCollectionWrapper diff --git a/src/ankisyncd/sync.py b/src/ankisyncd/sync.py index 5796246..d910fbe 100644 --- a/src/ankisyncd/sync.py +++ b/src/ankisyncd/sync.py @@ -22,7 +22,7 @@ from anki.lang import ngettext # https://github.com/ankitects/anki/blob/04b1ca75599f18eb783a8bf0bdeeeb32362f4da0/rslib/src/sync/http_client.rs#L11 -SYNC_VER = 10 +SYNC_VER = 11 # https://github.com/ankitects/anki/blob/cca3fcb2418880d0430a5c5c2e6b81ba260065b7/anki/consts.py#L50 SYNC_ZIP_SIZE = int(2.5 * 1024 * 1024) # https://github.com/ankitects/anki/blob/cca3fcb2418880d0430a5c5c2e6b81ba260065b7/anki/consts.py#L51 diff --git a/src/ankisyncd/sync_app.py b/src/ankisyncd/sync_app.py index f032f21..e687e44 100644 --- a/src/ankisyncd/sync_app.py +++ b/src/ankisyncd/sync_app.py @@ -26,6 +26,8 @@ import time import unicodedata import zipfile import types +import pyzstd +import pdb from webob import Response from webob.exc import * import urllib.parse @@ -41,7 +43,11 @@ from ankisyncd.users import get_user_manager logger = logging.getLogger("ankisyncd") - +from enum import Enum +# 继承枚举类 +class ClientType(Enum): + PC = 1 + Android = 2 class SyncCollectionHandler(Syncer): operations = [ "meta", @@ -463,7 +469,47 @@ class Requests(object): @POST.setter def POST(self, value): self._request_items_dict = value - + def wrap_body(self, body, env): + if "APP_CLIENT_TYPE" not in env: + return body + if env["APP_CLIENT_TYPE"] == ClientType.PC: + return pyzstd.compress(body) + # if env["APP_CLIENT_TYPE"] == ClientType.Android: + # return body + return body + def parseAndroid(self, body, env): + env["APP_CLIENT_TYPE"] = ClientType.Android + request_items_dict = {} + pattern = b'--Anki-sync-boundary\r\nContent-Disposition: form-data; name="(.*?)"\ +(?:; filename=".*?"\r\nContent-Type: application/octet-stream)*\r\n\r\n(.*?|[\s\S]*?)\r\n(?:--Anki-sync-boundary--\r\n$)*' + res = re.findall(pattern , body) + for k in res: + if len(k) < 2: + logger.error(f"error pattern match: {k}") + continue + v = k[1] if k[0] == b"data" else k[1].decode() + request_items_dict[k[0].decode()] = v + if "data" in request_items_dict and "c" in request_items_dict \ + and int(request_items_dict["c"]): + data = request_items_dict["data"] + with gzip.GzipFile(mode="rb", fileobj=io.BytesIO(data)) as gz: + data = gz.read() + request_items_dict["data"] = data + if "data" not in request_items_dict: + #pdb.set_trace() + pass + return request_items_dict + def parsePC(self, body, env): + env["APP_CLIENT_TYPE"] = ClientType.PC + request_items_dict = {} + body = pyzstd.decompress(body) + request_items_dict["data"] = body + http_anki_sync = env.get("HTTP_ANKI_SYNC", "") + if http_anki_sync != "": + anki_sync = json.loads(http_anki_sync) + for k in anki_sync.keys(): + request_items_dict[k] = anki_sync[k] + return request_items_dict @property def parse(self): """Return a MultiDict containing all the variables from a form @@ -472,98 +518,22 @@ class Requests(object): env = self.environ query_string = env["QUERY_STRING"] content_len = env.get("CONTENT_LENGTH", "0") - input = env.get("wsgi.input") - length = 0 if content_len == "" else int(content_len) - body = b"" request_items_dict = {} - if length == 0: - if input is None: - return request_items_dict - if env.get("HTTP_TRANSFER_ENCODING", "0") == "chunked": - # readlines and read(no argument) will block - # convert byte str to number base 16 - leng = int(input.readline(), 16) - c = 0 - bdry = b"" - data = [] - data_other = [] - while leng > 0: - c += 1 - dt = input.read(leng + 2) - if c == 1: - bdry = dt - elif c >= 3: - # data - data_other.append(dt) - leng = int(input.readline(), 16) - data_other = [item for item in data_other if item != b"\r\n\r\n"] - for item in data_other: - if bdry in item: - break - # only strip \r\n if there are extra \n - # eg b'?V\xc1\x8f>\xf9\xb1\n\r\n' - data.append(item[:-2]) - request_items_dict["data"] = b"".join(data) - others = data_other[len(data) :] - boundary = others[0] - others = b"".join(others).split(boundary.strip()) - others.pop() - others.pop(0) - for i in others: - i = i.splitlines() - key = re.findall(b'name="(.*?)"', i[2], flags=re.M)[0].decode( - "utf-8" - ) - v = i[-1].decode("utf-8") - request_items_dict[key] = v - return request_items_dict - - if query_string != "": - # GET method - body = query_string - request_items_dict = urllib.parse.parse_qs(body) - for k, v in request_items_dict.items(): - request_items_dict[k] = "".join(v) - return request_items_dict - - else: - body = env["wsgi.input"].read(length) - + input = env.get("wsgi.input") + if input is None: + return request_items_dict + length = 0 if content_len == "" else int(content_len) + length = length if length > 0 else int(input.readline(), 16) + body = input.read(length) if body is None or body == b"": return request_items_dict - # process body to dict - repeat = body.splitlines()[0] - items = re.split(repeat, body) - # del first ,last item - items.pop() - items.pop(0) - for item in items: - if b'name="data"' in item: - data_field = None - # remove \r\n - if b"application/octet-stream" in item: - # Ankidroid case - item = re.sub( - b'Content-Disposition: form-data; name="data"; filename="data"', - b"", - item, - ) - item = re.sub(b"Content-Type: application/octet-stream", b"", item) - data_field = item.strip() - else: - # PKzip file stream and others - item = re.sub( - b'Content-Disposition: form-data; name="data"; filename="data"', - b"", - item, - ) - data_field = item.strip() - request_items_dict["data"] = data_field - continue - item = re.sub(b"\r\n", b"", item, flags=re.MULTILINE) - key = re.findall(b'name="(.*?)"', item)[0].decode("utf-8") - v = item[item.rfind(b'"') + 1 :].decode("utf-8") - request_items_dict[key] = v + if 'application/octet-stream' in env.get("CONTENT_TYPE",""): + request_items_dict = self.parsePC(body, env) + else: + request_items_dict = self.parseAndroid(body, env) + length = request_items_dict if len(body) < 10240 else len(body) + logger.info(f">>>>>::request body or size: {length}") + self.request_items_dict = request_items_dict return request_items_dict @@ -583,8 +553,16 @@ class chunked(object): b, ) w = self.__wrapped__(*args, **kwargs) - resp = Response(w) - return resp(environ, start_response) + if type(w) is str: + w = w.encode() + length = w if len(w) < 10240 else len(w) + body = b.wrap_body(w, environ) + logger.info(f"<<<<<::response body or size: {length} compress size{len(body)}") + resp = Response(body, "200 OK" ,[ + ("anki-original-size", str(len(body))), + ]) + r = resp(environ, start_response) + return r def __get__(self, instance, cls): if instance is None: @@ -639,19 +617,6 @@ class SyncApp: return SyncUserSession( username, user_path, self.collection_manager, self.setup_new_collection ) - - def _decode_data(self, data, compression=0): - if compression: - with gzip.GzipFile(mode="rb", fileobj=io.BytesIO(data)) as gz: - data = gz.read() - - try: - data = json.loads(data.decode()) - except (ValueError, UnicodeDecodeError): - data = {"data": data} - - return data - def operation_hostKey(self, username, password): if not self.user_manager.authenticate(username, password): return @@ -697,22 +662,15 @@ class SyncApp: except KeyError: skey = None - try: - compression = int(req.POST["c"]) - except KeyError: - compression = 0 - try: data = req.POST["data"] - data = self._decode_data(data, compression) + data = json.loads(data.decode()) except KeyError: data = {} - if req.path.startswith(self.base_url): url = req.path[len(self.base_url) :] if url not in self.valid_urls: raise HTTPNotFound() - if url == "hostKey": result = self.operation_hostKey(data.get("u"), data.get("p")) if result: @@ -733,10 +691,11 @@ class SyncApp: session.version = data["v"] if "cv" in data: session.client_version = data["cv"] - self.session_manager.save(hkey, session) session = self.session_manager.load(hkey, self.create_session) thread = session.get_thread() + if "_pad" in data: + del data["_pad"] result = self._execute_handler_method_in_thread(url, data, session) # If it's a complex data type, we convert it to JSON if type(result) not in (str, bytes, Response): @@ -766,10 +725,10 @@ class SyncApp: if url not in self.valid_urls: raise HTTPNotFound() - if url == "begin": data["skey"] = session.skey - + if "v" in data: + del data["v"] result = self._execute_handler_method_in_thread(url, data, session) # If it's a complex data type, we convert it to JSON @@ -792,7 +751,6 @@ class SyncApp: # Retrieve the correct handler method. handler = session.get_handler_for_operation(method_name, col) handler_method = getattr(handler, method_name) - res = handler_method(**keyword_args) col.save()