sync for pc and android
This commit is contained in:
parent
35cf66d403
commit
f541b09e7f
BIN
glibc-2.18.tar.gz
Normal file
BIN
glibc-2.18.tar.gz
Normal file
Binary file not shown.
20
src/ankisyncd/ankisyncd.conf
Normal file
20
src/ankisyncd/ankisyncd.conf
Normal file
@ -0,0 +1,20 @@
|
||||
[sync_app]
|
||||
# change to 127.0.0.1 if you don't want the server to be accessible from the internet
|
||||
host = 0.0.0.0
|
||||
port = 27701
|
||||
data_root = ./collections
|
||||
base_url = /sync/
|
||||
base_media_url = /msync/
|
||||
auth_db_path = ./auth.db
|
||||
# optional, for session persistence between restarts
|
||||
session_db_path = ./session.db
|
||||
|
||||
# optional, for overriding the default managers and wrappers
|
||||
# # must inherit from ankisyncd.full_sync.FullSyncManager, e.g,
|
||||
# full_sync_manager = great_stuff.postgres.PostgresFullSyncManager
|
||||
# # must inherit from ankisyncd.session.SimpleSessionManager, e.g,
|
||||
# session_manager = great_stuff.postgres.PostgresSessionManager
|
||||
# # must inherit from ankisyncd.users.SimpleUserManager, e.g,
|
||||
# user_manager = great_stuff.postgres.PostgresUserManager
|
||||
# # must inherit from ankisyncd.collection.CollectionWrapper, e.g,
|
||||
# collection_wrapper = great_stuff.postgres.PostgresCollectionWrapper
|
||||
@ -22,7 +22,7 @@ from anki.lang import ngettext
|
||||
|
||||
|
||||
# https://github.com/ankitects/anki/blob/04b1ca75599f18eb783a8bf0bdeeeb32362f4da0/rslib/src/sync/http_client.rs#L11
|
||||
SYNC_VER = 10
|
||||
SYNC_VER = 11
|
||||
# https://github.com/ankitects/anki/blob/cca3fcb2418880d0430a5c5c2e6b81ba260065b7/anki/consts.py#L50
|
||||
SYNC_ZIP_SIZE = int(2.5 * 1024 * 1024)
|
||||
# https://github.com/ankitects/anki/blob/cca3fcb2418880d0430a5c5c2e6b81ba260065b7/anki/consts.py#L51
|
||||
|
||||
@ -26,6 +26,8 @@ import time
|
||||
import unicodedata
|
||||
import zipfile
|
||||
import types
|
||||
import pyzstd
|
||||
import pdb
|
||||
from webob import Response
|
||||
from webob.exc import *
|
||||
import urllib.parse
|
||||
@ -41,7 +43,11 @@ from ankisyncd.users import get_user_manager
|
||||
|
||||
logger = logging.getLogger("ankisyncd")
|
||||
|
||||
|
||||
from enum import Enum
|
||||
# 继承枚举类
|
||||
class ClientType(Enum):
|
||||
PC = 1
|
||||
Android = 2
|
||||
class SyncCollectionHandler(Syncer):
|
||||
operations = [
|
||||
"meta",
|
||||
@ -463,7 +469,47 @@ class Requests(object):
|
||||
@POST.setter
|
||||
def POST(self, value):
|
||||
self._request_items_dict = value
|
||||
|
||||
def wrap_body(self, body, env):
|
||||
if "APP_CLIENT_TYPE" not in env:
|
||||
return body
|
||||
if env["APP_CLIENT_TYPE"] == ClientType.PC:
|
||||
return pyzstd.compress(body)
|
||||
# if env["APP_CLIENT_TYPE"] == ClientType.Android:
|
||||
# return body
|
||||
return body
|
||||
def parseAndroid(self, body, env):
|
||||
env["APP_CLIENT_TYPE"] = ClientType.Android
|
||||
request_items_dict = {}
|
||||
pattern = b'--Anki-sync-boundary\r\nContent-Disposition: form-data; name="(.*?)"\
|
||||
(?:; filename=".*?"\r\nContent-Type: application/octet-stream)*\r\n\r\n(.*?|[\s\S]*?)\r\n(?:--Anki-sync-boundary--\r\n$)*'
|
||||
res = re.findall(pattern , body)
|
||||
for k in res:
|
||||
if len(k) < 2:
|
||||
logger.error(f"error pattern match: {k}")
|
||||
continue
|
||||
v = k[1] if k[0] == b"data" else k[1].decode()
|
||||
request_items_dict[k[0].decode()] = v
|
||||
if "data" in request_items_dict and "c" in request_items_dict \
|
||||
and int(request_items_dict["c"]):
|
||||
data = request_items_dict["data"]
|
||||
with gzip.GzipFile(mode="rb", fileobj=io.BytesIO(data)) as gz:
|
||||
data = gz.read()
|
||||
request_items_dict["data"] = data
|
||||
if "data" not in request_items_dict:
|
||||
#pdb.set_trace()
|
||||
pass
|
||||
return request_items_dict
|
||||
def parsePC(self, body, env):
|
||||
env["APP_CLIENT_TYPE"] = ClientType.PC
|
||||
request_items_dict = {}
|
||||
body = pyzstd.decompress(body)
|
||||
request_items_dict["data"] = body
|
||||
http_anki_sync = env.get("HTTP_ANKI_SYNC", "")
|
||||
if http_anki_sync != "":
|
||||
anki_sync = json.loads(http_anki_sync)
|
||||
for k in anki_sync.keys():
|
||||
request_items_dict[k] = anki_sync[k]
|
||||
return request_items_dict
|
||||
@property
|
||||
def parse(self):
|
||||
"""Return a MultiDict containing all the variables from a form
|
||||
@ -472,98 +518,22 @@ class Requests(object):
|
||||
env = self.environ
|
||||
query_string = env["QUERY_STRING"]
|
||||
content_len = env.get("CONTENT_LENGTH", "0")
|
||||
input = env.get("wsgi.input")
|
||||
length = 0 if content_len == "" else int(content_len)
|
||||
body = b""
|
||||
request_items_dict = {}
|
||||
if length == 0:
|
||||
if input is None:
|
||||
return request_items_dict
|
||||
if env.get("HTTP_TRANSFER_ENCODING", "0") == "chunked":
|
||||
# readlines and read(no argument) will block
|
||||
# convert byte str to number base 16
|
||||
leng = int(input.readline(), 16)
|
||||
c = 0
|
||||
bdry = b""
|
||||
data = []
|
||||
data_other = []
|
||||
while leng > 0:
|
||||
c += 1
|
||||
dt = input.read(leng + 2)
|
||||
if c == 1:
|
||||
bdry = dt
|
||||
elif c >= 3:
|
||||
# data
|
||||
data_other.append(dt)
|
||||
leng = int(input.readline(), 16)
|
||||
data_other = [item for item in data_other if item != b"\r\n\r\n"]
|
||||
for item in data_other:
|
||||
if bdry in item:
|
||||
break
|
||||
# only strip \r\n if there are extra \n
|
||||
# eg b'?V\xc1\x8f>\xf9\xb1\n\r\n'
|
||||
data.append(item[:-2])
|
||||
request_items_dict["data"] = b"".join(data)
|
||||
others = data_other[len(data) :]
|
||||
boundary = others[0]
|
||||
others = b"".join(others).split(boundary.strip())
|
||||
others.pop()
|
||||
others.pop(0)
|
||||
for i in others:
|
||||
i = i.splitlines()
|
||||
key = re.findall(b'name="(.*?)"', i[2], flags=re.M)[0].decode(
|
||||
"utf-8"
|
||||
)
|
||||
v = i[-1].decode("utf-8")
|
||||
request_items_dict[key] = v
|
||||
return request_items_dict
|
||||
|
||||
if query_string != "":
|
||||
# GET method
|
||||
body = query_string
|
||||
request_items_dict = urllib.parse.parse_qs(body)
|
||||
for k, v in request_items_dict.items():
|
||||
request_items_dict[k] = "".join(v)
|
||||
return request_items_dict
|
||||
|
||||
else:
|
||||
body = env["wsgi.input"].read(length)
|
||||
|
||||
input = env.get("wsgi.input")
|
||||
if input is None:
|
||||
return request_items_dict
|
||||
length = 0 if content_len == "" else int(content_len)
|
||||
length = length if length > 0 else int(input.readline(), 16)
|
||||
body = input.read(length)
|
||||
if body is None or body == b"":
|
||||
return request_items_dict
|
||||
# process body to dict
|
||||
repeat = body.splitlines()[0]
|
||||
items = re.split(repeat, body)
|
||||
# del first ,last item
|
||||
items.pop()
|
||||
items.pop(0)
|
||||
for item in items:
|
||||
if b'name="data"' in item:
|
||||
data_field = None
|
||||
# remove \r\n
|
||||
if b"application/octet-stream" in item:
|
||||
# Ankidroid case
|
||||
item = re.sub(
|
||||
b'Content-Disposition: form-data; name="data"; filename="data"',
|
||||
b"",
|
||||
item,
|
||||
)
|
||||
item = re.sub(b"Content-Type: application/octet-stream", b"", item)
|
||||
data_field = item.strip()
|
||||
else:
|
||||
# PKzip file stream and others
|
||||
item = re.sub(
|
||||
b'Content-Disposition: form-data; name="data"; filename="data"',
|
||||
b"",
|
||||
item,
|
||||
)
|
||||
data_field = item.strip()
|
||||
request_items_dict["data"] = data_field
|
||||
continue
|
||||
item = re.sub(b"\r\n", b"", item, flags=re.MULTILINE)
|
||||
key = re.findall(b'name="(.*?)"', item)[0].decode("utf-8")
|
||||
v = item[item.rfind(b'"') + 1 :].decode("utf-8")
|
||||
request_items_dict[key] = v
|
||||
if 'application/octet-stream' in env.get("CONTENT_TYPE",""):
|
||||
request_items_dict = self.parsePC(body, env)
|
||||
else:
|
||||
request_items_dict = self.parseAndroid(body, env)
|
||||
length = request_items_dict if len(body) < 10240 else len(body)
|
||||
logger.info(f">>>>>::request body or size: {length}")
|
||||
self.request_items_dict = request_items_dict
|
||||
return request_items_dict
|
||||
|
||||
|
||||
@ -583,8 +553,16 @@ class chunked(object):
|
||||
b,
|
||||
)
|
||||
w = self.__wrapped__(*args, **kwargs)
|
||||
resp = Response(w)
|
||||
return resp(environ, start_response)
|
||||
if type(w) is str:
|
||||
w = w.encode()
|
||||
length = w if len(w) < 10240 else len(w)
|
||||
body = b.wrap_body(w, environ)
|
||||
logger.info(f"<<<<<::response body or size: {length} compress size{len(body)}")
|
||||
resp = Response(body, "200 OK" ,[
|
||||
("anki-original-size", str(len(body))),
|
||||
])
|
||||
r = resp(environ, start_response)
|
||||
return r
|
||||
|
||||
def __get__(self, instance, cls):
|
||||
if instance is None:
|
||||
@ -639,19 +617,6 @@ class SyncApp:
|
||||
return SyncUserSession(
|
||||
username, user_path, self.collection_manager, self.setup_new_collection
|
||||
)
|
||||
|
||||
def _decode_data(self, data, compression=0):
|
||||
if compression:
|
||||
with gzip.GzipFile(mode="rb", fileobj=io.BytesIO(data)) as gz:
|
||||
data = gz.read()
|
||||
|
||||
try:
|
||||
data = json.loads(data.decode())
|
||||
except (ValueError, UnicodeDecodeError):
|
||||
data = {"data": data}
|
||||
|
||||
return data
|
||||
|
||||
def operation_hostKey(self, username, password):
|
||||
if not self.user_manager.authenticate(username, password):
|
||||
return
|
||||
@ -697,22 +662,15 @@ class SyncApp:
|
||||
except KeyError:
|
||||
skey = None
|
||||
|
||||
try:
|
||||
compression = int(req.POST["c"])
|
||||
except KeyError:
|
||||
compression = 0
|
||||
|
||||
try:
|
||||
data = req.POST["data"]
|
||||
data = self._decode_data(data, compression)
|
||||
data = json.loads(data.decode())
|
||||
except KeyError:
|
||||
data = {}
|
||||
|
||||
if req.path.startswith(self.base_url):
|
||||
url = req.path[len(self.base_url) :]
|
||||
if url not in self.valid_urls:
|
||||
raise HTTPNotFound()
|
||||
|
||||
if url == "hostKey":
|
||||
result = self.operation_hostKey(data.get("u"), data.get("p"))
|
||||
if result:
|
||||
@ -733,10 +691,11 @@ class SyncApp:
|
||||
session.version = data["v"]
|
||||
if "cv" in data:
|
||||
session.client_version = data["cv"]
|
||||
|
||||
self.session_manager.save(hkey, session)
|
||||
session = self.session_manager.load(hkey, self.create_session)
|
||||
thread = session.get_thread()
|
||||
if "_pad" in data:
|
||||
del data["_pad"]
|
||||
result = self._execute_handler_method_in_thread(url, data, session)
|
||||
# If it's a complex data type, we convert it to JSON
|
||||
if type(result) not in (str, bytes, Response):
|
||||
@ -766,10 +725,10 @@ class SyncApp:
|
||||
|
||||
if url not in self.valid_urls:
|
||||
raise HTTPNotFound()
|
||||
|
||||
if url == "begin":
|
||||
data["skey"] = session.skey
|
||||
|
||||
if "v" in data:
|
||||
del data["v"]
|
||||
result = self._execute_handler_method_in_thread(url, data, session)
|
||||
|
||||
# If it's a complex data type, we convert it to JSON
|
||||
@ -792,7 +751,6 @@ class SyncApp:
|
||||
# Retrieve the correct handler method.
|
||||
handler = session.get_handler_for_operation(method_name, col)
|
||||
handler_method = getattr(handler, method_name)
|
||||
|
||||
res = handler_method(**keyword_args)
|
||||
|
||||
col.save()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user