diff --git a/tests/assets/blue.jpg b/tests/assets/blue.jpg new file mode 100644 index 0000000..c958d13 Binary files /dev/null and b/tests/assets/blue.jpg differ diff --git a/tests/assets/test.conf b/tests/assets/test.conf new file mode 100644 index 0000000..2b9c700 --- /dev/null +++ b/tests/assets/test.conf @@ -0,0 +1,8 @@ +[sync_app] +host = 127.0.0.1 +port = 27701 +data_root = ./collections +base_url = /sync/ +base_media_url = /msync/ +auth_db_path = ./auth.db +session_db_path = ./session.db diff --git a/tests/helpers/file_utils.py b/tests/helpers/file_utils.py index 35764cd..2238b5c 100644 --- a/tests/helpers/file_utils.py +++ b/tests/helpers/file_utils.py @@ -169,3 +169,19 @@ class FileUtils(object): zip_file.close() return file_buffer.getvalue() + + def get_asset_path(self, relative_file_path): + """ + Retrieves the path of a file for testing from the "assets" directory. + + :param relative_file_path: the name of the file to retrieve, relative + to the "assets" directory + :return: the absolute path to the file in the "assets" directory. + """ + + join = os.path.join + + script_dir = os.path.dirname(os.path.realpath(__file__)) + support_dir = join(script_dir, os.pardir, "assets") + res = join(support_dir, relative_file_path) + return res diff --git a/tests/helpers/mock_servers.py b/tests/helpers/mock_servers.py new file mode 100644 index 0000000..6ac0332 --- /dev/null +++ b/tests/helpers/mock_servers.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- + + +import logging + + +from anki.sync import HttpSyncer, RemoteServer, RemoteMediaServer, FullSyncer + + +class MockServerConnection(object): + """ + Mock for HttpSyncer's con attribute, a httplib2 connection. All requests + that would normally got to the remote server will be redirected to our + server_app_to_test object. + """ + + def __init__(self, server_app_to_test): + self.test_app = server_app_to_test + + def request(self, uri, method='GET', headers=None, body=None): + if method == 'POST': + logging.debug("Posting to URI '{}'.".format(uri)) + logging.info("Posting to URI '{}'.".format(uri)) + test_response = self.test_app.post(uri, + params=body, + headers=headers, + status="*") + + resp = test_response.headers + resp.update({ + "status": str(test_response.status_int) + }) + cont = test_response.body + return resp, cont + else: + raise Exception('Unexpected HttpSyncer.req() behavior.') + + +class MockRemoteServer(RemoteServer): + """ + Mock for RemoteServer. All communication to our remote counterpart is + routed to our TestApp object. + """ + + def __init__(self, hkey, server_test_app): + # Create a custom connection object we will use to communicate with our + # 'remote' server counterpart. + connection = MockServerConnection(server_test_app) + HttpSyncer.__init__(self, hkey, connection) + + def syncURL(self): # Overrides RemoteServer.syncURL(). + return "/sync/" + + +class MockRemoteMediaServer(RemoteMediaServer): + """ + Mock for RemoteMediaServer. All communication to our remote counterpart is + routed to our TestApp object. + """ + + def __init__(self, col, hkey, server_test_app): + # Create a custom connection object we will use to communicate with our + # 'remote' server counterpart. + connection = MockServerConnection(server_test_app) + HttpSyncer.__init__(self, hkey, connection) + + def syncURL(self): # Overrides RemoteServer.syncURL(). + return "/msync/" diff --git a/tests/helpers/monkey_patches.py b/tests/helpers/monkey_patches.py new file mode 100644 index 0000000..f9ea83b --- /dev/null +++ b/tests/helpers/monkey_patches.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- + + +import os +import sqlite3 as sqlite +from anki.media import MediaManager +from anki.storage import DB + + +mediamanager_orig_funcs = { + "findChanges": None, + "mediaChangesZip": None, + "addFilesFromZip": None, + "syncDelete": None +} + + +db_orig_funcs = { + "__init__": None +} + + +def monkeypatch_mediamanager(): + """ + Monkey patches anki.media.MediaManager's methods so they chdir to + self.dir() before acting on its media directory and chdir back to the + original cwd after finishing. + """ + + def make_cwd_safe(original_func): + mediamanager_orig_funcs["findChanges"] = MediaManager.findChanges + mediamanager_orig_funcs["mediaChangesZip"] = MediaManager.mediaChangesZip + mediamanager_orig_funcs["addFilesFromZip"] = MediaManager.addFilesFromZip + mediamanager_orig_funcs["syncDelete"] = MediaManager.syncDelete + + def wrapper(instance, *args): + old_cwd = os.getcwd() + os.chdir(instance.dir()) + + res = original_func(instance, *args) + + os.chdir(old_cwd) + return res + return wrapper + + MediaManager.findChanges = make_cwd_safe(MediaManager.findChanges) + MediaManager.mediaChangesZip = make_cwd_safe(MediaManager.mediaChangesZip) + MediaManager.addFilesFromZip = make_cwd_safe(MediaManager.addFilesFromZip) + MediaManager.syncDelete = make_cwd_safe(MediaManager.syncDelete) + + +def unpatch_mediamanager(): + """Undoes monkey patches to Anki's MediaManager.""" + + MediaManager.findChanges = mediamanager_orig_funcs["findChanges"] + MediaManager.mediaChangesZip = mediamanager_orig_funcs["mediaChangesZip"] + MediaManager.addFilesFromZip = mediamanager_orig_funcs["addFilesFromZip"] + MediaManager.syncDelete = mediamanager_orig_funcs["syncDelete"] + + mediamanager_orig_funcs["findChanges"] = None + mediamanager_orig_funcs["mediaChangesZip"] = None + mediamanager_orig_funcs["mediaChangesZip"] = None + mediamanager_orig_funcs["mediaChangesZip"] = None + + +def monkeypatch_db(): + """ + Monkey patches Anki's DB.__init__ to connect to allow access to the db + connection from more than one thread, so that we can inspect and modify + the db created in the app in our test code. + """ + db_orig_funcs["__init__"] = DB.__init__ + + def patched___init__(self, path, text=None, timeout=0): + # Code taken from Anki's DB.__init__() + encpath = path + if isinstance(encpath, unicode): + encpath = path.encode("utf-8") + # Allow more than one thread to use this connection. + self._db = sqlite.connect(encpath, + timeout=timeout, + check_same_thread=False) + if text: + self._db.text_factory = text + self._path = path + self.echo = os.environ.get("DBECHO") # echo db modifications + self.mod = False # flag that db has been modified? + + DB.__init__ = patched___init__ + + +def unpatch_db(): + """Undoes monkey patches to Anki's DB.""" + + DB.__init__ = db_orig_funcs["__init__"] + db_orig_funcs["__init__"] = None diff --git a/tests/helpers/server_utils.py b/tests/helpers/server_utils.py new file mode 100644 index 0000000..c245402 --- /dev/null +++ b/tests/helpers/server_utils.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- + + +import filecmp +import logging +import os +import ConfigParser +import shutil + + +from ankisyncd.sync_app import SyncApp, SyncCollectionHandler, SyncMediaHandler +from helpers.file_utils import FileUtils + + +class ServerUtils(object): + def __init__(self): + self.fileutils = FileUtils() + + def clean_up(self): + self.fileutils.clean_up() + + def create_server_paths(self): + """ + Creates temporary files and dirs for our app to use during tests. + """ + + auth = self.fileutils.create_file_path(suffix='.db', + prefix='ankiserver_auth_db_') + session = self.fileutils.create_file_path(suffix='.db', + prefix='ankiserver_session_db_') + data = self.fileutils.create_dir(suffix='', + prefix='ankiserver_data_root_') + return { + "auth_db": auth, + "session_db": session, + "data_root": data + } + + @staticmethod + def create_sync_app(server_paths, config_path): + config = ConfigParser.SafeConfigParser() + config.read(config_path) + + # Use custom files and dirs in settings. + config.set("sync_app", "auth_db_path", server_paths["auth_db"]) + config.set("sync_app", "session_db_path", server_paths["session_db"]) + config.set("sync_app", "data_root", server_paths["data_root"]) + + return SyncApp(config) + + def get_session_for_hkey(self, server, hkey): + return server.session_manager.load(hkey) + + def get_thread_for_hkey(self, server, hkey): + session = self.get_session_for_hkey(server, hkey) + thread = session.get_thread() + return thread + + def get_col_wrapper_for_hkey(self, server, hkey): + print("getting col wrapper for hkey " + hkey) + print("all session keys: " + str(server.session_manager.sessions.keys())) + thread = self.get_thread_for_hkey(server, hkey) + col_wrapper = thread.wrapper + return col_wrapper + + def get_col_for_hkey(self, server, hkey): + col_wrapper = self.get_col_wrapper_for_hkey(server, hkey) + col_wrapper.open() # Make sure the col is opened. + return col_wrapper._CollectionWrapper__col + + def get_col_db_path_for_hkey(self, server, hkey): + col = self.get_col_for_hkey(server, hkey) + return col.db._path + + def get_syncer_for_hkey(self, server, hkey, syncer_type='collection'): + col = self.get_col_for_hkey(server, hkey) + + session = self.get_session_for_hkey(server, hkey) + + syncer_type = syncer_type.lower() + if syncer_type == 'collection': + handler_method = SyncCollectionHandler.operations[0] + elif syncer_type == 'media': + handler_method = SyncMediaHandler.operations[0] + + return session.get_handler_for_operation(handler_method, col) + + def add_files_to_mediasyncer(self, + media_syncer, + filepaths, + update_db=False, + bump_last_usn=False): + """ + If bumpLastUsn is True, the media syncer's lastUsn will be incremented + once for each added file. Use this when adding files to the server. + """ + + for filepath in filepaths: + logging.debug("Adding file '{}' to mediaSyncer".format(filepath)) + # Import file into media dir. + media_syncer.col.media.addFile(filepath) + if bump_last_usn: + # Need to bump lastUsn once for each file. + media_manager = media_syncer.col.media + media_manager.setLastUsn(media_syncer.col.media.lastUsn() + 1) + + if update_db: + media_syncer.col.media.findChanges() # Write changes to db. diff --git a/tests/sync_app_functional_media_test.py b/tests/sync_app_functional_media_test.py new file mode 100644 index 0000000..7cce65c --- /dev/null +++ b/tests/sync_app_functional_media_test.py @@ -0,0 +1,315 @@ +# -*- coding: utf-8 -*- + + +import filecmp +import os + + +from anki.sync import Syncer, MediaSyncer +from helpers.mock_servers import MockRemoteMediaServer +from helpers.monkey_patches import monkeypatch_mediamanager, unpatch_mediamanager +from sync_app_functional_test_base import SyncAppFunctionalTestBase + + +class SyncAppFunctionalMediaTest(SyncAppFunctionalTestBase): + + def setUp(self): + SyncAppFunctionalTestBase.setUp(self) + + monkeypatch_mediamanager() + self.hkey = self.mock_remote_server.hostKey("testuser", "testpassword") + client_collection = self.colutils.create_empty_col() + self.client_syncer = self.create_client_syncer(client_collection, + self.hkey, + self.server_test_app) + + def tearDown(self): + self.hkey = None + self.client_syncer = None + unpatch_mediamanager() + SyncAppFunctionalTestBase.tearDown(self) + + @staticmethod + def create_client_syncer(collection, hkey, server_test_app): + mock_remote_server = MockRemoteMediaServer(col=collection, + hkey=hkey, + server_test_app=server_test_app) + media_syncer = MediaSyncer(col=collection, + server=mock_remote_server) + return media_syncer + + def test_sync_empty_media_dbs(self): + # With both the client and the server having no media to sync, + # syncing should change nothing. + self.assertEqual('noChanges', self.client_syncer.sync()) + self.assertEqual('noChanges', self.client_syncer.sync()) + + def test_sync_file_from_server(self): + """ + Adds a file on the server. After syncing, client and server should have + the identical file in their media directories and media databases. + """ + client = self.client_syncer + server = self.serverutils.get_syncer_for_hkey(self.server_app, + self.hkey, + 'media') + + # Create a test file. + temp_file_path = self.fileutils.create_named_file(u"foo.jpg", "hello") + + # Add the test file to the server's collection. + self.serverutils.add_files_to_mediasyncer(server, + [temp_file_path], + update_db=True, + bump_last_usn=True) + + # Syncing should work. + self.assertEqual(client.sync(), 'OK') + + # The test file should be present in the server's and in the client's + # media directory. + self.assertTrue( + filecmp.cmp(os.path.join(client.col.media.dir(), u"foo.jpg"), + os.path.join(server.col.media.dir(), u"foo.jpg"))) + + # Further syncing should do nothing. + self.assertEqual(client.sync(), 'noChanges') + + def test_sync_file_from_client(self): + """ + Adds a file on the client. After syncing, client and server should have + the identical file in their media directories and media databases. + """ + join = os.path.join + client = self.client_syncer + server = self.serverutils.get_syncer_for_hkey(self.server_app, + self.hkey, + 'media') + + # Create a test file. + temp_file_path = self.fileutils.create_named_file(u"foo.jpg", "hello") + + # Add the test file to the client's media collection. + self.serverutils.add_files_to_mediasyncer(client, + [temp_file_path], + update_db=True, + bump_last_usn=False) + + # Syncing should work. + self.assertEqual(client.sync(), 'OK') + + # The same file should be present in both the client's and the server's + # media directory. + self.assertTrue(filecmp.cmp(join(client.col.media.dir(), u"foo.jpg"), + join(server.col.media.dir(), u"foo.jpg"))) + + # Further syncing should do nothing. + self.assertEqual(client.sync(), 'noChanges') + + # Except for timestamps, the media databases of client and server + # should be identical. + self.assertFalse( + self.dbutils.media_dbs_differ(client.col.media.db._path, + server.col.media.db._path) + ) + + def test_sync_different_files(self): + """ + Adds a file on the client and a file with different name and content on + the server. After syncing, both client and server should have both + files in their media directories and databases. + """ + join = os.path.join + isfile = os.path.isfile + client = self.client_syncer + server = self.serverutils.get_syncer_for_hkey(self.server_app, + self.hkey, + 'media') + + # Create two files and add one to the server and one to the client. + file_for_client, file_for_server = self.fileutils.create_named_files([ + (u"foo.jpg", "hello"), + (u"bar.jpg", "goodbye") + ]) + + self.serverutils.add_files_to_mediasyncer(client, + [file_for_client], + update_db=True) + self.serverutils.add_files_to_mediasyncer(server, + [file_for_server], + update_db=True, + bump_last_usn=True) + + # Syncing should work. + self.assertEqual(client.sync(), 'OK') + + # Both files should be present in the client's and in the server's + # media directories. + self.assertTrue(isfile(join(client.col.media.dir(), u"foo.jpg"))) + self.assertTrue(isfile(join(server.col.media.dir(), u"foo.jpg"))) + self.assertTrue(filecmp.cmp( + join(client.col.media.dir(), u"foo.jpg"), + join(server.col.media.dir(), u"foo.jpg")) + ) + self.assertTrue(isfile(join(client.col.media.dir(), u"bar.jpg"))) + self.assertTrue(isfile(join(server.col.media.dir(), u"bar.jpg"))) + self.assertTrue(filecmp.cmp( + join(client.col.media.dir(), u"bar.jpg"), + join(server.col.media.dir(), u"bar.jpg")) + ) + + # Further syncing should change nothing. + self.assertEqual(client.sync(), 'noChanges') + + def test_sync_different_contents(self): + """ + Adds a file to the client and a file with identical name but different + contents to the server. After syncing, both client and server should + have the server's version of the file in their media directories and + databases. + """ + join = os.path.join + isfile = os.path.isfile + client = self.client_syncer + server = self.serverutils.get_syncer_for_hkey(self.server_app, + self.hkey, + 'media') + + # Create two files with identical names but different contents and + # checksums. Add one to the server and one to the client. + file_for_client, file_for_server = self.fileutils.create_named_files([ + (u"foo.jpg", "hello"), + (u"foo.jpg", "goodbye") + ]) + + self.serverutils.add_files_to_mediasyncer(client, + [file_for_client], + update_db=True) + self.serverutils.add_files_to_mediasyncer(server, + [file_for_server], + update_db=True, + bump_last_usn=True) + + # Syncing should work. + self.assertEqual(client.sync(), 'OK') + + # A version of the file should be present in both the client's and the + # server's media directory. + self.assertTrue(isfile(join(client.col.media.dir(), u"foo.jpg"))) + self.assertEqual(os.listdir(client.col.media.dir()), ['foo.jpg']) + self.assertTrue(isfile(join(server.col.media.dir(), u"foo.jpg"))) + self.assertEqual(os.listdir(server.col.media.dir()), ['foo.jpg']) + self.assertEqual(client.sync(), 'noChanges') + + # Both files should have the contents of the server's version. + _checksum = client.col.media._checksum + self.assertEqual(_checksum(join(client.col.media.dir(), u"foo.jpg")), + _checksum(file_for_server)) + self.assertEqual(_checksum(join(server.col.media.dir(), u"foo.jpg")), + _checksum(file_for_server)) + + def test_sync_add_and_delete_on_client(self): + """ + Adds a file on the client. After syncing, the client and server should + both have the file. Then removes the file from the client's directory + and marks it as deleted in its database. After syncing again, the + server should have removed its version of the file from its media dir + and marked it as deleted in its db. + """ + join = os.path.join + isfile = os.path.isfile + client = self.client_syncer + server = self.serverutils.get_syncer_for_hkey(self.server_app, + self.hkey, + 'media') + + # Create a test file. + temp_file_path = self.fileutils.create_named_file(u"foo.jpg", "hello") + + # Add the test file to client's media collection. + self.serverutils.add_files_to_mediasyncer(client, + [temp_file_path], + update_db=True, + bump_last_usn=False) + + # Syncing client should work. + self.assertEqual(client.sync(), 'OK') + + # The same file should be present in both client's and the server's + # media directory. + self.assertTrue(filecmp.cmp(join(client.col.media.dir(), u"foo.jpg"), + join(server.col.media.dir(), u"foo.jpg"))) + + # Syncing client again should do nothing. + self.assertEqual(client.sync(), 'noChanges') + + # Remove files from client's media dir and write changes to its db. + os.remove(join(client.col.media.dir(), u"foo.jpg")) + + # TODO: client.col.media.findChanges() doesn't work here - why? + client.col.media._logChanges() + self.assertEqual(client.col.media.syncInfo(u"foo.jpg"), (None, 1)) + self.assertFalse(isfile(join(client.col.media.dir(), u"foo.jpg"))) + + # Syncing client again should work. + self.assertEqual(client.sync(), 'OK') + + # server should have picked up the removal from client. + self.assertEqual(server.col.media.syncInfo(u"foo.jpg"), (None, 0)) + self.assertFalse(isfile(join(server.col.media.dir(), u"foo.jpg"))) + + # Syncing client again should do nothing. + self.assertEqual(client.sync(), 'noChanges') + + def test_sync_compare_database_to_expected(self): + """ + Adds a test image file to the client's media directory. After syncing, + the server's database should, except for timestamps, be identical to a + database containing the expected data. + """ + client = self.client_syncer + + # Add a test image file to the client's media collection but don't + # update its media db since the desktop client updates that, using + # findChanges(), only during syncs. + support_file = self.fileutils.get_asset_path(u'blue.jpg') + self.assertTrue(os.path.isfile(support_file)) + self.serverutils.add_files_to_mediasyncer(client, + [support_file], + update_db=False) + + # Syncing should work. + self.assertEqual(client.sync(), "OK") + + # Create temporary db file with expected results. + chksum = client.col.media._checksum(support_file) + sql = (""" + CREATE TABLE meta (dirMod int, lastUsn int); + + INSERT INTO `meta` (dirMod, lastUsn) VALUES (123456789,1); + + CREATE TABLE media ( + fname text not null primary key, + csum text, + mtime int not null, + dirty int not null + ); + + INSERT INTO `media` (fname, csum, mtime, dirty) VALUES ( + 'blue.jpg', + '%s', + 1441483037, + 0 + ); + + CREATE INDEX idx_media_dirty on media (dirty); + """ % chksum) + + temp_db_path = self.dbutils.create_sqlite_db_with_sql(sql) + + # Except for timestamps, the client's db after sync should be identical + # to the expected data. + self.assertFalse(self.dbutils.media_dbs_differ( + client.col.media.db._path, + temp_db_path + )) diff --git a/tests/sync_app_functional_test_base.py b/tests/sync_app_functional_test_base.py new file mode 100644 index 0000000..3702284 --- /dev/null +++ b/tests/sync_app_functional_test_base.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- + + +import os +import unittest +from webtest import TestApp + + +from ankisyncd.users import UserManager +from helpers.collection_utils import CollectionUtils +from helpers.db_utils import DBUtils +from helpers.file_utils import FileUtils +from helpers.mock_servers import MockRemoteServer +from helpers.monkey_patches import monkeypatch_db, unpatch_db +from helpers.server_utils import ServerUtils + + +class SyncAppFunctionalTestBase(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.fileutils = FileUtils() + cls.colutils = CollectionUtils() + cls.serverutils = ServerUtils() + cls.dbutils = DBUtils() + + @classmethod + def tearDownClass(cls): + cls.fileutils.clean_up() + cls.fileutils = None + + cls.colutils.clean_up() + cls.colutils = None + + cls.serverutils.clean_up() + cls.serverutils = None + + cls.dbutils.clean_up() + cls.dbutils = None + + def setUp(self): + monkeypatch_db() + + # Create temporary files and dirs the server will use. + self.server_paths = self.serverutils.create_server_paths() + + # Add a test user to the temp auth db the server will use. + self.user_manager = UserManager(self.server_paths['auth_db'], + self.server_paths['data_root']) + self.user_manager.add_user('testuser', 'testpassword') + + # Get absolute path to development ini file. + script_dir = os.path.dirname(os.path.realpath(__file__)) + ini_file_path = os.path.join(script_dir, + "assets", + "test.conf") + + # Create SyncApp instance using the dev ini file and the temporary + # paths. + self.server_app = self.serverutils.create_sync_app(self.server_paths, + ini_file_path) + + # Wrap the SyncApp object in TestApp instance for testing. + self.server_test_app = TestApp(self.server_app) + + # MockRemoteServer instance needed for testing normal collection + # syncing and for retrieving hkey for other tests. + self.mock_remote_server = MockRemoteServer(hkey=None, + server_test_app=self.server_test_app) + + def tearDown(self): + self.server_paths = {} + self.user_manager = None + + # Shut down server. + self.server_app.collection_manager.shutdown() + self.server_app = None + + self.client_server_connection = None + + unpatch_db()