From 10c4c8fcdfde9311b1ca44877d93427363b08dd4 Mon Sep 17 00:00:00 2001 From: Antoine Mazeas Date: Mon, 16 Jan 2023 02:14:05 +0100 Subject: [PATCH] Add mandatory MODDB: prefix as per maintainer wish + add more test coverage Signed-off-by: Antoine Mazeas --- docs/installers.rst | 7 +- lutris/installer/installer.py | 6 +- lutris/installer/installer_file.py | 2 +- lutris/util/moddb.py | 37 ++++++++++ lutris/util/moddb/__init__.py | 0 lutris/util/moddb/downloadhelper.py | 27 ------- setup.py | 1 - tests/test_moddb_helper.py | 109 +++++++++++++++++++++++++--- 8 files changed, 145 insertions(+), 44 deletions(-) create mode 100644 lutris/util/moddb.py delete mode 100644 lutris/util/moddb/__init__.py delete mode 100644 lutris/util/moddb/downloadhelper.py diff --git a/docs/installers.rst b/docs/installers.rst index d50eb935c..0a4095957 100644 --- a/docs/installers.rst +++ b/docs/installers.rst @@ -396,12 +396,13 @@ the platform rotates the actual download links every few hours, making it impractical to set these links as source url in installers. Lutris has routines to overcome this limitation (with blessing from moddb.com). When specifying a file hosted on moddb.com, please use the url of the files details -page (the one with the red "Download now" button). +page (the one with the red "Download now" button). You must prefix the URL +with ``MODDB:``. Example URLs for ModDB files:: - https://www.moddb.com/games/{game-title}/downloads/{file-title} - https://www.moddb.com/mods/{mod-title}/downloads/{file-title} + MODDB:https://www.moddb.com/games/{game-title}/downloads/{file-title} + MODDB:https://www.moddb.com/mods/{mod-title}/downloads/{file-title} Writing the installation script =============================== diff --git a/lutris/installer/installer.py b/lutris/installer/installer.py index 6168460fc..ed282f35b 100644 --- a/lutris/installer/installer.py +++ b/lutris/installer/installer.py @@ -15,7 +15,7 @@ from lutris.services import SERVICES from lutris.util.game_finder import find_linux_game_executable, find_windows_game_executable from lutris.util.gog import convert_gog_config_to_lutris, get_gog_config_from_path, get_gog_game_path from lutris.util.log import logger -from lutris.util.moddb import downloadhelper as moddbhelper +from lutris.util.moddb import ModDB class LutrisInstaller: # pylint: disable=too-many-instance-attributes @@ -159,8 +159,8 @@ class LutrisInstaller: # pylint: disable=too-many-instance-attributes # Run variable substitution on the URLs from the script for file in self.files: file.set_url(self.interpreter._substitute(file.url)) - if moddbhelper.is_moddb_url(file.url): - file.set_url(moddbhelper.get_moddb_download_url(file.url)) + if file.url.startswith("MODDB:"): + file.set_url(ModDB().transform_url(file.url[6:])) if installer_file_id and self.service: logger.info("Getting files for %s", installer_file_id) diff --git a/lutris/installer/installer_file.py b/lutris/installer/installer_file.py index cfeeea117..70c0d9a9e 100644 --- a/lutris/installer/installer_file.py +++ b/lutris/installer/installer_file.py @@ -143,7 +143,7 @@ class InstallerFile: def is_downloadable(self): """Return True if the file can be downloaded (even from the local filesystem)""" - return self.url.startswith(("http", "file")) + return self.url.startswith(("http", "file", "MODDB")) def uses_pga_cache(self, create=False): """Determines whether the installer files are stored in a PGA cache diff --git a/lutris/util/moddb.py b/lutris/util/moddb.py new file mode 100644 index 000000000..4003e5e22 --- /dev/null +++ b/lutris/util/moddb.py @@ -0,0 +1,37 @@ +"""Helper functions to assist downloading files from ModDB""" +import moddb +import re +import types + +MODDB_FQDN = 'https://www.moddb.com' +MODDB_URL_MATCHER = '^https://(www\.)?moddb\.com' + + +class ModDB: + def __init__(self, parse_page_method: types.MethodType = moddb.parse_page): + self.parse = parse_page_method + + def transform_url(self, moddb_permalink_url): + if not self._is_moddb_url(moddb_permalink_url): + raise RuntimeError("provided url must be from moddb.com") + + return MODDB_FQDN + self._autoselect_moddb_mirror(self._get_html_and_resolve_mirrors_list(moddb_permalink_url))._url + + def _is_moddb_url(self, url): + return re.match(MODDB_URL_MATCHER, url.lower()) is not None + + def _autoselect_moddb_mirror(self, mirrors_list): + # dumb autoselect for now: rank mirrors by capacity (lower is better), pick first (lowest load) + return sorted(mirrors_list, key=lambda m: m.capacity)[0] + + def _get_html_and_resolve_mirrors_list(self, moddb_permalink_url): + moddb_obj = self.parse(moddb_permalink_url) + if not isinstance(moddb_obj, moddb.pages.File): + raise RuntimeError("supplied url does not point to the page of a file hosted on moddb.com") + + mirrors_list = moddb_obj.get_mirrors() + if not any(mirrors_list): + raise RuntimeError("no available mirror for the file hosted on moddb.com") + + return mirrors_list + diff --git a/lutris/util/moddb/__init__.py b/lutris/util/moddb/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/lutris/util/moddb/downloadhelper.py b/lutris/util/moddb/downloadhelper.py deleted file mode 100644 index d500d6fb8..000000000 --- a/lutris/util/moddb/downloadhelper.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Helper functions to assist downloading files from ModDB""" -import moddb -import re - -MODDB_FQDN = 'https://www.moddb.com' -MODDB_URL_MATCHER = '^https://(www\.)?moddb\.com' - -def is_moddb_url(url): - return re.match(MODDB_URL_MATCHER, url.lower()) is not None - -def get_moddb_download_url(moddb_permalink_url): - return MODDB_FQDN + __autoselect_moddb_mirror(__get_html_and_resolve_mirrors_list(moddb_permalink_url))._url - -def __autoselect_moddb_mirror(mirrors_list): - # dumb autoselect for now: rank mirrors by capacity (lower is better), pick first (lowest load) - return sorted(mirrors_list, key=lambda m: m.capacity)[0] - -def __get_html_and_resolve_mirrors_list(moddb_permalink_url): - moddb_obj = moddb.parse_page(moddb_permalink_url) - if not isinstance(moddb_obj, moddb.File): - raise RuntimeError("supplied url does not point to the page of a file hosted on moddb.com") - - mirrors_list = moddb_obj.get_mirrors() - if not any(mirrors_list): - raise RuntimeError("no available mirror for the file hosted on moddb.com") - - return mirrors_list diff --git a/setup.py b/setup.py index e75215d9d..ce267b794 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,6 @@ setup( 'lutris.util.egs', 'lutris.util.graphics', 'lutris.util.mame', - 'lutris.util.moddb', 'lutris.util.steam', 'lutris.util.steam.vdf', 'lutris.util.retroarch', diff --git a/tests/test_moddb_helper.py b/tests/test_moddb_helper.py index 3ec7778e5..243de65a3 100644 --- a/tests/test_moddb_helper.py +++ b/tests/test_moddb_helper.py @@ -1,37 +1,128 @@ import unittest -from lutris.util.moddb import downloadhelper as moddb +import moddb +from lutris.util.moddb import ModDB class ModDBHelperTests(unittest.TestCase): + def setUp(self): + self.mirrors_list = [] + self.page_type = self.ModDBFileObj + self.helper_obj = ModDB(self.parse) + + def with_mirror(self, url: str, capacity: float): + self.mirrors_list.append(moddb.boxes.Mirror(url = url, capacity = capacity)) + return self + + def with_page_type(self, page_type): + self.page_type = page_type + + def parse(self, url): + return self.page_type(self.page_type, self.mirrors_list) + + class ModDBFileObj(moddb.pages.File): + def __init__(self, page_type, mirrors_list): + self.mirrors_list = mirrors_list + def get_mirrors(self): + return self.mirrors_list + + class ModDBSomeOtherObj: + def __init__(self, page_type, mirrors_list): + pass + + ## ctor + def test_ctor_default_method(self): + hlpr = ModDB() + self.assertEqual(hlpr.parse, moddb.parse_page) + + def test_ctor_custom_method(self): + def custom(): + pass + hlpr = ModDB(custom) + self.assertEqual(hlpr.parse, custom) + + ## transform_url + def test_transform_url_url_match_happy_path(self): + self \ + .with_mirror("/first_url", 12.4) + + moddb_url = 'https://moddb.com' + transformed = self.helper_obj.transform_url(moddb_url) + self.assertEqual(transformed, 'https://www.moddb.com/first_url') + + def test_transform_url_url_not_match_throws(self): + self \ + .with_mirror("/first_url", 12.4) + moddb_url = 'https://not_moddb.com' + with self.assertRaises(RuntimeError): + transformed = self.helper_obj.transform_url(moddb_url) + + def test_transform_url_page_type_correct_happy_path(self): + self \ + .with_mirror("/first_url", 12.4) \ + .with_page_type(self.ModDBFileObj) + moddb_url = 'https://moddb.com' + transformed = self.helper_obj.transform_url(moddb_url) + self.assertEqual(transformed, 'https://www.moddb.com/first_url') + + def test_transform_url_page_type_incorrect_throws(self): + self \ + .with_mirror("/first_url", 12.4) \ + .with_page_type(self.ModDBSomeOtherObj) + moddb_url = 'https://moddb.com' + with self.assertRaises(RuntimeError): + transformed = self.helper_obj.transform_url(moddb_url) + + def test_transform_url_single_mirror_happy_path(self): + self \ + .with_mirror("/first_url", 12.4) + moddb_url = 'https://moddb.com' + transformed = self.helper_obj.transform_url(moddb_url) + self.assertEqual(transformed, 'https://www.moddb.com/first_url') + + def test_transform_url_multiple_mirror_select_lowest_capacity(self): + self \ + .with_mirror("/first_url", 12.4) \ + .with_mirror("/second_url", 57.4) \ + .with_mirror("/lowest_load", 0) + moddb_url = 'https://moddb.com' + transformed = self.helper_obj.transform_url(moddb_url) + self.assertEqual(transformed, 'https://www.moddb.com/lowest_load') + + def test_transform_url_no_mirrors_throws(self): + moddb_url = 'https://moddb.com' + with self.assertRaises(RuntimeError): + transformed = self.helper_obj.transform_url(moddb_url) + + ## is_moddb_url def test_is_moddb_url_has_www_success(self): url = 'https://www.moddb.com/something' - self.assertTrue(moddb.is_moddb_url(url)) + self.assertTrue(self.helper_obj._is_moddb_url(url)) def test_is_moddb_url_no_slug_has_www_success(self): url = 'https://www.moddb.com' - self.assertTrue(moddb.is_moddb_url(url)) + self.assertTrue(self.helper_obj._is_moddb_url(url)) def test_is_moddb_url_no_www_success(self): url = 'https://moddb.com/something' - self.assertTrue(moddb.is_moddb_url(url)) + self.assertTrue(self.helper_obj._is_moddb_url(url)) def test_is_moddb_url_no_slug_no_www_success(self): url = 'https://moddb.com' - self.assertTrue(moddb.is_moddb_url(url)) + self.assertTrue(self.helper_obj._is_moddb_url(url)) def test_is_moddb_url_other_subdomain_failure(self): url = 'https://subdomain.moddb.com/something' - self.assertFalse(moddb.is_moddb_url(url)) + self.assertFalse(self.helper_obj._is_moddb_url(url)) def test_is_moddb_url_no_slug_other_subdomain_failure(self): url = 'https://subdomain.moddb.com' - self.assertFalse(moddb.is_moddb_url(url)) + self.assertFalse(self.helper_obj._is_moddb_url(url)) def test_is_moddb_url_random_domain_failure(self): url = 'https://somedomain.com/something' - self.assertFalse(moddb.is_moddb_url(url)) + self.assertFalse(self.helper_obj._is_moddb_url(url)) def test_is_moddb_url_no_slug_random_domain_failure(self): url = 'https://somedomain.com' - self.assertFalse(moddb.is_moddb_url(url)) + self.assertFalse(self.helper_obj._is_moddb_url(url))