Add mandatory MODDB: prefix as per maintainer wish

+ add more test coverage

Signed-off-by: Antoine Mazeas <antoine@karthanis.net>

docs/installers.rst

@@ -396,12 +396,13 @@ the platform rotates the actual download links every few hours, making it
 impractical to set these links as source url in installers. Lutris has
 routines to overcome this limitation (with blessing from moddb.com). When
 specifying a file hosted on moddb.com, please use the url of the files details
-page (the one with the red "Download now" button).
+page (the one with the red "Download now" button). You must prefix the URL
+with ``MODDB:``.
 
 Example URLs for ModDB files::
 
-    https://www.moddb.com/games/{game-title}/downloads/{file-title}
-    https://www.moddb.com/mods/{mod-title}/downloads/{file-title}
+    MODDB:https://www.moddb.com/games/{game-title}/downloads/{file-title}
+    MODDB:https://www.moddb.com/mods/{mod-title}/downloads/{file-title}
 
 Writing the installation script
 ===============================

lutris/installer/installer.py

@@ -15,7 +15,7 @@ from lutris.services import SERVICES
 from lutris.util.game_finder import find_linux_game_executable, find_windows_game_executable
 from lutris.util.gog import convert_gog_config_to_lutris, get_gog_config_from_path, get_gog_game_path
 from lutris.util.log import logger
-from lutris.util.moddb import downloadhelper as moddbhelper
+from lutris.util.moddb import ModDB
 
 
 class LutrisInstaller:  # pylint: disable=too-many-instance-attributes
@@ -159,8 +159,8 @@ class LutrisInstaller:  # pylint: disable=too-many-instance-attributes
         # Run variable substitution on the URLs from the script
         for file in self.files:
             file.set_url(self.interpreter._substitute(file.url))
-            if moddbhelper.is_moddb_url(file.url):
-                file.set_url(moddbhelper.get_moddb_download_url(file.url))
+            if file.url.startswith("MODDB:"):
+                file.set_url(ModDB().transform_url(file.url[6:]))
 
         if installer_file_id and self.service:
             logger.info("Getting files for %s", installer_file_id)

lutris/installer/installer_file.py

@@ -143,7 +143,7 @@ class InstallerFile:
 
     def is_downloadable(self):
         """Return True if the file can be downloaded (even from the local filesystem)"""
-        return self.url.startswith(("http", "file"))
+        return self.url.startswith(("http", "file", "MODDB"))
 
     def uses_pga_cache(self, create=False):
         """Determines whether the installer files are stored in a PGA cache

lutris/util/moddb.py

@@ -0,0 +1,37 @@
+"""Helper functions to assist downloading files from ModDB"""
+import moddb
+import re
+import types
+
+MODDB_FQDN = 'https://www.moddb.com'
+MODDB_URL_MATCHER = '^https://(www\.)?moddb\.com'
+
+
+class ModDB:
+    def __init__(self, parse_page_method: types.MethodType = moddb.parse_page):
+        self.parse = parse_page_method
+
+    def transform_url(self, moddb_permalink_url):
+        if not self._is_moddb_url(moddb_permalink_url):
+            raise RuntimeError("provided url must be from moddb.com")
+
+        return MODDB_FQDN + self._autoselect_moddb_mirror(self._get_html_and_resolve_mirrors_list(moddb_permalink_url))._url
+
+    def _is_moddb_url(self, url):
+        return re.match(MODDB_URL_MATCHER, url.lower()) is not None
+
+    def _autoselect_moddb_mirror(self, mirrors_list):
+        # dumb autoselect for now: rank mirrors by capacity (lower is better), pick first (lowest load)
+        return sorted(mirrors_list, key=lambda m: m.capacity)[0]
+
+    def _get_html_and_resolve_mirrors_list(self, moddb_permalink_url):
+        moddb_obj = self.parse(moddb_permalink_url)
+        if not isinstance(moddb_obj, moddb.pages.File):
+            raise RuntimeError("supplied url does not point to the page of a file hosted on moddb.com")
+
+        mirrors_list = moddb_obj.get_mirrors()
+        if not any(mirrors_list):
+            raise RuntimeError("no available mirror for the file hosted on moddb.com")
+
+        return mirrors_list
+

lutris/util/moddb/downloadhelper.py

@@ -1,27 +0,0 @@
-"""Helper functions to assist downloading files from ModDB"""
-import moddb
-import re
-
-MODDB_FQDN = 'https://www.moddb.com'
-MODDB_URL_MATCHER = '^https://(www\.)?moddb\.com'
-
-def is_moddb_url(url):
-    return re.match(MODDB_URL_MATCHER, url.lower()) is not None
-
-def get_moddb_download_url(moddb_permalink_url):
-    return MODDB_FQDN + __autoselect_moddb_mirror(__get_html_and_resolve_mirrors_list(moddb_permalink_url))._url
-
-def __autoselect_moddb_mirror(mirrors_list):
-    # dumb autoselect for now: rank mirrors by capacity (lower is better), pick first (lowest load)
-    return sorted(mirrors_list, key=lambda m: m.capacity)[0]
-
-def __get_html_and_resolve_mirrors_list(moddb_permalink_url):
-    moddb_obj = moddb.parse_page(moddb_permalink_url)
-    if not isinstance(moddb_obj, moddb.File):
-        raise RuntimeError("supplied url does not point to the page of a file hosted on moddb.com")
-
-    mirrors_list = moddb_obj.get_mirrors()
-    if not any(mirrors_list):
-        raise RuntimeError("no available mirror for the file hosted on moddb.com")
-
-    return mirrors_list

setup.py

@@ -45,7 +45,6 @@ setup(
         'lutris.util.egs',
         'lutris.util.graphics',
         'lutris.util.mame',
-        'lutris.util.moddb',
         'lutris.util.steam',
         'lutris.util.steam.vdf',
         'lutris.util.retroarch',

tests/test_moddb_helper.py

@@ -1,37 +1,128 @@
 import unittest
-from lutris.util.moddb import downloadhelper as moddb
+import moddb
+from lutris.util.moddb import ModDB
 
 
 class ModDBHelperTests(unittest.TestCase):
+  def setUp(self):
+    self.mirrors_list = []
+    self.page_type = self.ModDBFileObj
+    self.helper_obj = ModDB(self.parse)
+
+  def with_mirror(self, url: str, capacity: float):
+    self.mirrors_list.append(moddb.boxes.Mirror(url = url, capacity = capacity))
+    return self
+
+  def with_page_type(self, page_type):
+    self.page_type = page_type
+
+  def parse(self, url):
+    return self.page_type(self.page_type, self.mirrors_list)
+
+  class ModDBFileObj(moddb.pages.File):
+    def __init__(self, page_type, mirrors_list):
+      self.mirrors_list = mirrors_list
+    def get_mirrors(self):
+      return self.mirrors_list
+
+  class ModDBSomeOtherObj:
+    def __init__(self, page_type, mirrors_list):
+      pass
+
+  ## ctor
+  def test_ctor_default_method(self):
+    hlpr = ModDB()
+    self.assertEqual(hlpr.parse, moddb.parse_page)
+
+  def test_ctor_custom_method(self):
+    def custom():
+      pass
+    hlpr = ModDB(custom)
+    self.assertEqual(hlpr.parse, custom)
+
+  ## transform_url
+  def test_transform_url_url_match_happy_path(self):
+    self \
+      .with_mirror("/first_url", 12.4)
+
+    moddb_url = 'https://moddb.com'
+    transformed = self.helper_obj.transform_url(moddb_url)
+    self.assertEqual(transformed, 'https://www.moddb.com/first_url')
+
+  def test_transform_url_url_not_match_throws(self):
+    self \
+      .with_mirror("/first_url", 12.4)
+    moddb_url = 'https://not_moddb.com'
+    with self.assertRaises(RuntimeError):
+        transformed = self.helper_obj.transform_url(moddb_url)
+
+  def test_transform_url_page_type_correct_happy_path(self):
+    self \
+      .with_mirror("/first_url", 12.4) \
+      .with_page_type(self.ModDBFileObj)
+    moddb_url = 'https://moddb.com'
+    transformed = self.helper_obj.transform_url(moddb_url)
+    self.assertEqual(transformed, 'https://www.moddb.com/first_url')
+
+  def test_transform_url_page_type_incorrect_throws(self):
+    self \
+      .with_mirror("/first_url", 12.4) \
+      .with_page_type(self.ModDBSomeOtherObj)
+    moddb_url = 'https://moddb.com'
+    with self.assertRaises(RuntimeError):
+        transformed = self.helper_obj.transform_url(moddb_url)
+
+  def test_transform_url_single_mirror_happy_path(self):
+    self \
+      .with_mirror("/first_url", 12.4)
+    moddb_url = 'https://moddb.com'
+    transformed = self.helper_obj.transform_url(moddb_url)
+    self.assertEqual(transformed, 'https://www.moddb.com/first_url')
+
+  def test_transform_url_multiple_mirror_select_lowest_capacity(self):
+    self \
+      .with_mirror("/first_url", 12.4) \
+      .with_mirror("/second_url", 57.4) \
+      .with_mirror("/lowest_load", 0)
+    moddb_url = 'https://moddb.com'
+    transformed = self.helper_obj.transform_url(moddb_url)
+    self.assertEqual(transformed, 'https://www.moddb.com/lowest_load')
+
+  def test_transform_url_no_mirrors_throws(self):
+    moddb_url = 'https://moddb.com'
+    with self.assertRaises(RuntimeError):
+        transformed = self.helper_obj.transform_url(moddb_url)
+
+  ## is_moddb_url
   def test_is_moddb_url_has_www_success(self):
     url = 'https://www.moddb.com/something'
-    self.assertTrue(moddb.is_moddb_url(url))
+    self.assertTrue(self.helper_obj._is_moddb_url(url))
   
   def test_is_moddb_url_no_slug_has_www_success(self):
     url = 'https://www.moddb.com'
-    self.assertTrue(moddb.is_moddb_url(url))
+    self.assertTrue(self.helper_obj._is_moddb_url(url))
 
   def test_is_moddb_url_no_www_success(self):
     url = 'https://moddb.com/something'
-    self.assertTrue(moddb.is_moddb_url(url))
+    self.assertTrue(self.helper_obj._is_moddb_url(url))
   
   def test_is_moddb_url_no_slug_no_www_success(self):
     url = 'https://moddb.com'
-    self.assertTrue(moddb.is_moddb_url(url))
+    self.assertTrue(self.helper_obj._is_moddb_url(url))
 
   def test_is_moddb_url_other_subdomain_failure(self):
     url = 'https://subdomain.moddb.com/something'
-    self.assertFalse(moddb.is_moddb_url(url))
+    self.assertFalse(self.helper_obj._is_moddb_url(url))
   
   def test_is_moddb_url_no_slug_other_subdomain_failure(self):
     url = 'https://subdomain.moddb.com'
-    self.assertFalse(moddb.is_moddb_url(url))
+    self.assertFalse(self.helper_obj._is_moddb_url(url))
 
   def test_is_moddb_url_random_domain_failure(self):
     url = 'https://somedomain.com/something'
-    self.assertFalse(moddb.is_moddb_url(url))
+    self.assertFalse(self.helper_obj._is_moddb_url(url))
   
   def test_is_moddb_url_no_slug_random_domain_failure(self):
     url = 'https://somedomain.com'
-    self.assertFalse(moddb.is_moddb_url(url))
+    self.assertFalse(self.helper_obj._is_moddb_url(url))