feat(downloader)!: Per client timeout config

Add support for setting the download client's timeout. This involves adding support for configuring each download client individually, as opposed to just with a list of URLs and as such requires a configuration change.
rpatterson · Nov 9, 2023 · 9fea973 · 9fea973
1 parent 7ed42e3
commit 9fea973
Show file tree

Hide file tree

Showing 16 changed files with 148 additions and 89 deletions.
diff --git a/README.rst b/README.rst
@@ -262,8 +262,8 @@ sub-command performs the following operations.
    For each of these groups in order, loop through each item in the group and:
 
    #. Check disk space against the margin configured by
-      ``download-clients/max-download-bandwidth`` and
-      ``download-clients/min-download-time-margin``
+      ``download-clients/*/max-download-bandwidth`` and
+      ``download-clients/*/min-download-time-margin``
 
    #. If there's sufficient disk space, remove any bandwidth limits set previously and
       continue to the next operation if any.

diff --git a/newsfragments/+download-client-timeout.feature.rst b/newsfragments/+download-client-timeout.feature.rst
@@ -0,0 +1,12 @@
+Add support for setting the download client's timeout.  This involves adding support for
+configuring each download client individually, as opposed to only a list of URLs, and as
+such requires a configuration change::
+
+  ...
+  download-clients:
+    Transmission:
+      url: "http://transmission:secret@localhost:9091/transmission/"
+      timeout: "30.0"
+      max-download-bandwidth: 100
+      min-download-time-margin: 600
+  ...
diff --git a/src/prunerr/downloadclient.py b/src/prunerr/downloadclient.py
@@ -50,21 +50,41 @@ def __repr__(self):
         """
         Readable, informative, and specific representation to ease debugging.
         """
-        return f"<{type(self).__name__} at {self.config.get('url')!r}>"
+        return f"<{type(self).__name__} at {self.config.get('name')!r}>"
 
     def update(self, config):
         """
         Update configuration, connect the RPC client, and update the list of items.
         """
         self.config = config
+
+        if not self.config.get("url"):
+            raise utils.PrunerrValidationError(
+                "Download client configuration must include a URL under"
+                f" `download-clients/*/url`: {self.runner.config_file}"
+            )
+
+        # Pull defaults from the example configuration:
+        example_confg = next(
+            iter(self.runner.example_confg["download-clients"].values()),
+        )
+        self.config.setdefault(
+            "max-download-bandwidth",
+            example_confg["max-download-bandwidth"],
+        )
+        self.config.setdefault(
+            "min-download-time-margin",
+            example_confg["min-download-time-margin"],
+        )
+
         self.config.setdefault(
             "password",
             urllib.parse.urlsplit(self.config["url"]).password,
         )
         self.config["url"] = utils.normalize_url(self.config["url"])
 
         # Configuration specific to Prunerr, IOW not taken from the download client
-        self.config["min-free-space"] = calc_free_space_margin(self.runner.config)
+        self.config["min-free-space"] = calc_free_space_margin(self.config)
         self.operations = prunerr.operations.PrunerrOperations(
             self,
             self.runner.config.get("indexers", {}),
@@ -93,6 +113,10 @@ def update(self, config):
             path=split_url.path,
             username=split_url.username,
             password=self.config["password"],
+            timeout=self.config.get(
+                "timeout",
+                transmission_rpc.constants.DEFAULT_TIMEOUT,
+            ),
         )
 
         # Update any Servarr references or data that depends on the download client
@@ -295,9 +319,7 @@ def resume_downloading(self, session):
         """
         Resume downloading if it's been stopped.
         """
-        speed_limit_down = self.runner.config["download-clients"][
-            "max-download-bandwidth"
-        ]
+        speed_limit_down = self.config["max-download-bandwidth"]
         if session.speed_limit_down_enabled and (
             not speed_limit_down or speed_limit_down != session.speed_limit_down
         ):
@@ -409,7 +431,7 @@ def config_from_url(auth_url):
     """
     auth_url_split = urllib.parse.urlsplit(auth_url)
     url = utils.normalize_url(auth_url)
-    return (url, {"url": url, "password": auth_url_split.password})
+    return {"url": url, "password": auth_url_split.password}
 
 
 def calc_free_space_margin(config):
@@ -424,7 +446,7 @@ def calc_free_space_margin(config):
     return (
         (
             # Convert bandwidth bits to bytes
-            config["download-clients"]["max-download-bandwidth"]
+            config["max-download-bandwidth"]
             / 8
         )
         * (
@@ -434,7 +456,7 @@ def calc_free_space_margin(config):
         )
         * (
             # Multiply by seconds of download time margin
-            config["download-clients"]["min-download-time-margin"]
+            config["min-download-time-margin"]
         )
     )
 

diff --git a/src/prunerr/home/.config/prunerr.yml b/src/prunerr/home/.config/prunerr.yml
@@ -20,26 +20,26 @@ servarrs:
     api-key: ""
     type: "radarr"
 download-clients:
-  ## Authenticated URLs for all download clients.  These will be reconciled with the
-  ## download clients configured in each Servarr instance by `Host`, `Port`, `Url Base`
-  ## and `Username`.  Download clients not attached to Servarr instance will still be
-  ## managed by Prunerr without the features that require Servarr:
-  urls:
-    - "http://transmission:secret@localhost:9091/transmission/"
-  ## Calculate an appropriate margin of disk space to keep free when deciding whether to
-  ## prune download items based the maximum download bandwidth/speed in Mbps and the
-  ## amount of time in seconds at that rate for which download clients should be able to
-  ## continue downloading without exhausting disk space.
-  ## 100 Mb/s, e.g. dedicated ethernet, default
-  max-download-bandwidth: 100
-  ## 60 seconds * 60 minutes = 1 hour, default
-  # min-download-time-margin: 3600
-  ## 60 seconds * 10 daemon poll margin = 10 minutes
-  min-download-time-margin: 600
-  ## Should the maximum download bandwidth/speed be set in the download client as a
-  ## limit when resuming downloads after previously stopping?  May be useful for QoS to
-  ## optimize real download throughput.
-  # resume-set-download-bandwidth-limit: false
+  Transmission:
+    ## Authenticated URLs for all download clients.  These will be reconciled with the
+    ## download clients configured in each Servarr instance by `Host`, `Port`, `Url Base`
+    ## and `Username`.  Download clients not attached to Servarr instance will still be
+    ## managed by Prunerr without the features that require Servarr:
+    url: "http://transmission:secret@localhost:9091/transmission/"
+    ## Calculate an appropriate margin of disk space to keep free when deciding whether to
+    ## prune download items based the maximum download bandwidth/speed in Mbps and the
+    ## amount of time in seconds at that rate for which download clients should be able to
+    ## continue downloading without exhausting disk space.
+    ## 100 Mb/s, e.g. dedicated ethernet, default
+    max-download-bandwidth: 100
+    ## 60 seconds * 60 minutes = 1 hour, default
+    # min-download-time-margin: 3600
+    ## 60 seconds * 10 daemon poll margin = 10 minutes
+    min-download-time-margin: 600
+    ## Should the maximum download bandwidth/speed be set in the download client as a
+    ## limit when resuming downloads after previously stopping?  May be useful for QoS to
+    ## optimize real download throughput.
+    # resume-set-download-bandwidth-limit: false
 indexers:
   ## Determine the indexer/tracker for download items by matching the hostnames of
   ## tracker announce and scrape URLs.  The matched indexer name,

diff --git a/src/prunerr/runner.py b/src/prunerr/runner.py
@@ -24,12 +24,6 @@
 logger = logging.getLogger(__name__)
 
 
-class PrunerrValidationError(Exception):
-    """
-    Incorrect Prunerr configuration.
-    """
-
-
 class PrunerrRunner:
     """
     Run Prunerr sub-commands across multiple Servarr instances and download clients.
@@ -55,11 +49,11 @@ def validate(self) -> dict:
         Parse the configuration file and raise meaningful errors for improper values.
 
         :return: The parsed YAML configuration as a Python mapping
-        :raises PrunerrValidationError: The YAML configuration file has a problem
+        :raises utils.PrunerrValidationError: The YAML configuration file has a problem
         """
         # Refresh the Prunerr configuration from the file
         if not self.config_file.is_file():
-            raise PrunerrValidationError(
+            raise utils.PrunerrValidationError(
                 f"Configuration file not found: {self.config_file}"
             )
         with self.config_file.open(encoding="utf-8") as config_opened:
@@ -75,21 +69,13 @@ def validate(self) -> dict:
                 self.config[top_key] = {}
 
         # Raise helpful errors for required values:
-        if not self.config.get("download-clients", {}).get("urls"):
-            raise PrunerrValidationError(
-                "Configuration file must include at least one URL under"
-                f" `download-clients/urls`: {self.config_file}"
+        if not self.config.get("download-clients"):
+            raise utils.PrunerrValidationError(
+                "Configuration file must include at least one download client"
+                f" configuration under  `download-clients`: {self.config_file}"
             )
 
         # Pull defaults from the example configuration:
-        self.config["download-clients"].setdefault(
-            "max-download-bandwidth",
-            self.example_confg["download-clients"]["max-download-bandwidth"],
-        )
-        self.config["download-clients"].setdefault(
-            "min-download-time-margin",
-            self.example_confg["download-clients"]["min-download-time-margin"],
-        )
         self.config.setdefault("daemon", {}).setdefault(
             "poll",
             self.example_confg["daemon"]["poll"],
@@ -126,10 +112,17 @@ def update(self) -> dict:
 
         # Update download client RPC clients
         # Download clients not connected to a Servarr instance
-        download_client_configs = dict(
-            prunerr.downloadclient.config_from_url(download_client_auth_url)
-            for download_client_auth_url in self.config["download-clients"]["urls"]
-        )
+        download_client_configs = {}
+        for download_client_name, download_client_config in self.config[
+            "download-clients"
+        ].items():
+            download_client_config.setdefault("name", download_client_name)
+            download_client_config.update(
+                prunerr.downloadclient.config_from_url(download_client_config["url"]),
+            )
+            download_client_configs[
+                download_client_config["url"]
+            ] = download_client_config
         # Reconcile with download clients defined in Servarr settings
         for servarr in self.servarrs.values():
             for download_client_url in servarr.download_clients.keys():

diff --git a/src/prunerr/utils.py b/src/prunerr/utils.py
@@ -54,6 +54,12 @@
 )
 
 
+class PrunerrValidationError(Exception):
+    """
+    Incorrect Prunerr configuration.
+    """
+
+
 def normalize_url(url):
     """
     Return the given URL in the same form regardless of port or authentication.

diff --git a/tests/prunerrtests/__init__.py b/tests/prunerrtests/__init__.py
@@ -128,9 +128,14 @@ class attributes to allow sub-classes to use the same convenience attributes but
         # Convenient access to the parsed configuration file
         with self.CONFIG.open(encoding="utf-8") as config_opened:
             self.config = yaml.safe_load(config_opened)
-        if "max-download-bandwidth" in self.config.get("download-clients", {}):
+        if self.config.get("download-clients"):
+            download_client_config = next(
+                iter(
+                    self.config["download-clients"].values(),
+                )
+            )
             self.min_free_space = prunerr.downloadclient.calc_free_space_margin(
-                self.config,
+                download_client_config,
             )
         # Convenient access to parsed mocked API/RPC request responses
         self.servarr_download_client_responses = {}
@@ -162,10 +167,9 @@ class attributes to allow sub-classes to use the same convenience attributes but
                     )
                 ]
         self.download_client_items_responses = {}
-        self.download_client_urls = self.config.get(
-            "download-clients",
-            {},
-        ).get("urls", [])
+        self.download_client_urls = [
+            config["url"] for config in self.config.get("download-clients", {}).values()
+        ]
         for download_client_url in self.download_client_urls:
             self.set_up_download_item_files(download_client_url)
 

diff --git a/tests/prunerrtests/home/daemon/.config/prunerr-example.yml b/tests/prunerrtests/home/daemon/.config/prunerr-example.yml
@@ -6,6 +6,7 @@ daemon:
   poll: 1
 servarrs:
 download-clients:
-  max-download-bandwidth: 100
-  min-download-time-margin: 3600
+  Transmission:
+    max-download-bandwidth: 100
+    min-download-time-margin: 3600
 indexers:
diff --git a/tests/prunerrtests/home/daemon/.config/prunerr.yml b/tests/prunerrtests/home/daemon/.config/prunerr.yml
@@ -13,10 +13,10 @@ servarrs:
     api-key: ""
     type: "radarr"
 download-clients:
-  urls:
-    - "http://transmission:secret@localhost:9091/transmission/"
-  max-download-bandwidth: 100
-  min-download-time-margin: 600
+  Transmission:
+    url: "http://transmission:secret@localhost:9091/transmission/"
+    max-download-bandwidth: 100
+    min-download-time-margin: 600
 indexers:
   hostnames:
     ExamplePrivateTracker:

diff --git a/tests/prunerrtests/home/download-client-only/.config/prunerr.yml b/tests/prunerrtests/home/download-client-only/.config/prunerr.yml
@@ -3,5 +3,5 @@
 # SPDX-License-Identifier: MIT
 
 download-clients:
-  urls:
-    - "http://transmission:secret@localhost:9091/transmission/"
+  Transmission:
+    url: "http://transmission:secret@localhost:9091/transmission/"
diff --git a/tests/prunerrtests/home/download-clients/.config/prunerr.yml b/tests/prunerrtests/home/download-clients/.config/prunerr.yml
@@ -13,14 +13,18 @@ servarrs:
     type: "radarr"
 
 download-clients:
-  urls:
-    - "http://transmission:secret@localhost:9091/transmission/"
-    - "http://transmission@192.168.1.1:9091/transmission/"
-    - "http://192.168.1.2/transmission/"
-    - "https://transmission.example.com"
-    - "https://transmission.foo.example.com"
-  max-download-bandwidth: 100
-  min-download-time-margin: 600
+  Local:
+    url: "http://transmission:secret@localhost:9091/transmission/"
+    max-download-bandwidth: 100
+    min-download-time-margin: 600
+  Foo:
+    url: "http://transmission@192.168.1.1:9091/transmission/"
+  Bar:
+    url: "http://192.168.1.2/transmission/"
+  Example:
+    url: "https://transmission.example.com"
+  FooExample:
+    url: "https://transmission.foo.example.com"
 indexers:
   hostnames:
     ExamplePrivateTracker:

diff --git a/tests/prunerrtests/home/download-items/.config/prunerr.yml b/tests/prunerrtests/home/download-items/.config/prunerr.yml
@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: MIT
 
 download-clients:
-  max-download-bandwidth: 100
-  min-download-time-margin: 3600
-  urls:
-    - "http://transmission:secret@localhost:9091/transmission/"
+  Transmission:
+    max-download-bandwidth: 100
+    min-download-time-margin: 3600
+    url: "http://transmission:secret@localhost:9091/transmission/"
diff --git a/tests/prunerrtests/home/move-exec/.config/prunerr.yml b/tests/prunerrtests/home/move-exec/.config/prunerr.yml
@@ -9,5 +9,5 @@ servarrs:
     type: "sonarr"
 
 download-clients:
-  urls:
-    - "http://transmission:secret@localhost:9091/transmission/"
+  Transmission:
+    url: "http://transmission:secret@localhost:9091/transmission/"
diff --git a/tests/prunerrtests/home/review-edge-cases/.config/prunerr.yml b/tests/prunerrtests/home/review-edge-cases/.config/prunerr.yml
@@ -12,8 +12,8 @@ servarrs:
     api-key: ""
     type: "radarr"
 download-clients:
-  urls:
-    - "http://transmission:secret@localhost:9091/transmission/"
+  Transmission:
+    url: "http://transmission:secret@localhost:9091/transmission/"
 indexers:
   hostnames:
     ExamplePrivateTracker: