fix(hvcs): prevent double url schemes urls in changelog (#676)

* fix(hvcs): prevent double protocol scheme urls in changelogs

  Due to a typo and conditional stripping of the url scheme the
  hvcs_domain and hvcs_api_domain values would contain protocol schemes
  when a user specified one but the defaults would not. It would cause
  the api_url and remote_url to end up as "https://https://domain.com"

* fix(bitbucket): correct url parsing & prevent double url schemes

* fix(gitea): correct url parsing & prevent double url schemes

* fix(github): correct url parsing & prevent double url schemes

* fix(gitlab): correct url parsing & prevent double url schemes

* test(hvcs): ensure api domains are derived correctly

---------

Co-authored-by: codejedi365 <codejedi365@gmail.com>

semantic_release/hvcs/bitbucket.py

@@ -10,6 +10,8 @@
 import os
 from functools import lru_cache
 
+from urllib3.util.url import Url, parse_url
+
 from semantic_release.hvcs._base import HvcsBase
 from semantic_release.hvcs.token_auth import TokenAuth
 from semantic_release.hvcs.util import build_requests_session
@@ -34,9 +36,11 @@
 class Bitbucket(HvcsBase):
     """Bitbucket helper class"""
 
-    API_VERSION = "2.0"
     DEFAULT_DOMAIN = "bitbucket.org"
-    DEFAULT_API_DOMAIN = "api.bitbucket.org"
+    DEFAULT_API_SUBDOMAIN_PREFIX = "api"
+    DEFAULT_API_DOMAIN = f"{DEFAULT_API_SUBDOMAIN_PREFIX}.{DEFAULT_DOMAIN}"
+    DEFAULT_API_PATH_CLOUD = "/2.0"
+    DEFAULT_API_PATH_ONPREM = "/rest/api/1.0"
     DEFAULT_ENV_TOKEN_NAME = "BITBUCKET_TOKEN"  # noqa: S105
 
     def __init__(
@@ -47,12 +51,51 @@ def __init__(
         token: str | None = None,
     ) -> None:
         self._remote_url = remote_url
-        self.hvcs_domain = hvcs_domain or self.DEFAULT_DOMAIN.replace("https://", "")
-        # ref: https://developer.atlassian.com/cloud/bitbucket/rest/intro/#uri-uuid
-        self.hvcs_api_domain = hvcs_api_domain or self.DEFAULT_API_DOMAIN.replace(
-            "https://", ""
-        )
-        self.api_url = f"https://{self.hvcs_api_domain}/{self.API_VERSION}"
+
+        domain_url = parse_url(hvcs_domain or self.DEFAULT_DOMAIN)
+
+        # Strip any scheme, query or fragment from the domain
+        self.hvcs_domain = Url(
+            host=domain_url.host, port=domain_url.port, path=domain_url.path
+        ).url.rstrip("/")
+
+        if self.hvcs_domain == self.DEFAULT_DOMAIN:
+            # BitBucket Cloud detected, which means it uses a separate api domain
+            self.hvcs_api_domain = self.DEFAULT_API_DOMAIN
+
+            # ref: https://developer.atlassian.com/cloud/bitbucket/rest/intro/#uri-uuid
+            self.api_url = Url(
+                scheme="https",
+                host=self.hvcs_api_domain,
+                path=self.DEFAULT_API_PATH_CLOUD
+            ).url.rstrip("/")
+
+        else:
+            # BitBucket Server (on premise) detected, which uses a path prefix for the api
+            # ref: https://developer.atlassian.com/server/bitbucket/how-tos/command-line-rest/
+            api_domain_parts = parse_url(
+                hvcs_api_domain
+                or Url(
+                    # infer from Domain url and append the api path
+                    scheme=domain_url.scheme,
+                    host=self.hvcs_domain,
+                    path=self.DEFAULT_API_PATH_ONPREM,
+                ).url
+            )
+
+            # Strip any scheme, query or fragment from the api domain
+            self.hvcs_api_domain = Url(
+                host=api_domain_parts.host,
+                port=api_domain_parts.port,
+                path=str.replace(api_domain_parts.path or "", self.DEFAULT_API_PATH_ONPREM, ""),
+            ).url.rstrip("/")
+
+            self.api_url = Url(
+                scheme=api_domain_parts.scheme or "https",
+                host=self.hvcs_api_domain,
+                path=self.DEFAULT_API_PATH_ONPREM
+            ).url.rstrip("/")
+
         self.token = token
         auth = None if not self.token else TokenAuth(self.token)
         self.session = build_requests_session(auth=auth)

semantic_release/hvcs/gitea.py

@@ -37,7 +37,6 @@ class Gitea(HvcsBase):
 
     DEFAULT_DOMAIN = "gitea.com"
     DEFAULT_API_PATH = "/api/v1"
-    DEFAULT_API_DOMAIN = f"{DEFAULT_DOMAIN}{DEFAULT_API_PATH}"
     DEFAULT_ENV_TOKEN_NAME = "GITEA_TOKEN"  # noqa: S105
 
     # pylint: disable=super-init-not-called
@@ -74,10 +73,14 @@ def __init__(
         self.hvcs_api_domain = Url(
             host=api_domain_parts.host,
             port=api_domain_parts.port,
-            path=api_domain_parts.path,
+            path=str.replace(api_domain_parts.path or "", self.DEFAULT_API_PATH, ""),
         ).url.rstrip("/")
 
-        self.api_url = f"https://{self.hvcs_api_domain}"
+        self.api_url = Url(
+            scheme=api_domain_parts.scheme or "https",
+            host=self.hvcs_api_domain,
+            path=self.DEFAULT_API_PATH,
+        ).url.rstrip("/")
 
         self.token = token
         auth = None if not self.token else TokenAuth(self.token)

semantic_release/hvcs/github.py

@@ -9,6 +9,7 @@
 from functools import lru_cache
 
 from requests import HTTPError
+from urllib3.util.url import Url, parse_url
 
 from semantic_release.helpers import logged_function
 from semantic_release.hvcs._base import HvcsBase
@@ -36,8 +37,8 @@ class Github(HvcsBase):
     """Github helper class"""
 
     DEFAULT_DOMAIN = "github.com"
-    DEFAULT_API_DOMAIN = "api.github.com"
-    DEFAULT_UPLOAD_DOMAIN = "uploads.github.com"
+    DEFAULT_API_SUBDOMAIN_PREFIX = "api"
+    DEFAULT_API_DOMAIN = f"{DEFAULT_API_SUBDOMAIN_PREFIX}.{DEFAULT_DOMAIN}"
     DEFAULT_ENV_TOKEN_NAME = "GH_TOKEN"  # noqa: S105
 
     def __init__(
@@ -50,19 +51,34 @@ def __init__(
         self._remote_url = remote_url
 
         # ref: https://docs.github.com/en/actions/reference/environment-variables#default-environment-variables
-        self.hvcs_domain = hvcs_domain or os.getenv(
-            "GITHUB_SERVER_URL", self.DEFAULT_DOMAIN
-        ).replace("https://", "")
+        domain_url = parse_url(
+            hvcs_domain or os.getenv("GITHUB_SERVER_URL", "") or self.DEFAULT_DOMAIN
+        )
+
+        # Strip any scheme, query or fragment from the domain
+        self.hvcs_domain = Url(
+            host=domain_url.host, port=domain_url.port, path=domain_url.path
+        ).url.rstrip("/")
 
-        # not necessarily prefixed with "api." in the case of a custom domain, so
-        # can't just default to "api.github.com"
         # ref: https://docs.github.com/en/actions/reference/environment-variables#default-environment-variables
-        self.hvcs_api_domain = hvcs_api_domain or os.getenv(
-            "GITHUB_API_URL", self.DEFAULT_API_DOMAIN
-        ).replace("https://", "")
+        api_domain_parts = parse_url(
+            hvcs_api_domain
+            or os.getenv("GITHUB_API_URL", "")
+            or Url(
+                # infer from Domain url and prepend the default api subdomain
+                scheme=domain_url.scheme,
+                host=f"{self.DEFAULT_API_SUBDOMAIN_PREFIX}.{self.hvcs_domain}",
+            ).url
+        )
+
+        # Strip any scheme, query or fragment from the api domain
+        self.hvcs_api_domain = Url(
+            host=api_domain_parts.host,
+            port=api_domain_parts.port,
+            path=api_domain_parts.path,
+        ).url.rstrip("/")
 
         self.api_url = f"https://{self.hvcs_api_domain}"
-        self.upload_url = f"https://{self.DEFAULT_UPLOAD_DOMAIN}"
 
         self.token = token
         auth = None if not self.token else TokenAuth(self.token)

semantic_release/hvcs/gitlab.py

@@ -6,9 +6,9 @@
 import mimetypes
 import os
 from functools import lru_cache
-from urllib.parse import urlsplit
 
 import gitlab
+from urllib3.util.url import Url, parse_url
 
 from semantic_release.helpers import logged_function
 from semantic_release.hvcs._base import HvcsBase
@@ -44,6 +44,7 @@ class Gitlab(HvcsBase):
     # It is missing the permission to push to the repository, but has all others (releases, packages, etc.)
 
     DEFAULT_DOMAIN = "gitlab.com"
+    DEFAULT_API_PATH = "/api/v4"
 
     def __init__(
         self,
@@ -53,26 +54,44 @@ def __init__(
         token: str | None = None,
     ) -> None:
         self._remote_url = remote_url
-        self.hvcs_domain = (
-            hvcs_domain or self._domain_from_environment() or self.DEFAULT_DOMAIN
+
+        domain_url = parse_url(
+            hvcs_domain or os.getenv("CI_SERVER_URL", "") or self.DEFAULT_DOMAIN
         )
-        self.hvcs_api_domain = hvcs_api_domain or self.hvcs_domain.replace(
-            "https://", ""
+
+        # Strip any scheme, query or fragment from the domain
+        self.hvcs_domain = Url(
+            host=domain_url.host, port=domain_url.port, path=domain_url.path
+        ).url.rstrip("/")
+
+        api_domain_parts = parse_url(
+            hvcs_api_domain
+            or os.getenv("CI_API_V4_URL", "")
+            or Url(
+                # infer from Domain url and append the default api path
+                scheme=domain_url.scheme,
+                host=self.hvcs_domain,
+                path=self.DEFAULT_API_PATH,
+            ).url
         )
-        self.api_url = os.getenv("CI_SERVER_URL", f"https://{self.hvcs_api_domain}")
+
+        # Strip any scheme, query or fragment from the api domain
+        self.hvcs_api_domain = Url(
+            host=api_domain_parts.host,
+            port=api_domain_parts.port,
+            path=str.replace(api_domain_parts.path or "", self.DEFAULT_API_PATH, ""),
+        ).url.rstrip("/")
+
+        self.api_url = Url(
+            scheme=api_domain_parts.scheme or "https",
+            host=self.hvcs_api_domain,
+            path=self.DEFAULT_API_PATH,
+        ).url.rstrip("/")
 
         self.token = token
         auth = None if not self.token else TokenAuth(self.token)
         self.session = build_requests_session(auth=auth)
 
-    @staticmethod
-    def _domain_from_environment() -> str | None:
-        """Use Gitlab-CI environment variable to get the server domain, if available"""
-        if "CI_SERVER_URL" in os.environ:
-            url = urlsplit(os.environ["CI_SERVER_URL"])
-            return f"{url.netloc}{url.path}".rstrip("/")
-        return os.getenv("CI_SERVER_HOST")
-
     @lru_cache(maxsize=1)
     def _get_repository_owner_and_name(self) -> tuple[str, str]:
         """

tests/unit/semantic_release/hvcs/test_bitbucket.py

@@ -6,7 +6,7 @@
 
 from semantic_release.hvcs.bitbucket import Bitbucket
 
-from tests.const import EXAMPLE_REPO_NAME, EXAMPLE_REPO_OWNER
+from tests.const import EXAMPLE_HVCS_DOMAIN, EXAMPLE_REPO_NAME, EXAMPLE_REPO_OWNER
 
 
 @pytest.fixture
@@ -16,11 +16,54 @@ def default_bitbucket_client():
 
 
 @pytest.mark.parametrize(
-    (
-        "patched_os_environ, hvcs_domain, hvcs_api_domain, "
-        "expected_hvcs_domain, expected_hvcs_api_domain"
+    str.join(
+        ", ",
+        [
+            "patched_os_environ",
+            "hvcs_domain",
+            "hvcs_api_domain",
+            "expected_hvcs_domain",
+            "expected_hvcs_api_domain",
+        ],
     ),
-    [({}, None, None, Bitbucket.DEFAULT_DOMAIN, Bitbucket.DEFAULT_API_DOMAIN)],
+    [
+        # Default values (BitBucket Cloud)
+        ({}, None, None, Bitbucket.DEFAULT_DOMAIN, Bitbucket.DEFAULT_API_DOMAIN),
+        (
+            # Explicitly set default values
+            {},
+            f"https://{Bitbucket.DEFAULT_DOMAIN}",
+            f"https://{Bitbucket.DEFAULT_API_DOMAIN}",
+            Bitbucket.DEFAULT_DOMAIN,
+            Bitbucket.DEFAULT_API_DOMAIN
+        ),
+        (
+            # Explicitly defined api
+            {},
+            f"https://{EXAMPLE_HVCS_DOMAIN}",
+            f"https://api.{EXAMPLE_HVCS_DOMAIN}",
+            EXAMPLE_HVCS_DOMAIN,
+            f"api.{EXAMPLE_HVCS_DOMAIN}",
+        ),
+        (
+            # Custom domain for on premise BitBucket Server (derive api endpoint)
+            # No env vars as CI is handled by Bamboo or Jenkins Integration
+            {},
+            f"https://{EXAMPLE_HVCS_DOMAIN}",
+            None,
+            EXAMPLE_HVCS_DOMAIN,
+            EXAMPLE_HVCS_DOMAIN,
+        ),
+        (
+            # Custom domain with path prefix
+            # No env vars as CI is handled by Bamboo or Jenkins (which require user defined defaults)
+            {},
+            "special.custom.server/bitbucket",
+            None,
+            "special.custom.server/bitbucket",
+            "special.custom.server/bitbucket",
+        ),
+    ],
 )
 @pytest.mark.parametrize(
     "remote_url",
@@ -39,6 +82,13 @@ def test_bitbucket_client_init(
     remote_url,
     token,
 ):
+    # API paths are different in BitBucket Cloud (bitbucket.org) vs BitBucket Data Center
+    expected_api_url = (
+        f"https://{expected_hvcs_api_domain}/2.0"
+        if expected_hvcs_domain == "bitbucket.org"
+        else f"https://{expected_hvcs_api_domain}/rest/api/1.0"
+    )
+
     with mock.patch.dict(os.environ, patched_os_environ, clear=True):
         client = Bitbucket(
             remote_url=remote_url,
@@ -47,11 +97,11 @@ def test_bitbucket_client_init(
             token=token,
         )
 
-        assert client.hvcs_domain == expected_hvcs_domain
-        assert client.hvcs_api_domain == expected_hvcs_api_domain
-        assert client.api_url == f"https://{client.hvcs_api_domain}/2.0"
-        assert client.token == token
-        assert client._remote_url == remote_url
+        assert expected_hvcs_domain == client.hvcs_domain
+        assert expected_hvcs_api_domain == client.hvcs_api_domain
+        assert expected_api_url == client.api_url
+        assert token == client.token
+        assert remote_url == client._remote_url
         assert hasattr(client, "session")
         assert isinstance(getattr(client, "session", None), Session)