Merge pull request #433 from tigitz/add-http-download

Add http_download function to simplify downloading files

CHANGELOG.md

@@ -14,6 +14,7 @@
 * Add `context()` function in expression language to enable a task
 * Add `notificationTitle` property to `Context` to set the application name for
   notifications title
+* Add `http_download()` function to simplify the process of downloading files
 
 ### Minor
 
@@ -30,6 +31,8 @@
   functions instead
 * Deprecate `AfterApplicationInitializationEvent` event. Use
   `FunctionsResolvedEvent` instead
+* Deprecate `request()` in favor of `http_request()` for consistency with newly
+  introduced `http_*` function
 
 ### Fixes
 

doc/going-further/helpers/http-request.md

@@ -1,27 +1,62 @@
 # HTTP requests
 
-## The `request()` function
+## The `http_request()` function
 
-The `request()` function allows to make HTTP requests easily. It performs HTTP
-request and returns an instance of
+The `http_request()` function allows to make HTTP(S) requests easily. It
+performs HTTP(S) request and returns an instance of
 `Symfony\Contracts\HttpClient\ResponseInterface`:
 
 ```php
 use Castor\Attribute\AsTask;
 
 use function Castor\io;
-use function Castor\request;
+use function Castor\http_request;
 
 #[AsTask()]
 function foo()
 {
-    io()->writeln(request('GET', 'https://example.org')->getContent());
+    io()->writeln(http_request('GET', 'https://example.org')->getContent());
 }
 ```
 
+## The `http_download()` function
+
+The `http_download()` function simplifies the process of downloading files
+through HTTP(S) protocol. It writes the response content directly to a specified
+file path.
+
+The `stream` parameter controls whether the download is chunked (`true`, default
+value), which is useful for large files as it uses less memory, or in one go
+(`false`).
+
+```php
+use Castor\Attribute\AsTask;
+
+use function Castor\io;
+use function Castor\http_download;
+
+#[AsTask()]
+function foo()
+{
+    http_download('https://example.org/file', '/path/to/your/local/file', stream: true);
+    io()->writeln('Download completed!');
+}
+```
+
+When running Castor in verbose mode, `http_download()` outputs useful logs,
+including a progress indicator to track the download status.
+
+```
+18:55:09 INFO      [castor] Filename determined for http download ["filename" => "100MB-speedtest","url" => "http://eu-central-1.linodeobjects.com/speedtest/100MB-speedtest"]
+18:55:11 INFO      [castor] Download progress: 29.72 MB/100.00 MB (29.72%) at 18.40 MB/s, ETA: 3s ["url" => "http://eu-central-1.linodeobjects.com/speedtest/100MB-speedtest"]
+18:55:13 INFO      [castor] Download progress: 74.94 MB/100.00 MB (74.94%) at 20.73 MB/s, ETA: 1s ["url" => "http://eu-central-1.linodeobjects.com/speedtest/100MB-speedtest"]
+18:55:14 INFO      [castor] Download progress: 100.00 MB/100.00 MB (100.00%) at 20.69 MB/s, ETA: 0s ["url" => "http://eu-central-1.linodeobjects.com/speedtest/100MB-speedtest"]
+18:55:14 INFO      [castor] Download finished ["url" => "http://eu-central-1.linodeobjects.com/speedtest/100MB-speedtest","filePath" => "/www/castor/100MB-speedtest","size" => "100.00 MB"]
+```
+
 ## The `http_client()` function
 
-If you need to have a full control on the HTTP client, you can access the
+If you need to have a full control on the HTTP(S) client, you can access the
 `HttpClientInterface` directly with the `http_client()` function:
 
 ```php
@@ -41,6 +76,6 @@ function foo()
 }
 ```
 
-You can check
-the [Symfony documentation](https://symfony.com/doc/current/http_client.html)
-for more information about this component and how to use it.
+You can check the [Symfony
+documentation](https://symfony.com/doc/current/http_client.html) for more
+information about this component and how to use it.

doc/reference.md

@@ -31,7 +31,7 @@ Castor provides the following built-in functions:
 - [`open`](going-further/helpers/open.md)
 - [`output`](going-further/helpers/console-and-io.md#the-output-function)
 - [`parallel`](going-further/helpers/parallel.md#the-parallel-function)
-- [`request`](going-further/helpers/http-request.md#the-request-function)
+- [`http_request`](going-further/helpers/http-request.md#the-http-request-function)
 - [`run`](getting-started/run.md#the-run-function)
 - [`ssh_download`](going-further/helpers/ssh.md#the-ssh_download-function)
 - [`ssh_run`](going-further/helpers/ssh.md#the-ssh_run-function)

examples/http.php

@@ -1,16 +1,48 @@
 <?php
 
+namespace http;
+
 use Castor\Attribute\AsTask;
 
+use function Castor\fs;
+use function Castor\http_download;
+use function Castor\http_request;
 use function Castor\io;
-use function Castor\request;
 
 #[AsTask(description: 'Make HTTP request')]
-function httpRequest(): void
+function request(): void
 {
     $url = $_SERVER['ENDPOINT'] ?? 'https://example.com';
 
-    $response = request('GET', $url);
+    $response = http_request('GET', $url);
 
     io()->writeln($response->getContent());
 }
+
+#[AsTask(description: 'Download a file through HTTP')]
+function download(): void
+{
+    $downloadUrl = 'http://eu-central-1.linodeobjects.com/speedtest/100MB-speedtest';
+
+    if (isset($_SERVER['ENDPOINT'])) {
+        $downloadUrl = $_SERVER['ENDPOINT'] . '/big-file.php';
+    }
+
+    $downloadedFilePath = '/tmp/castor-tests/examples/http-download-dummy-file';
+
+    try {
+        $response = http_download($downloadUrl, $downloadedFilePath, stream: false);
+
+        io()->writeln(
+            sprintf(
+                'Successfully downloaded file of size "%s" from url "%s" to "%s" with status code "%s"',
+                filesize($downloadedFilePath),
+                $downloadUrl,
+                $downloadedFilePath,
+                $response->getStatusCode()
+            )
+        );
+    } finally {
+        fs()->remove($downloadedFilePath);
+    }
+}

phpstan-baseline.neon

@@ -35,6 +35,11 @@ parameters:
 			count: 1
 			path: src/functions.php
 
+		-
+			message: "#^Function Castor\\\\request\\(\\) has parameter \\$args with no type specified\\.$#"
+			count: 1
+			path: src/functions.php
+
 		-
 			message: "#^Function Castor\\\\ssh\\(\\) has parameter \\$args with no type specified\\.$#"
 			count: 1

src/Container.php

@@ -7,6 +7,7 @@
 use Castor\Fingerprint\FingerprintHelper;
 use Castor\Helper\Notifier;
 use Castor\Helper\Waiter;
+use Castor\Http\HttpDownloader;
 use Castor\Import\Importer;
 use Castor\Runner\ParallelRunner;
 use Castor\Runner\ProcessRunner;
@@ -37,6 +38,7 @@ public function __construct(
         public readonly Filesystem $fs,
         public readonly FingerprintHelper $fingerprintHelper,
         public readonly HttpClientInterface $httpClient,
+        public readonly HttpDownloader $httpDownloader,
         public readonly Importer $importer,
         public readonly InputInterface $input,
         public readonly Kernel $kernel,

src/Http/HttpDownloader.php

@@ -0,0 +1,174 @@
+<?php
+
+namespace Castor\Http;
+
+use Castor\Helper\PathHelper;
+use Psr\Log\LoggerInterface;
+use Psr\Log\NullLogger;
+use Symfony\Component\Filesystem\Filesystem;
+use Symfony\Contracts\HttpClient\HttpClientInterface;
+use Symfony\Contracts\HttpClient\ResponseInterface;
+
+/** @internal */
+class HttpDownloader
+{
+    public function __construct(
+        private readonly HttpClientInterface $httpClient,
+        private readonly Filesystem $filesystem,
+        private readonly LoggerInterface $logger = new NullLogger(),
+    ) {
+    }
+
+    /**
+     * @param string|null          $filePath Path to save the downloaded content. If null, the filename is determined from the URL or content disposition.
+     * @param bool                 $stream   Controls whether the download is chunked (`true`), which is useful for large files as it uses less memory, or in one go (`false`)
+     * @param array<string, mixed> $options  default values at {@see HttpClientInterface::OPTIONS_DEFAULTS}
+     */
+    public function download(string $url, ?string $filePath = null, string $method = 'GET', array $options = [], bool $stream = true): ResponseInterface
+    {
+        $this->logger->info('Starting http download', ['url' => $url]);
+
+        $lastLogTime = time();
+        $startTime = microtime(true);
+        $finalLogDone = false;
+        $userProvidedOnProgress = $options['on_progress'] ?? function (int $downloadedSize, int $totalSize) {};
+        $totalDownloadedSize = 0;
+
+        $options['on_progress'] = function (int $downloadedSize, int $totalSize) use ($userProvidedOnProgress, &$totalDownloadedSize, &$lastLogTime, &$finalLogDone, $url, $startTime) {
+            $totalDownloadedSize = $downloadedSize;
+            $percentage = $this->calculatePercentage($downloadedSize, $totalSize);
+            $speed = $this->calculateSpeed($downloadedSize, $startTime);
+            $formattedRemainingTime = $this->calculateRemainingTime($downloadedSize, $totalSize, (int) $speed);
+            $logMessage = $totalSize > 0
+                ? sprintf(
+                    'Download progress: %s/%s (%.2f%%) at %s/s, ETA: %s',
+                    $this->formatSize($downloadedSize),
+                    $this->formatSize($totalSize),
+                    $percentage,
+                    $this->formatSize((int) $speed),
+                    $formattedRemainingTime
+                )
+                : sprintf(
+                    'Download progress: %s at %s/s',
+                    $this->formatSize($downloadedSize),
+                    $this->formatSize((int) $speed)
+                );
+
+            if (
+                // Logs progress if 2 secs elapsed and below 100%
+                (time() - $lastLogTime >= 2 && $percentage < 100)
+                // Logs 100% only once; avoids multiple logs if data continues after reaching 100%
+                || ($percentage >= 100 && !$finalLogDone)
+            ) {
+                $this->logger->info($logMessage, ['url' => $url]);
+                $lastLogTime = time();
+
+                if ($percentage >= 100) {
+                    $finalLogDone = true;
+                }
+            }
+
+            $userProvidedOnProgress($downloadedSize, $totalSize);
+        };
+
+        $response = $this->httpClient->request($method, $url, $options);
+
+        if (null === $filePath) {
+            $filename = $this->extractFileName($response, $url);
+            $filePath = PathHelper::getRoot() . \DIRECTORY_SEPARATOR . $filename;
+            $this->logger->info('Filename determined for http download', ['filename' => $filename, 'url' => $url]);
+        }
+
+        $this->filesystem->mkdir(\dirname($filePath));
+
+        if (!$stream) {
+            $content = $response->getContent();
+            file_put_contents($filePath, $content);
+            $this->logger->info('Download finished', ['url' => $url, 'filePath' => $filePath, 'size' => $this->formatSize($totalDownloadedSize)]);
+
+            return $response;
+        }
+
+        $fileStream = fopen($filePath, 'w');
+        if (false === $fileStream) {
+            throw new \RuntimeException(sprintf('Cannot open file "%s" for writing.', $filePath));
+        }
+
+        foreach ($this->httpClient->stream($response) as $chunk) {
+            fwrite($fileStream, $chunk->getContent());
+        }
+
+        fclose($fileStream);
+
+        $this->logger->info('Download finished', ['url' => $url, 'filePath' => $filePath, 'size' => $this->formatSize($totalDownloadedSize)]);
+
+        return $response;
+    }
+
+    private function calculatePercentage(int $downloadedSize, int $totalSize): float
+    {
+        return $totalSize > 0 ? round(($downloadedSize / $totalSize) * 100, 2) : 0;
+    }
+
+    private function calculateSpeed(int $downloadedSize, float $startTime): float
+    {
+        $elapsedTime = microtime(true) - $startTime;
+
+        return $elapsedTime > 0 ? $downloadedSize / $elapsedTime : 0;
+    }
+
+    private function calculateRemainingTime(int $downloadedSize, int $totalSize, float $speed): string
+    {
+        $remainingTime = $speed > 0 ? ($totalSize - $downloadedSize) / $speed : 0;
+
+        return $this->formatTime((int) $remainingTime);
+    }
+
+    private function formatTime(float $seconds): string
+    {
+        if ($seconds < 60) {
+            return sprintf('%ds', $seconds);
+        }
+
+        $minutes = floor($seconds / 60);
+        $seconds %= 60;
+        if ($minutes < 60) {
+            return sprintf('%dm %ds', $minutes, $seconds);
+        }
+
+        $hours = floor($minutes / 60);
+        $minutes %= 60;
+
+        return sprintf('%dh %dm %ds', $hours, $minutes, $seconds);
+    }
+
+    private function formatSize(int $bytes): string
+    {
+        if ($bytes < 1024) {
+            return $bytes . ' B';
+        }
+
+        $units = ['KB', 'MB', 'GB', 'TB'];
+        $log = log($bytes, 1024);
+        $pow = floor($log);
+        $size = $bytes / (1024 ** $pow);
+
+        return sprintf('%.2f %s', $size, $units[$pow - 1]);
+    }
+
+    private function extractFileName(ResponseInterface $response, string $url): string
+    {
+        $disposition = $response->getHeaders(false)['content-disposition'][0] ?? null;
+        if (null !== $disposition && preg_match('/filename="([^"]+)"/', $disposition, $matches)) {
+            $filename = $matches[1];
+        } else {
+            $parsedUrl = parse_url($url, \PHP_URL_PATH);
+            if (!\is_string($parsedUrl)) {
+                throw new \RuntimeException(sprintf('Could not extract file name from URL: %s', $url));
+            }
+            $filename = basename($parsedUrl);
+        }
+
+        return $filename;
+    }
+}

src/functions.php

@@ -420,13 +420,29 @@ function get_cache(): CacheItemPoolInterface&CacheInterface
 }
 
 /**
- * @param array<string, mixed> $options
- *
- * @see HttpClientInterface::OPTIONS_DEFAULTS
+ * @deprecated Since castor/castor 0.16. Use Castor\http_request() instead
+ */
+function request(...$args): ResponseInterface
+{
+    trigger_deprecation('jolicode/castor', '0.16', 'The "%s()" function is deprecated, use "Castor\%s()" instead.', __FUNCTION__, 'http_request');
+
+    return http_request(...$args);
+}
+
+/**
+ * @param array<string, mixed> $options default values at {@see HttpClientInterface::OPTIONS_DEFAULTS}
+ */
+function http_request(string $method, string $url, array $options = []): ResponseInterface
+{
+    return Container::get()->httpClient->request($method, $url, $options);
+}
+
+/**
+ * @param array<string, mixed> $options default values at {@see HttpClientInterface::OPTIONS_DEFAULTS}
  */
-function request(string $method, string $url, array $options = []): ResponseInterface
+function http_download(string $url, ?string $filePath = null, string $method = 'GET', array $options = [], bool $stream = true): ResponseInterface
 {
-    return http_client()->request($method, $url, $options);
+    return Container::get()->httpDownloader->download($url, $filePath, $method, $options, $stream);
 }
 
 function http_client(): HttpClientInterface