diff --git a/Sources/Vapor/HTTP/Headers/HTTPHeaders+ContentRange.swift b/Sources/Vapor/HTTP/Headers/HTTPHeaders+ContentRange.swift index 3130786111..493c26b082 100644 --- a/Sources/Vapor/HTTP/Headers/HTTPHeaders+ContentRange.swift +++ b/Sources/Vapor/HTTP/Headers/HTTPHeaders+ContentRange.swift @@ -5,7 +5,7 @@ extension HTTPHeaders { /// The unit in which `ContentRange`s and `Range`s are specified. This is usually `bytes`. /// See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Range - public enum RangeUnit: Equatable { + public enum RangeUnit: Sendable, Equatable { case bytes case custom(value: String) @@ -21,7 +21,7 @@ extension HTTPHeaders { /// Represents the HTTP `Range` request header. /// See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Range - public struct Range: Equatable { + public struct Range: Sendable, Equatable { public let unit: RangeUnit public let ranges: [HTTPHeaders.Range.Value] @@ -134,7 +134,7 @@ extension HTTPHeaders.Range { /// Represents one value of the `Range` request header. /// /// See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Range - public enum Value: Equatable { + public enum Value: Sendable, Equatable { ///Integer with single trailing dash, e.g. `25-` case start(value: Int) ///Integer with single leading dash, e.g. `-25` diff --git a/Sources/Vapor/Middleware/FileMiddleware.swift b/Sources/Vapor/Middleware/FileMiddleware.swift index 9fc959bce2..d1c4b6867e 100644 --- a/Sources/Vapor/Middleware/FileMiddleware.swift +++ b/Sources/Vapor/Middleware/FileMiddleware.swift @@ -9,7 +9,8 @@ public final class FileMiddleware: Middleware { private let publicDirectory: String private let defaultFile: String? private let directoryAction: DirectoryAction - + private let advancedETagComparison: Bool + public struct BundleSetupError: Equatable, Error { /// The description of this error. @@ -22,6 +23,15 @@ public final class FileMiddleware: Middleware { public static let publicDirectoryIsNotAFolder: Self = .init(description: "Cannot find any actual folder for the given Public Directory") } + struct ETagHashes: StorageKey { + public typealias Value = [String: FileHash] + + public struct FileHash { + let lastModified: Date + let digestHex: String + } + } + /// Creates a new `FileMiddleware`. /// /// - parameters: @@ -29,10 +39,12 @@ public final class FileMiddleware: Middleware { /// - defaultFile: The name of the default file to look for and serve if a request hits any public directory. Starting with `/` implies /// an absolute path from the public directory root. If `nil`, no default files are served. /// - directoryAction: Determines the action to take when the request doesn't have a trailing slash but matches a directory. - public init(publicDirectory: String, defaultFile: String? = nil, directoryAction: DirectoryAction = .none) { + /// - advancedETagComparison: The method used when ETags are generated. If true, a byte-by-byte hash is created (and cached), otherwise a simple comparison based on the file's last modified date and size. + public init(publicDirectory: String, defaultFile: String? = nil, directoryAction: DirectoryAction = .none, advancedETagComparison: Bool = true) { self.publicDirectory = publicDirectory.addTrailingSlash() self.defaultFile = defaultFile self.directoryAction = directoryAction + self.advancedETagComparison = advancedETagComparison } public func respond(to request: Request, chainingTo next: Responder) -> EventLoopFuture { @@ -88,10 +100,9 @@ public final class FileMiddleware: Middleware { return next.respond(to: request) } } - + // stream the file - let res = request.fileio.streamFile(at: absPath) - return request.eventLoop.makeSucceededFuture(res) + return request.fileio.streamFile(at: absPath, advancedETagComparison: advancedETagComparison) } /// Creates a new `FileMiddleware` for a server contained in an Xcode Project. diff --git a/Sources/Vapor/Utilities/FileIO.swift b/Sources/Vapor/Utilities/FileIO.swift index 73c9a6fed7..7a3820989f 100644 --- a/Sources/Vapor/Utilities/FileIO.swift +++ b/Sources/Vapor/Utilities/FileIO.swift @@ -3,6 +3,7 @@ import NIOCore import NIOHTTP1 import NIOPosix import Logging +import Crypto import NIOConcurrencyHelpers extension Request { @@ -122,6 +123,7 @@ public struct FileIO: Sendable { /// - mediaType: HTTPMediaType, if not specified, will be created from file extension. /// - onCompleted: Closure to be run on completion of stream. /// - returns: A `200 OK` response containing the file stream and appropriate headers. + @available(*, deprecated, message: "Use the new `streamFile` method which returns EventLoopFuture") @preconcurrency public func streamFile( at path: String, chunkSize: Int = NonBlockingFileIO.defaultChunkSize, @@ -157,7 +159,7 @@ public struct FileIO: Sendable { // Respond with lastModified header headers.lastModified = HTTPHeaders.LastModified(value: modifiedAt) - + // Generate ETag value, "HEX value of last modified date" + "-" + "file size" let fileETag = "\"\(modifiedAt.timeIntervalSince1970)-\(fileSize)\"" headers.replaceOrAdd(name: .eTag, value: fileETag) @@ -218,10 +220,137 @@ public struct FileIO: Sendable { onCompleted(result) } }, count: byteCount, byteBufferAllocator: request.byteBufferAllocator) - + return response } + /// Generates a chunked `Response` for the specified file. This method respects values in + /// the `"ETag"` header and is capable of responding `304 Not Modified` if the file in question + /// has not been modified since last served. If `advancedETagComparison` is set to true, + /// the response will have its ETag field set to a byte-by-byte hash of the requested file. If set to false, a simple ETag consisting of the last modified date and file size + /// will be used. This method will also set the `"Content-Type"` header + /// automatically if an appropriate `MediaType` can be found for the file's suffix. + /// + /// router.get("file-stream") { req in + /// return req.fileio.streamFile(at: "/path/to/file.txt") + /// } + /// + /// - parameters: + /// - path: Path to file on the disk. + /// - chunkSize: Maximum size for the file data chunks. + /// - mediaType: HTTPMediaType, if not specified, will be created from file extension. + /// - advancedETagComparison: The method used when ETags are generated. If true, a byte-by-byte hash is created (and cached), otherwise a simple comparison based on the file's last modified date and size. + /// - onCompleted: Closure to be run on completion of stream. + /// - returns: A `200 OK` response containing the file stream and appropriate headers. + public func streamFile( + at path: String, + chunkSize: Int = NonBlockingFileIO.defaultChunkSize, + mediaType: HTTPMediaType? = nil, + advancedETagComparison: Bool, + onCompleted: @escaping @Sendable (Result) -> () = { _ in } + ) -> EventLoopFuture { + // Get file attributes for this file. + guard + let attributes = try? FileManager.default.attributesOfItem(atPath: path), + let modifiedAt = attributes[.modificationDate] as? Date, + let fileSize = (attributes[.size] as? NSNumber)?.intValue + else { + return request.eventLoop.makeSucceededFuture(Response(status: .internalServerError)) + } + + let contentRange: HTTPHeaders.Range? + if let rangeFromHeaders = request.headers.range { + if rangeFromHeaders.unit == .bytes && rangeFromHeaders.ranges.count == 1 { + contentRange = rangeFromHeaders + } else { + contentRange = nil + } + } else if request.headers.contains(name: .range) { + // Range header was supplied but could not be parsed i.e. it was invalid + request.logger.debug("Range header was provided in request but was invalid") + let response = Response(status: .badRequest) + return request.eventLoop.makeSucceededFuture(response) + } else { + contentRange = nil + } + + var eTagFuture: EventLoopFuture + + if advancedETagComparison { + eTagFuture = generateETagHash(path: path, lastModified: modifiedAt) + } else { + // Generate ETag value, "last modified date in epoch time" + "-" + "file size" + eTagFuture = request.eventLoop.makeSucceededFuture("\"\(modifiedAt.timeIntervalSince1970)-\(fileSize)\"") + } + + return eTagFuture.map { fileETag in + // Create empty headers array. + var headers: HTTPHeaders = [:] + + // Respond with lastModified header + headers.lastModified = HTTPHeaders.LastModified(value: modifiedAt) + + headers.replaceOrAdd(name: .eTag, value: fileETag) + + // Check if file has been cached already and return NotModified response if the etags match + if fileETag == request.headers.first(name: .ifNoneMatch) { + // Per RFC 9110 here: https://www.rfc-editor.org/rfc/rfc9110.html#status.304 + // and here: https://www.rfc-editor.org/rfc/rfc9110.html#name-content-encoding + // A 304 response MUST include the ETag header and a Content-Length header matching what the original resource's content length would have been were this a 200 response. + headers.replaceOrAdd(name: .contentLength, value: fileSize.description) + return Response(status: .notModified, version: .http1_1, headersNoUpdate: headers, body: .empty) + } + + // Create the HTTP response. + let response = Response(status: .ok, headers: headers) + let offset: Int64 + let byteCount: Int + if let contentRange = contentRange { + response.status = .partialContent + response.headers.add(name: .accept, value: contentRange.unit.serialize()) + if let firstRange = contentRange.ranges.first { + do { + let range = try firstRange.asResponseContentRange(limit: fileSize) + response.headers.contentRange = HTTPHeaders.ContentRange(unit: contentRange.unit, range: range) + (offset, byteCount) = try firstRange.asByteBufferBounds(withMaxSize: fileSize, logger: request.logger) + } catch { + let response = Response(status: .badRequest) + return response + } + } else { + offset = 0 + byteCount = fileSize + } + } else { + offset = 0 + byteCount = fileSize + } + // Set Content-Type header based on the media type + // Only set Content-Type if file not modified and returned above. + if + let fileExtension = path.components(separatedBy: ".").last, + let type = mediaType ?? HTTPMediaType.fileExtension(fileExtension) + { + response.headers.contentType = type + } + response.body = .init(stream: { stream in + self.read(path: path, fromOffset: offset, byteCount: byteCount, chunkSize: chunkSize) { chunk in + return stream.write(.buffer(chunk)) + }.whenComplete { result in + switch result { + case .failure(let error): + stream.write(.error(error), promise: nil) + case .success: + stream.write(.end, promise: nil) + } + onCompleted(result) + } + }, count: byteCount, byteBufferAllocator: request.byteBufferAllocator) + + return response + } + } + /// Private read method. `onRead` closure uses ByteBuffer and expects future return. /// There may be use in publicizing this in the future for reads that must be async. private func read( @@ -279,6 +408,26 @@ public struct FileIO: Sendable { } } } + + /// Generates a fresh ETag for a file or returns its currently cached one. + /// - Parameters: + /// - path: The file's path. + /// - lastModified: When the file was last modified. + /// - Returns: An `EventLoopFuture` which holds the ETag. + private func generateETagHash(path: String, lastModified: Date) -> EventLoopFuture { + if let hash = request.application.storage[FileMiddleware.ETagHashes.self]?[path], hash.lastModified == lastModified { + return request.eventLoop.makeSucceededFuture(hash.digestHex) + } else { + return collectFile(at: path).map { buffer in + let digest = SHA256.hash(data: buffer.readableBytesView) + + // update hash in dictionary + request.application.storage[FileMiddleware.ETagHashes.self]?[path] = FileMiddleware.ETagHashes.FileHash(lastModified: lastModified, digestHex: digest.hex) + + return digest.hex + } + } + } } extension HTTPHeaders.Range.Value { diff --git a/Tests/VaporTests/FileTests.swift b/Tests/VaporTests/FileTests.swift index c089588b74..0dd9345c63 100644 --- a/Tests/VaporTests/FileTests.swift +++ b/Tests/VaporTests/FileTests.swift @@ -3,17 +3,40 @@ import XCTest import Vapor import NIOCore import NIOHTTP1 +import Crypto final class FileTests: XCTestCase { func testStreamFile() throws { let app = Application(.testing) defer { app.shutdown() } + app.get("file-stream") { req -> EventLoopFuture in + return req.fileio.streamFile(at: #file, advancedETagComparison: true) { result in + do { + try result.get() + } catch { + XCTFail("File Stream should have succeeded") + } + } + } + + try app.testable(method: .running).test(.GET, "/file-stream") { res in + let test = "the quick brown fox" + XCTAssertNotNil(res.headers.first(name: .eTag)) + XCTAssertContains(res.body.string, test) + } + } + + @available(*, deprecated) + func testLegacyStreamFile() throws { + let app = Application(.testing) + defer { app.shutdown() } + app.get("file-stream") { req in return req.fileio.streamFile(at: #filePath) { result in do { try result.get() - } catch { + } catch { XCTFail("File Stream should have succeeded") } } @@ -30,8 +53,8 @@ final class FileTests: XCTestCase { let app = Application(.testing) defer { app.shutdown() } - app.get("file-stream") { req in - return req.fileio.streamFile(at: #filePath) + app.get("file-stream") { req -> EventLoopFuture in + return req.fileio.streamFile(at: #file, advancedETagComparison: true) } var headers = HTTPHeaders() @@ -47,13 +70,13 @@ final class FileTests: XCTestCase { let app = Application(.testing) defer { app.shutdown() } - app.get("file-stream") { req -> Response in + app.get("file-stream") { req -> EventLoopFuture in var tmpPath: String repeat { tmpPath = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString).path } while (FileManager.default.fileExists(atPath: tmpPath)) - return req.fileio.streamFile(at: tmpPath) { result in + return req.fileio.streamFile(at: tmpPath, advancedETagComparison: true) { result in do { try result.get() XCTFail("File Stream should have failed") @@ -66,13 +89,59 @@ final class FileTests: XCTestCase { XCTAssertTrue(res.body.string.isEmpty) } } + + func testAdvancedETagHeaders() throws { + let app = Application(.testing) + defer { app.shutdown() } + + app.get("file-stream") { req -> EventLoopFuture in + return req.fileio.streamFile(at: #file, advancedETagComparison: true) { result in + do { + try result.get() + } catch { + XCTFail("File Stream should have succeeded") + } + } + } + + try app.testable(method: .running).test(.GET, "/file-stream") { res in + let fileData = try Data(contentsOf: URL(fileURLWithPath: #file)) + let digest = SHA256.hash(data: fileData) + let eTag = res.headers.first(name: "etag") + XCTAssertEqual(eTag, digest.hex) + } + } + + func testSimpleETagHeaders() throws { + let app = Application(.testing) + defer { app.shutdown() } + + app.get("file-stream") { req -> EventLoopFuture in + return req.fileio.streamFile(at: #file, advancedETagComparison: false) { result in + do { + try result.get() + } catch { + XCTFail("File Stream should have succeeded") + } + } + } + + try app.testable(method: .running).test(.GET, "/file-stream") { res in + let attributes = try FileManager.default.attributesOfItem(atPath: #file) + let modifiedAt = attributes[.modificationDate] as! Date + let fileSize = (attributes[.size] as? NSNumber)!.intValue + let fileETag = "\"\(modifiedAt.timeIntervalSince1970)-\(fileSize)\"" + + XCTAssertEqual(res.headers.first(name: .eTag), fileETag) + } + } func testStreamFileContentHeaderTail() throws { let app = Application(.testing) defer { app.shutdown() } - app.get("file-stream") { req in - return req.fileio.streamFile(at: #filePath) { result in + app.get("file-stream") { req -> EventLoopFuture in + return req.fileio.streamFile(at: #file, advancedETagComparison: true) { result in do { try result.get() } catch { @@ -102,8 +171,8 @@ final class FileTests: XCTestCase { let app = Application(.testing) defer { app.shutdown() } - app.get("file-stream") { req in - return req.fileio.streamFile(at: #filePath) { result in + app.get("file-stream") { req -> EventLoopFuture in + return req.fileio.streamFile(at: #file, advancedETagComparison: true) { result in do { try result.get() } catch { @@ -111,7 +180,7 @@ final class FileTests: XCTestCase { } } } - + var headerRequest = HTTPHeaders() headerRequest.range = .init(unit: .bytes, ranges: [.start(value: 20)]) try app.testable(method: .running(port: 0)).test(.GET, "/file-stream", headers: headerRequest) { res in @@ -133,8 +202,8 @@ final class FileTests: XCTestCase { let app = Application(.testing) defer { app.shutdown() } - app.get("file-stream") { req in - return req.fileio.streamFile(at: #filePath) { result in + app.get("file-stream") { req -> EventLoopFuture in + return req.fileio.streamFile(at: #file, advancedETagComparison: true) { result in do { try result.get() } catch { @@ -165,7 +234,7 @@ final class FileTests: XCTestCase { defer { app.shutdown() } app.get("file-stream") { req in - return req.fileio.streamFile(at: #filePath) { result in + return req.fileio.streamFile(at: #file, advancedETagComparison: true) { result in do { try result.get() } catch { @@ -191,8 +260,8 @@ final class FileTests: XCTestCase { let app = Application(.testing) defer { app.shutdown() } - app.get("file-stream") { req in - return req.fileio.streamFile(at: #filePath) { result in + app.get("file-stream") { req -> EventLoopFuture in + return req.fileio.streamFile(at: #file, advancedETagComparison: true) { result in do { try result.get() } catch { @@ -217,8 +286,8 @@ final class FileTests: XCTestCase { let app = Application(.testing) defer { app.shutdown() } - app.get("file-stream") { req in - return req.fileio.streamFile(at: #filePath) { result in + app.get("file-stream") { req -> EventLoopFuture in + return req.fileio.streamFile(at: #file, advancedETagComparison: true) { result in do { try result.get() } catch { @@ -243,8 +312,8 @@ final class FileTests: XCTestCase { let app = Application(.testing) defer { app.shutdown() } - app.get("file-stream") { req in - return req.fileio.streamFile(at: #filePath) { result in + app.get("file-stream") { req -> EventLoopFuture in + return req.fileio.streamFile(at: #file, advancedETagComparison: true) { result in do { try result.get() } catch { @@ -423,8 +492,8 @@ final class FileTests: XCTestCase { let app = Application(.testing) defer { app.shutdown() } - app.get("file-stream") { req in - return req.fileio.streamFile(at: #filePath) + app.get("file-stream") { req -> EventLoopFuture in + return req.fileio.streamFile(at: #file, advancedETagComparison: true) } var headers = HTTPHeaders()