Advanced ETag Comparison now supported

Sources/Vapor/Middleware/FileMiddleware.swift

@@ -9,7 +9,8 @@ public final class FileMiddleware: Middleware {
     private let publicDirectory: String
     private let defaultFile: String?
     private let directoryAction: DirectoryAction
-
+    private let advancedETagComparison: Bool
+
     public struct BundleSetupError: Equatable, Error {
 
         /// The description of this error.
@@ -22,17 +23,28 @@ public final class FileMiddleware: Middleware {
         public static let publicDirectoryIsNotAFolder: Self = .init(description: "Cannot find any actual folder for the given Public Directory")
     }
 
+    public struct ETagHashes: StorageKey {
+        public typealias Value = [String: FileHash]
+
+        public struct FileHash {
+            var lastModified: Date
+            var digest: SHA256Digest
+        }
+    }
+
     /// Creates a new `FileMiddleware`.
     ///
     /// - parameters:
     ///     - publicDirectory: The public directory to serve files from.
     ///     - defaultFile: The name of the default file to look for and serve if a request hits any public directory. Starting with `/` implies
     ///     an absolute path from the public directory root. If `nil`, no default files are served.
     ///     - directoryAction: Determines the action to take when the request doesn't have a trailing slash but matches a directory.
-    public init(publicDirectory: String, defaultFile: String? = nil, directoryAction: DirectoryAction = .none) {
+    ///     - advancedETagComparison: The method used when ETags are generated. If true, a byte-by-byte hash is created (and cached), otherwise a simple comparison based on the file's last modified date and size.
+    public init(publicDirectory: String, defaultFile: String? = nil, directoryAction: DirectoryAction = .none, advancedETagComparison: Bool = true) {
         self.publicDirectory = publicDirectory.addTrailingSlash()
         self.defaultFile = defaultFile
         self.directoryAction = directoryAction
+        self.advancedETagComparison = advancedETagComparison
     }
 
     public func respond(to request: Request, chainingTo next: Responder) -> EventLoopFuture<Response> {
@@ -86,10 +98,9 @@ public final class FileMiddleware: Middleware {
                 return next.respond(to: request)
             }
         }
-        
+
         // stream the file
-        let res = request.fileio.streamFile(at: absPath)
-        return request.eventLoop.makeSucceededFuture(res)
+        return request.fileio.streamFile(at: absPath, advancedETagComparison: advancedETagComparison)
     }
 
     /// Creates a new `FileMiddleware` for a server contained in an Xcode Project.

Sources/Vapor/Utilities/FileIO.swift

@@ -3,6 +3,8 @@ import NIOCore
 import NIOHTTP1
 import NIOPosix
 import Logging
+import Crypto
+import NIOConcurrencyHelpers
 
 extension Request {
     public var fileio: FileIO {
@@ -121,6 +123,7 @@ public struct FileIO {
     ///     - mediaType: HTTPMediaType, if not specified, will be created from file extension.
     ///     - onCompleted: Closure to be run on completion of stream.
     /// - returns: A `200 OK` response containing the file stream and appropriate headers.
+    @available(*, deprecated, message: "Use the new `streamFile` method which returns EventLoopFuture<Response>")
     public func streamFile(
         at path: String,
         chunkSize: Int = NonBlockingFileIO.defaultChunkSize,
@@ -156,7 +159,7 @@ public struct FileIO {
 
         // Respond with lastModified header
         headers.lastModified = HTTPHeaders.LastModified(value: modifiedAt)
-        
+
         // Generate ETag value, "HEX value of last modified date" + "-" + "file size"
         let fileETag = "\"\(modifiedAt.timeIntervalSince1970)-\(fileSize)\""
         headers.replaceOrAdd(name: .eTag, value: fileETag)
@@ -215,10 +218,134 @@ public struct FileIO {
                 onCompleted(result)
             }
         }, count: byteCount, byteBufferAllocator: request.byteBufferAllocator)
-        
+
         return response
     }
 
+    /// Generates a chunked `Response` for the specified file. This method respects values in
+    /// the `"ETag"` header and is capable of responding `304 Not Modified` if the file in question
+    /// has not been modified since last served. This method will also set the `"Content-Type"` header
+    /// automatically if an appropriate `MediaType` can be found for the file's suffix.
+    ///
+    ///     router.get("file-stream") { req in
+    ///         return req.fileio.streamFile(at: "/path/to/file.txt")
+    ///     }
+    ///
+    /// - parameters:
+    ///     - path: Path to file on the disk.
+    ///     - chunkSize: Maximum size for the file data chunks.
+    ///     - mediaType: HTTPMediaType, if not specified, will be created from file extension.
+    ///     - advancedETagComparison: The method used when ETags are generated. If true, a byte-by-byte hash is created (and cached), otherwise a simple comparison based on the file's last modified date and size.
+    ///     - onCompleted: Closure to be run on completion of stream.
+    /// - returns: A `200 OK` response containing the file stream and appropriate headers.
+    public func streamFile(
+        at path: String,
+        chunkSize: Int = NonBlockingFileIO.defaultChunkSize,
+        mediaType: HTTPMediaType? = nil,
+        advancedETagComparison: Bool,
+        onCompleted: @escaping (Result<Void, Error>) -> () = { _ in }
+    ) -> EventLoopFuture<Response> {
+        // Get file attributes for this file.
+        guard
+            let attributes = try? FileManager.default.attributesOfItem(atPath: path),
+            let modifiedAt = attributes[.modificationDate] as? Date,
+            let fileSize = (attributes[.size] as? NSNumber)?.intValue
+        else {
+            return request.eventLoop.makeSucceededFuture(Response(status: .internalServerError))
+        }
+
+        let contentRange: HTTPHeaders.Range?
+        if let rangeFromHeaders = request.headers.range {
+            if rangeFromHeaders.unit == .bytes && rangeFromHeaders.ranges.count == 1 {
+                contentRange = rangeFromHeaders
+            } else {
+                contentRange = nil
+            }
+        } else if request.headers.contains(name: .range) {
+            // Range header was supplied but could not be parsed i.e. it was invalid
+            request.logger.debug("Range header was provided in request but was invalid")
+            let response = Response(status: .badRequest)
+            return request.eventLoop.makeSucceededFuture(response)
+        } else {
+            contentRange = nil
+        }
+        // Create empty headers array.
+        var headers: HTTPHeaders = [:]
+
+        // Respond with lastModified header
+        headers.lastModified = HTTPHeaders.LastModified(value: modifiedAt)
+
+        var eTagFuture: EventLoopFuture<String>
+
+        if advancedETagComparison {
+            eTagFuture = generateETagHash(path: path, lastModified: modifiedAt)
+        } else {
+            // Generate ETag value, "HEX value of last modified date" + "-" + "file size"
+            eTagFuture = request.eventLoop.makeSucceededFuture("\"\(modifiedAt.timeIntervalSince1970)-\(fileSize)\"")
+        }
+
+        return eTagFuture.map { fileETag in
+            headers.replaceOrAdd(name: .eTag, value: fileETag)
+
+            // Check if file has been cached already and return NotModified response if the etags match
+            if fileETag == request.headers.first(name: .ifNoneMatch) {
+                // Per RFC 9110 here: https://www.rfc-editor.org/rfc/rfc9110.html#status.304
+                // and here: https://www.rfc-editor.org/rfc/rfc9110.html#name-content-encoding
+                // A 304 response MUST include the ETag header and a Content-Length header matching what the original resource's content length would have been were this a 200 response.
+                headers.replaceOrAdd(name: .contentLength, value: fileSize.description)
+                return Response(status: .notModified, version: .http1_1, headersNoUpdate: headers, body: .empty)
+            }
+
+            // Create the HTTP response.
+            let response = Response(status: .ok, headers: headers)
+            let offset: Int64
+            let byteCount: Int
+            if let contentRange = contentRange {
+                response.status = .partialContent
+                response.headers.add(name: .accept, value: contentRange.unit.serialize())
+                if let firstRange = contentRange.ranges.first {
+                    do {
+                        let range = try firstRange.asResponseContentRange(limit: fileSize)
+                        response.headers.contentRange = HTTPHeaders.ContentRange(unit: contentRange.unit, range: range)
+                        (offset, byteCount) = try firstRange.asByteBufferBounds(withMaxSize: fileSize, logger: request.logger)
+                    } catch {
+                        let response = Response(status: .badRequest)
+                        return response
+                    }
+                } else {
+                    offset = 0
+                    byteCount = fileSize
+                }
+            } else {
+                offset = 0
+                byteCount = fileSize
+            }
+            // Set Content-Type header based on the media type
+            // Only set Content-Type if file not modified and returned above.
+            if
+                let fileExtension = path.components(separatedBy: ".").last,
+                let type = mediaType ?? HTTPMediaType.fileExtension(fileExtension)
+            {
+                response.headers.contentType = type
+            }
+            response.body = .init(stream: { stream in
+                self.read(path: path, fromOffset: offset, byteCount: byteCount, chunkSize: chunkSize) { chunk in
+                    return stream.write(.buffer(chunk))
+                }.whenComplete { result in
+                    switch result {
+                    case .failure(let error):
+                        stream.write(.error(error), promise: nil)
+                    case .success:
+                        stream.write(.end, promise: nil)
+                    }
+                    onCompleted(result)
+                }
+            }, count: byteCount, byteBufferAllocator: request.byteBufferAllocator)
+
+            return response
+        }
+    }
+
     /// Private read method. `onRead` closure uses ByteBuffer and expects future return.
     /// There may be use in publicizing this in the future for reads that must be async.
     private func read(
@@ -265,11 +392,37 @@ public struct FileIO {
             done.whenComplete { _ in
                 try? fd.close()
             }
+
             return done
         } catch {
             return self.request.eventLoop.makeFailedFuture(error)
         }
     }
+
+    /// Generates a fresh ETag for a file or returns its currently cached one.
+    /// - Parameters:
+    ///   - path: The file's path.
+    ///   - lastModified: When the file was last modified.
+    /// - Returns: An `EventLoopFuture<String>` which holds the ETag.
+    private func generateETagHash(path: String, lastModified: Date) -> EventLoopFuture<String> {
+        return NIOLock().withLock {
+            var hashingDictionary = request.application.storage[FileMiddleware.ETagHashes.self] ?? [:]
+
+            if let hash = hashingDictionary[path], hash.lastModified == lastModified {
+                return request.eventLoop.makeSucceededFuture(hash.digest.hex)
+            } else {
+                return collectFile(at: path).map { buffer in
+                    let digest = SHA256.hash(data: buffer.readableBytesView)
+
+                    // update hash in dictionary
+                    hashingDictionary[path] = FileMiddleware.ETagHashes.FileHash(lastModified: lastModified, digest: digest)
+                    request.application.storage[FileMiddleware.ETagHashes.self] = hashingDictionary
+
+                    return digest.hex
+                }
+            }
+        }
+    }
 }
 
 extension HTTPHeaders.Range.Value {

Tests/VaporTests/FileTests.swift

@@ -9,8 +9,8 @@ final class FileTests: XCTestCase {
         let app = Application(.testing)
         defer { app.shutdown() }
 
-        app.get("file-stream") { req in
-            return req.fileio.streamFile(at: #file) { result in
+        app.get("file-stream") { req -> EventLoopFuture<Response> in
+            return req.fileio.streamFile(at: #file, advancedETagComparison: true) { result in
                 do {
                     try result.get()
                 } catch { 
@@ -30,8 +30,8 @@ final class FileTests: XCTestCase {
         let app = Application(.testing)
         defer { app.shutdown() }
 
-        app.get("file-stream") { req in
-            return req.fileio.streamFile(at: #file)
+        app.get("file-stream") { req -> EventLoopFuture<Response> in
+            return req.fileio.streamFile(at: #file, advancedETagComparison: true)
         }
 
         var headers = HTTPHeaders()
@@ -47,13 +47,13 @@ final class FileTests: XCTestCase {
         let app = Application(.testing)
         defer { app.shutdown() }
 
-        app.get("file-stream") { req -> Response in
+        app.get("file-stream") { req -> EventLoopFuture<Response> in
             var tmpPath: String
             repeat {
                 tmpPath = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString).path
             } while (FileManager.default.fileExists(atPath: tmpPath))
 
-            return req.fileio.streamFile(at: tmpPath) { result in
+            return req.fileio.streamFile(at: tmpPath, advancedETagComparison: true) { result in
                 do {
                     try result.get()
                     XCTFail("File Stream should have failed")
@@ -71,8 +71,8 @@ final class FileTests: XCTestCase {
         let app = Application(.testing)
         defer { app.shutdown() }
 
-        app.get("file-stream") { req in
-            return req.fileio.streamFile(at: #file) { result in
+        app.get("file-stream") { req -> EventLoopFuture<Response> in
+            return req.fileio.streamFile(at: #file, advancedETagComparison: true) { result in
                 do {
                     try result.get()
                 } catch {
@@ -103,8 +103,8 @@ final class FileTests: XCTestCase {
         let app = Application(.testing)
         defer { app.shutdown() }
 
-        app.get("file-stream") { req in
-            return req.fileio.streamFile(at: #file) { result in
+        app.get("file-stream") { req -> EventLoopFuture<Response> in
+            return req.fileio.streamFile(at: #file, advancedETagComparison: true) { result in
                 do {
                     try result.get()
                 } catch {
@@ -135,8 +135,8 @@ final class FileTests: XCTestCase {
         let app = Application(.testing)
         defer { app.shutdown() }
 
-        app.get("file-stream") { req in
-            return req.fileio.streamFile(at: #file) { result in
+        app.get("file-stream") { req -> EventLoopFuture<Response> in
+            return req.fileio.streamFile(at: #file, advancedETagComparison: true) { result in
                 do {
                     try result.get()
                 } catch {
@@ -167,8 +167,8 @@ final class FileTests: XCTestCase {
         let app = Application(.testing)
         defer { app.shutdown() }
 
-        app.get("file-stream") { req in
-            return req.fileio.streamFile(at: #file) { result in
+        app.get("file-stream") { req -> EventLoopFuture<Response> in
+            return req.fileio.streamFile(at: #file, advancedETagComparison: true) { result in
                 do {
                     try result.get()
                 } catch {
@@ -193,8 +193,8 @@ final class FileTests: XCTestCase {
         let app = Application(.testing)
         defer { app.shutdown() }
 
-        app.get("file-stream") { req in
-            return req.fileio.streamFile(at: #file) { result in
+        app.get("file-stream") { req -> EventLoopFuture<Response> in
+            return req.fileio.streamFile(at: #file, advancedETagComparison: true) { result in
                 do {
                     try result.get()
                 } catch {
@@ -219,8 +219,8 @@ final class FileTests: XCTestCase {
         let app = Application(.testing)
         defer { app.shutdown() }
 
-        app.get("file-stream") { req in
-            return req.fileio.streamFile(at: #file) { result in
+        app.get("file-stream") { req -> EventLoopFuture<Response> in
+            return req.fileio.streamFile(at: #file, advancedETagComparison: true) { result in
                 do {
                     try result.get()
                 } catch {
@@ -374,8 +374,8 @@ final class FileTests: XCTestCase {
         let app = Application(.testing)
         defer { app.shutdown() }
 
-        app.get("file-stream") { req in
-            return req.fileio.streamFile(at: #file)
+        app.get("file-stream") { req -> EventLoopFuture<Response> in
+            return req.fileio.streamFile(at: #file, advancedETagComparison: true)
         }
 
         var headers = HTTPHeaders()

Tests/VaporTests/RequestTests.swift

@@ -261,13 +261,13 @@ final class RequestTests: XCTestCase {
         app.http.client.configuration.redirectConfiguration = .disallow
 
         app.get("redirect_normal") {
-            $0.redirect(to: "foo", type: .normal)
+            $0.redirect(to: "foo", redirectType: .normal)
         }
         app.get("redirect_permanent") {
-            $0.redirect(to: "foo", type: .permanent)
+            $0.redirect(to: "foo", redirectType: .permanent)
         }
         app.post("redirect_temporary") {
-            $0.redirect(to: "foo", type: .temporary)
+            $0.redirect(to: "foo", redirectType: .temporary)
         }
 
         try app.server.start(address: .hostname("localhost", port: 8080))