Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Default max body size for streaming request body collection #2312

Merged
merged 16 commits into from May 1, 2020
Merged
Show file tree
Hide file tree
Changes from 8 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 4 additions & 0 deletions Sources/Vapor/Request/Request+Body.swift
Expand Up @@ -42,6 +42,10 @@ extension Request {
return buffer
}
case .collected(let buffer):
if let max = max, buffer.readableBytes > max {
return self.request.eventLoop.future(error: Abort(.payloadTooLarge))
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think we need this check here. The point of max is to avoid allocating too much memory. In this case, the memory has already been allocated we are just passing them a reference.

}

return self.request.eventLoop.makeSucceededFuture(buffer)
case .none:
return self.request.eventLoop.makeSucceededFuture(nil)
Expand Down
4 changes: 3 additions & 1 deletion Sources/Vapor/Routing/Routes.swift
@@ -1,11 +1,13 @@
public final class Routes: RoutesBuilder, CustomStringConvertible {
public var defaultMaxBodySize: Int?
public var all: [Route]

public var description: String {
return self.all.description
}

public init() {
self.defaultMaxBodySize = 1_000_000
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we do 1<<20 so it's 1MB and not 1MiB?

self.all = []
}

Expand Down
41 changes: 37 additions & 4 deletions Sources/Vapor/Routing/RoutesBuilder+Group.swift
Expand Up @@ -13,7 +13,7 @@ extension RoutesBuilder {
/// - path: Group path components separated by commas.
/// - returns: Newly created `Router` wrapped in the path.
public func grouped(_ path: PathComponent...) -> RoutesBuilder {
return HTTPRoutesGroup(root: self, path: path)
return HTTPRoutesGroup(root: self, path: path, defaultMaxBodySize: self.defaultMaxBodySize)
}

/// Creates a new `Router` that will automatically prepend the supplied path components.
Expand All @@ -29,7 +29,36 @@ extension RoutesBuilder {
/// - path: Group path components separated by commas.
/// - configure: Closure to configure the newly created `Router`.
public func group(_ path: PathComponent..., configure: (RoutesBuilder) throws -> ()) rethrows {
try configure(HTTPRoutesGroup(root: self, path: path))
try configure(HTTPRoutesGroup(root: self, path: path, defaultMaxBodySize: self.defaultMaxBodySize))
}

/// Creates a new `Router` with a different default max body size for its routes than the rest of the application.
///
/// let large = router.group(maxSize: 1_000_000)
/// large.post("image", use: self.uploadImage)
///
/// - parameters:
/// - defaultMaxBodySize: The maximum number of bytes that a request body can contain in the new group
/// `nil` means there is no limit.
/// - returns: A newly created `Router` with a new max body size.
public func group(maxSize defaultMaxBodySize: Int?) -> RoutesBuilder {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we should use the full defaultMaxBodySize label here. I'm not sure maxSize or even maxBodySize are clear enough.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should be grouped

return HTTPRoutesGroup(root: self, defaultMaxBodySize: defaultMaxBodySize)
}

/// Creates a new `Router` with a different default max body size for its routes than the rest of the application.
///
/// router.grouped(maxSize: 1_000_000) { large
/// large.post("image", use: self.uploadImage)
/// }
///
/// - parameters:
/// - defaultMaxBodySize: The maximum number of bytes that a request body can contain in the new group
/// `nil` means there is no limit.
/// - configure: Closure to configure the newly created `Router`.
/// - builder: The new builder with the new max body size.
/// - returns: A newly created `Router` with a new max body size.
public func grouped(maxSize defaultMaxBodySize: Int?, configure: (_ builder: RoutesBuilder) throws -> ()) rethrows {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should be group. The ...ed ones return a route builder.

try configure(HTTPRoutesGroup(root: self, defaultMaxBodySize: defaultMaxBodySize))
}
}

Expand All @@ -40,11 +69,15 @@ private final class HTTPRoutesGroup: RoutesBuilder {

/// Additional components.
let path: [PathComponent]


/// The default max body size for requests in the group.
let defaultMaxBodySize: Int?

/// Creates a new `PathGroup`.
init(root: RoutesBuilder, path: [PathComponent]) {
init(root: RoutesBuilder, path: [PathComponent] = [], defaultMaxBodySize: Int? = nil) {
self.root = root
self.path = path
self.defaultMaxBodySize = defaultMaxBodySize
}

/// See `HTTPRoutesBuilder`.
Expand Down
30 changes: 21 additions & 9 deletions Sources/Vapor/Routing/RoutesBuilder+Method.swift
Expand Up @@ -2,11 +2,11 @@
public enum HTTPBodyStreamStrategy {
/// The HTTP request's body will be collected into memory before the route handler is
/// called. The max size will determine how much data can be collected. The default is
/// `1 << 14`.
/// 1,000,000 bytes, or 1 megabyte.
///
/// See `collect(maxSize:)` to set a lower max body size.
public static var collect: HTTPBodyStreamStrategy {
return .collect(maxSize: 1 << 14)
return .collect(maxSize: 1_000_000)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should go back to being nil so that it relies on the defaultMaxBodySize.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Isn't nil supposed to mean 'just keep accepting data until we reach the end of the body'?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was thinking nil here would mean "defer to global default". If that is also nil, then you could get unlimited body size. Maybe it's worth making this more explicit with an enum:

enum MaxSize {
    // Defer to application config
    case default
    // Provide a custom value
    // - warning: Passing `nil` here means no restrictions on how much memory an incoming request can use.
    case override(ByteCount?)
}

}

/// The HTTP request's body will not be collected first before the route handler is called
Expand Down Expand Up @@ -126,7 +126,7 @@ extension RoutesBuilder {
public func on<Response>(
_ method: HTTPMethod,
_ path: PathComponent...,
body: HTTPBodyStreamStrategy = .collect,
body: HTTPBodyStreamStrategy? = nil,
use closure: @escaping (Request) throws -> Response
) -> Route
where Response: ResponseEncodable
Expand All @@ -140,21 +140,33 @@ extension RoutesBuilder {
public func on<Response>(
_ method: HTTPMethod,
_ path: [PathComponent],
body: HTTPBodyStreamStrategy = .collect,
body: HTTPBodyStreamStrategy? = nil,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should keep this as .collect

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we make this .collect, how do we use the default max body size set for the routes builder?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Adding to #2312 (comment), my previous comments were intending that .collect(maxSize: nil) would use the default max body size. I think an enum could also make sense.

I don't think optionality makes sense for HTTPBodyStreamStrategy as a whole though. Max body size only affects the collect case, stream is never capped. So we should express the optionality within the collect case. Either with an optional or an enum.

use closure: @escaping (Request) throws -> Response
) -> Route
where Response: ResponseEncodable
{
let streamStrategy = body ?? .collect(maxSize: self.defaultMaxBodySize)
let responder = BasicResponder { request in
if case .collect(let max) = body, request.body.data == nil {
return request.body.collect(max: max).flatMapThrowing { _ in
switch streamStrategy {
case let .collect(maxSize):
let collected: EventLoopFuture<Void>

if let data = request.body.data {
collected = request.eventLoop.tryFuture {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

indentation

if let max = maxSize, data.readableBytes > max { throw Abort(.payloadTooLarge) }
}
} else {
collected = request.body.collect(max: maxSize).transform(to: ())
}

return collected.flatMapThrowing {
return try closure(request)
}.encodeResponse(for: request)
} else {
return try closure(request)
.encodeResponse(for: request)
case .stream:
return try closure(request).encodeResponse(for: request)
}
}

let route = Route(
method: method,
path: path,
Expand Down
6 changes: 6 additions & 0 deletions Sources/Vapor/Routing/RoutesBuilder.swift
@@ -1,7 +1,13 @@
public protocol RoutesBuilder {
var defaultMaxBodySize: Int? { get }
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not super happy that I have to add this (at least we can have a default value). If there are any ideas on how it can be removed, I'll take them.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed. Let me take a stab at this and see if I can get it to work without this requirement.


func add(_ route: Route)
}

extension RoutesBuilder {
public var defaultMaxBodySize: Int? { 1_000_000 }
}

extension UUID: LosslessStringConvertible {
public init?(_ description: String) {
self.init(uuidString: description)
Expand Down
55 changes: 55 additions & 0 deletions Tests/VaporTests/RouteTests.swift
Expand Up @@ -268,4 +268,59 @@ final class RouteTests: XCTestCase {
XCTAssertEqual(res.body.string, "bar")
}
}

func testConfigurableMaxBodySize() throws {
let app = Application(.testing)
defer { app.shutdown() }

XCTAssertEqual(app.routes.defaultMaxBodySize, 1_000_000)
app.routes.defaultMaxBodySize = 50
XCTAssertEqual(app.routes.defaultMaxBodySize, 50)

let group = app.routes.group(maxSize: 1_000_000_000).grouped("uploads")
XCTAssertEqual(group.defaultMaxBodySize, 1_000_000_000)

group.on(.POST, "small", body: .collect(maxSize: 1_000) , use: { request in return "small" })
group.on(.POST, "medium", body: .collect(maxSize: 1_000_000) , use: { request in return "medium" })
group.post("large", use: { request in return "large" })


var smallBuffer = ByteBufferAllocator().buffer(capacity: 1_001)
smallBuffer.writeBytes(Array(repeating: 48, count: 1_000))
try app.test(.POST, "/uploads/small", body: smallBuffer, afterResponse: { response in
XCTAssertEqual(response.status, .ok)
})

smallBuffer.clear()
smallBuffer.writeBytes(Array(repeating: 48, count: 1_001))
try app.test(.POST, "/uploads/small", body: smallBuffer, afterResponse: { response in
XCTAssertEqual(response.status, .payloadTooLarge)
})


var mediumBuffer = ByteBufferAllocator().buffer(capacity: 1_000_001)
mediumBuffer.writeBytes(Array(repeating: 48, count: 1_000_000))
try app.test(.POST, "/uploads/medium", body: mediumBuffer, afterResponse: { response in
XCTAssertEqual(response.status, .ok)
})

mediumBuffer.clear()
mediumBuffer.writeBytes(Array(repeating: 48, count: 1_000_001))
try app.test(.POST, "/uploads/medium", body: mediumBuffer, afterResponse: { response in
XCTAssertEqual(response.status, .payloadTooLarge)
})


var largeBuffer = ByteBufferAllocator().buffer(capacity: 1_000_000_001)
largeBuffer.writeBytes(Array(repeating: 48, count: 1_000_000_000))
try app.test(.POST, "/uploads/large", body: largeBuffer, afterResponse: { response in
XCTAssertEqual(response.status, .ok)
})

largeBuffer.clear()
largeBuffer.writeBytes(Array(repeating: 48, count: 1_000_000_001))
try app.test(.POST, "/uploads/large", body: largeBuffer, afterResponse: { response in
XCTAssertEqual(response.status, .payloadTooLarge)
})
}
}