From d2c226a3c054306753ca787d5e49adfdfa7ec52a Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Tue, 29 Jan 2019 12:18:35 -0800 Subject: [PATCH] Set prefix, delimiter params even when empty We have never set values which are empty on the request because they are perhaps not useful in the List query, but this assumption is wrong when there are restricted policies for a given user, because empty is actually a valid value in IAM or Bucket policy conditions. For example following condition would never work with our ListObjects call and AWS cli would work fine. "Condition": { "StringEquals": { "s3:prefix": [ "", "data/", "data" ], "s3:delimiter": [ "/", "" ] } } The reason is empty or not prefix and delimiter should be added to the query param in List operation, such that server can use the value to validate the policies for the incoming request. Refer minio/minio-go#1064 --- Minio/ApiEndpoints/BucketOperations.cs | 46 ++++++++++++----------- Minio/ApiEndpoints/IBucketOperations.cs | 14 +++---- Minio/ApiEndpoints/IObjectOperations.cs | 10 ++--- Minio/ApiEndpoints/ObjectOperations.cs | 50 ++++--------------------- 4 files changed, 43 insertions(+), 77 deletions(-) diff --git a/Minio/ApiEndpoints/BucketOperations.cs b/Minio/ApiEndpoints/BucketOperations.cs index 12f6e6ff5..cdbdf3232 100644 --- a/Minio/ApiEndpoints/BucketOperations.cs +++ b/Minio/ApiEndpoints/BucketOperations.cs @@ -72,7 +72,7 @@ public async Task MakeBucketAsync(string bucketName, string location = "us-east- location = this.Region; } } - + // Set Target URL Uri requestUrl = RequestUtil.MakeTargetURL(this.BaseUrl, this.Secure,location); SetTargetURL(requestUrl); @@ -134,20 +134,27 @@ public async Task RemoveBucketAsync(string bucketName, CancellationToken cancell /// List all objects non-recursively in a bucket with a given prefix, optionally emulating a directory /// /// Bucket to list objects from - /// Filters all objects not beginning with a given prefix - /// Set to false to emulate a directory + /// Filters all objects beginning with a given prefix + /// Set to true to recursively list all objects /// Optional cancellation token to cancel the operation /// An observable of items that client can subscribe to - public IObservable ListObjectsAsync(string bucketName, string prefix = null, bool recursive = true, CancellationToken cancellationToken = default(CancellationToken)) + public IObservable ListObjectsAsync(string bucketName, string prefix = "", bool recursive = false, CancellationToken cancellationToken = default(CancellationToken)) { return Observable.Create( async obs => { bool isRunning = true; string marker = null; + + var delimiter = "/"; + if (recursive) + { + delimiter = ""; + } + while (isRunning) { - Tuple> result = await GetObjectListAsync(bucketName, prefix, recursive, marker, cancellationToken).ConfigureAwait(false); + Tuple> result = await GetObjectListAsync(bucketName, prefix, delimiter, marker, cancellationToken).ConfigureAwait(false); Item lastItem = null; foreach (Item item in result.Item2) { @@ -171,35 +178,30 @@ public IObservable ListObjectsAsync(string bucketName, string prefix = nul /// Gets the list of objects in the bucket filtered by prefix /// /// Bucket to list objects from - /// Filters all objects not beginning with a given prefix - /// Set to false to emulate a directory + /// Filters all objects starting with a given prefix + /// Delimit the output upto this character /// marks location in the iterator sequence /// Task with a tuple populated with objects /// Optional cancellation token to cancel the operation - private async Task>> GetObjectListAsync(string bucketName, string prefix, bool recursive, string marker, CancellationToken cancellationToken = default(CancellationToken)) + private async Task>> GetObjectListAsync(string bucketName, string prefix, string delimiter, string marker, CancellationToken cancellationToken = default(CancellationToken)) { var queries = new List(); - if (!recursive) - { - queries.Add("delimiter=%2F"); - } - if (prefix != null) - { - queries.Add("prefix=" + Uri.EscapeDataString(prefix)); - } + queries.Add("delimiter="+ Uri.EscapeDataString(delimiter)); + queries.Add("prefix=" + Uri.EscapeDataString(prefix)); + queries.Add("max-keys=1000"); + if (marker != null) { queries.Add("marker=" + Uri.EscapeDataString(marker)); } - queries.Add("max-keys=1000"); string query = string.Join("&", queries); var request = await this.CreateRequest(Method.GET, bucketName, resourcePath: "?" + query) .ConfigureAwait(false); - + var response = await this.ExecuteTaskAsync(this.NoErrorHandlers, request, cancellationToken).ConfigureAwait(false); var contentBytes = System.Text.Encoding.UTF8.GetBytes(response.Content); @@ -292,7 +294,7 @@ public async Task GetBucketNotificationsAsync(string bucketN resourcePath: "?notification") .ConfigureAwait(false); BucketNotification notification = null; - + var response = await this.ExecuteTaskAsync(this.NoErrorHandlers, request, cancellationToken).ConfigureAwait(false); var contentBytes = System.Text.Encoding.UTF8.GetBytes(response.Content); using (var stream = new MemoryStream(contentBytes)) @@ -314,11 +316,11 @@ public async Task SetBucketNotificationsAsync(string bucketName, BucketNotificat var request = await this.CreateRequest(Method.PUT, bucketName, resourcePath: "?notification") .ConfigureAwait(false); - + request.XmlSerializer = new RestSharp.Serializers.DotNetXmlSerializer(); request.RequestFormat = DataFormat.Xml; request.AddBody(notification); - + IRestResponse response = await this.ExecuteTaskAsync(this.NoErrorHandlers, request, cancellationToken).ConfigureAwait(false); } @@ -335,4 +337,4 @@ public Task RemoveAllBucketNotificationsAsync(string bucketName, CancellationTok return SetBucketNotificationsAsync(bucketName, notification, cancellationToken); } } -} \ No newline at end of file +} diff --git a/Minio/ApiEndpoints/IBucketOperations.cs b/Minio/ApiEndpoints/IBucketOperations.cs index 33c8cfacc..a50a5d942 100644 --- a/Minio/ApiEndpoints/IBucketOperations.cs +++ b/Minio/ApiEndpoints/IBucketOperations.cs @@ -58,11 +58,11 @@ public interface IBucketOperations /// List all objects non-recursively in a bucket with a given prefix, optionally emulating a directory /// /// Bucket to list objects from - /// Filters all objects not beginning with a given prefix - /// Set to false to emulate a directory + /// Filter all incomplete uploads starting with this prefix + /// List incomplete uploads recursively /// /// An observable of items that client can subscribe to - IObservable ListObjectsAsync(string bucketName, string prefix = null, bool recursive = true, CancellationToken cancellationToken = default(CancellationToken)); + IObservable ListObjectsAsync(string bucketName, string prefix = "", bool recursive = false, CancellationToken cancellationToken = default(CancellationToken)); /// /// Get bucket policy @@ -80,7 +80,7 @@ public interface IBucketOperations /// Optional cancellation token to cancel the operation /// Returns Task that sets the current bucket policy Task SetPolicyAsync(String bucketName, String policyJson, CancellationToken cancellationToken = default(CancellationToken)); - + /// /// Gets the notification configuration set for this bucket /// @@ -105,8 +105,8 @@ public interface IBucketOperations /// optional cancellation token /// Task RemoveAllBucketNotificationsAsync(string bucketName, CancellationToken cancellationToken = default(CancellationToken)); - + // Task ListenBucketNotificationsAsync(string bucketName, string prefix = "", string suffix = "", List events,CancellationToken cancellationToken = default(CancellationToken)); - + } -} \ No newline at end of file +} diff --git a/Minio/ApiEndpoints/IObjectOperations.cs b/Minio/ApiEndpoints/IObjectOperations.cs index 8f98be1da..423af234f 100644 --- a/Minio/ApiEndpoints/IObjectOperations.cs +++ b/Minio/ApiEndpoints/IObjectOperations.cs @@ -94,10 +94,10 @@ public interface IObjectOperations /// /// Bucket to list all incomplepte uploads from /// prefix to list all incomplete uploads - /// option to list incomplete uploads recursively + /// Set to true to recursively list all incomplete uploads /// Optional cancellation token to cancel the operation /// A lazily populated list of incomplete uploads - IObservable ListIncompleteUploads(string bucketName, string prefix, bool recursive, CancellationToken cancellationToken = default(CancellationToken)); + IObservable ListIncompleteUploads(string bucketName, string prefix = "", bool recursive = false, CancellationToken cancellationToken = default(CancellationToken)); /// /// Remove incomplete uploads from a given bucket and objectName @@ -151,7 +151,7 @@ public interface IObjectOperations Task GetObjectAsync(string bucketName, string objectName, string filePath, ServerSideEncryption sse = null ,CancellationToken cancellationToken = default(CancellationToken)); /// - /// Presigned get url - returns a presigned url to access an object's data without credentials.URL can have a maximum expiry of + /// Presigned get url - returns a presigned url to access an object's data without credentials.URL can have a maximum expiry of /// upto 7 days or a minimum of 1 second.Additionally, you can override a set of response headers using reqParams. /// /// Bucket to retrieve object from @@ -161,7 +161,7 @@ public interface IObjectOperations Task PresignedGetObjectAsync(string bucketName, string objectName, int expiresInt, Dictionary reqParams = null); /// - /// Presigned Put url - returns a presigned url to upload an object without credentials.URL can have a maximum expiry of + /// Presigned Put url - returns a presigned url to upload an object without credentials.URL can have a maximum expiry of /// upto 7 days or a minimum of 1 second. /// /// Bucket to retrieve object from @@ -176,4 +176,4 @@ public interface IObjectOperations Task>> PresignedPostPolicyAsync(PostPolicy policy); } -} \ No newline at end of file +} diff --git a/Minio/ApiEndpoints/ObjectOperations.cs b/Minio/ApiEndpoints/ObjectOperations.cs index 624b3b540..c84901bd9 100644 --- a/Minio/ApiEndpoints/ObjectOperations.cs +++ b/Minio/ApiEndpoints/ObjectOperations.cs @@ -523,10 +523,10 @@ private async Task PutObjectAsync(string bucketName, string objectName, { var queries = new List(); queries.Add("uploads"); - if (prefix != null) - { - queries.Add("prefix=" + Uri.EscapeDataString(prefix)); - } + queries.Add("prefix=" + Uri.EscapeDataString(prefix)); + queries.Add("delimiter=" + Uri.EscapeDataString(delimiter)); + queries.Add("max-uploads=1000"); + if (keyMarker != null) { queries.Add("key-marker=" + Uri.EscapeDataString(keyMarker)); @@ -535,12 +535,6 @@ private async Task PutObjectAsync(string bucketName, string objectName, { queries.Add("upload-id-marker=" + uploadIdMarker); } - if (delimiter != null) - { - queries.Add("delimiter=" + delimiter); - } - - queries.Add("max-uploads=1000"); string query = string.Join("&", queries); @@ -571,15 +565,15 @@ private async Task PutObjectAsync(string bucketName, string objectName, /// Lists all incomplete uploads in a given bucket and prefix recursively /// /// Bucket to list all incomplepte uploads from - /// prefix to list all incomplete uploads - /// option to list incomplete uploads recursively + /// Filter all incomplete uploads starting with this prefix + /// Set to true to recursively list all incomplete uploads /// Optional cancellation token to cancel the operation /// A lazily populated list of incomplete uploads public IObservable ListIncompleteUploads(string bucketName, string prefix = "", bool recursive = true, CancellationToken cancellationToken = default(CancellationToken)) { if (recursive) { - return this.listIncompleteUploads(bucketName, prefix, null, cancellationToken); + return this.listIncompleteUploads(bucketName, prefix, "", cancellationToken); } return this.listIncompleteUploads(bucketName, prefix, "/", cancellationToken); } @@ -617,36 +611,6 @@ private IObservable listIncompleteUploads(string bucketName, string pref } - /// - /// Find uploadId of most recent unsuccessful attempt to upload object to bucket. - /// - /// - /// - /// Optional cancellation token to cancel the operation - /// - private async Task getLatestIncompleteUploadIdAsync(string bucketName, string objectName, CancellationToken cancellationToken) - { - Upload latestUpload = null; - var uploads = await this.ListIncompleteUploads(bucketName, objectName, cancellationToken: cancellationToken).ToArray(); - - foreach (Upload upload in uploads) - { - if (objectName == upload.Key && (latestUpload == null || latestUpload.Initiated.CompareTo(upload.Initiated) < 0)) - { - latestUpload = upload; - - } - } - if (latestUpload != null) - { - return latestUpload.UploadId; - } - else - { - return null; - } - - } /// /// Remove incomplete uploads from a given bucket and objectName ///