From a118b2c45716997a9ffffefe73a0ddc866034d89 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Tue, 29 Jan 2019 12:18:35 -0800 Subject: [PATCH] Set prefix, delimiter params even when empty We have never set values which are empty on the request because they are perhaps not useful in the List query, but this assumption is wrong when there are restricted policies for a given user, because empty is actually a valid value in IAM or Bucket policy conditions. For example following condition would never work with our ListObjects call and AWS cli would work fine. "Condition": { "StringEquals": { "s3:prefix": [ "", "data/", "data" ], "s3:delimiter": [ "/", "" ] } } The reason is empty or not prefix and delimiter should be added to the query param in List operation, such that server can use the value to validate the policies for the incoming request. Refer minio/minio-go#1064 --- Minio.Functional.Tests/FunctionalTest.cs | 66 ++++++++++++------------ Minio/ApiEndpoints/BucketOperations.cs | 46 +++++++++-------- Minio/ApiEndpoints/IBucketOperations.cs | 14 ++--- Minio/ApiEndpoints/IObjectOperations.cs | 10 ++-- Minio/ApiEndpoints/ObjectOperations.cs | 50 +++--------------- 5 files changed, 76 insertions(+), 110 deletions(-) diff --git a/Minio.Functional.Tests/FunctionalTest.cs b/Minio.Functional.Tests/FunctionalTest.cs index a07f405eb..66ff1c9b4 100644 --- a/Minio.Functional.Tests/FunctionalTest.cs +++ b/Minio.Functional.Tests/FunctionalTest.cs @@ -76,7 +76,7 @@ private static String CreateFile(int size, string dataFileName = null) File.WriteAllBytes(fileName, data); return GetFilePath(fileName); } - + return GetFilePath(dataFileName); } @@ -94,8 +94,8 @@ public static String GetRandomName(int length = 5) } return "miniodotnet" + result.ToString(); } - // Return true if running in Mint mode - public static bool IsMintEnv() + // Return true if running in Mint mode + public static bool IsMintEnv() { return !String.IsNullOrEmpty(Environment.GetEnvironmentVariable("MINT_DATA_DIR")); } @@ -146,13 +146,13 @@ public static void Main(string[] args) else minioClient = new MinioClient(endPoint, accessKey, secretKey); - // Assign parameters before starting the test + // Assign parameters before starting the test string bucketName = GetRandomName(); string objectName = GetRandomName(); string destBucketName = GetRandomName(); string destObjectName = GetRandomName(); - // Set app Info + // Set app Info minioClient.SetAppInfo("app-name", "app-version"); // Set HTTP Tracing On // minioClient.SetTraceOn(new JsonNetLogger()); @@ -277,7 +277,7 @@ private static void runCoreTests(MinioClient minioClient) ListObjects_Test1(minioClient).Wait(); RemoveObject_Test1(minioClient).Wait(); CopyObject_Test1(minioClient).Wait(); - + // Test SetPolicyAsync function SetBucketPolicy_Test1(minioClient).Wait(); @@ -313,7 +313,7 @@ private async static Task BucketExists_Test(MinioClient minio) catch (MinioException ex) { new MintLogger("BucketExists_Test",bucketExistsSignature,"Tests whether BucketExists passes",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(),args).Log(); - } + } } private async static Task MakeBucket_Test1(MinioClient minio) @@ -337,7 +337,7 @@ private async static Task MakeBucket_Test1(MinioClient minio) catch (MinioException ex) { new MintLogger("MakeBucket_Test1",makeBucketSignature,"Tests whether MakeBucket passes",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message,ex.ToString(),args).Log(); - } + } } private async static Task MakeBucket_Test2(MinioClient minio) @@ -363,7 +363,7 @@ private async static Task MakeBucket_Test2(MinioClient minio) { new MintLogger("MakeBucket_Test2",makeBucketSignature,testType,TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(),args).Log(); } - + } private async static Task MakeBucket_Test3(MinioClient minio, bool aws = false) { @@ -504,7 +504,7 @@ private async static Task PutObject_Test1(MinioClient minio) } } - + private async static Task PutObject_Test2(MinioClient minio) { DateTime startTime = DateTime.Now; @@ -671,7 +671,7 @@ private async static Task PutObject_Test7(MinioClient minio) await Setup_Test(minio, bucketName); using (System.IO.MemoryStream filestream = rsg.GenerateStreamFromSeed(1 * MB)) { - + long size = -1; long file_write_size = filestream.Length; @@ -744,7 +744,7 @@ private async static Task PutGetStatEncryptedObject_Test1(MinioClient minio) }; try { - // Putobject with SSE-C encryption. + // Putobject with SSE-C encryption. await Setup_Test(minio, bucketName); Aes aesEncryption = Aes.Create(); aesEncryption.KeySize = 256; @@ -1268,7 +1268,7 @@ private async static Task CopyObject_Test5(MinioClient minio) new MintLogger("CopyObject_Test5",copyObjectSignature,"Tests whether CopyObject multi-part copy upload for large files works",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(),args).Log(); } } - + } private async static Task CopyObject_Test6(MinioClient minio) @@ -1702,7 +1702,7 @@ private async static Task GetObject_Test1(MinioClient minio) { new MintLogger("GetObject_Test1",getObjectSignature1,"Tests whether GetObject as stream works",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(),args).Log(); } - + } private async static Task GetObject_Test2(MinioClient minio) { @@ -1736,7 +1736,7 @@ private async static Task GetObject_Test2(MinioClient minio) { new MintLogger("GetObject_Test2",getObjectSignature1,"Tests for non-existent GetObject",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(),args).Log(); } - + } private async static Task GetObject_Test3(MinioClient minio) { @@ -1822,7 +1822,7 @@ private async static Task FGetObject_Test1(MinioClient minio) { new MintLogger("FGetObject_Test1",getObjectSignature3,"Tests whether FGetObject passes for small upload",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(),args).Log(); } - + } private async static Task FPutObject_Test1(MinioClient minio) @@ -1913,8 +1913,8 @@ private async static Task ListObjects_Test1(MinioClient minio) tasks[i]= PutObject_Task(minio, bucketName, objectName + i.ToString(), null, null, 0, null, rsg.GenerateStreamFromSeed(1*MB)); } await Task.WhenAll(tasks); - - ListObjects_Test(minio, bucketName, prefix, 2,false).Wait(); + + ListObjects_Test(minio, bucketName, prefix, 2, false).Wait(); System.Threading.Thread.Sleep(5000); await minio.RemoveObjectAsync(bucketName, objectName + "0"); await minio.RemoveObjectAsync(bucketName, objectName + "1"); @@ -1939,7 +1939,7 @@ private async static Task ListObjects_Test2(MinioClient minio) { await Setup_Test(minio, bucketName); - ListObjects_Test(minio, bucketName, null, 0).Wait(5000); + ListObjects_Test(minio, bucketName, "", 0).Wait(5000); await TearDown(minio, bucketName); new MintLogger("ListObjects_Test2",listObjectsSignature,"Tests whether ListObjects passes when bucket is empty",TestStatus.PASS,(DateTime.Now - startTime),args:args).Log(); } @@ -1971,7 +1971,7 @@ private async static Task ListObjects_Test3(MinioClient minio) } await Task.WhenAll(tasks); - ListObjects_Test(minio, bucketName, prefix, 2,true).Wait(); + ListObjects_Test(minio, bucketName, prefix, 2, true).Wait(); System.Threading.Thread.Sleep(5000); await minio.RemoveObjectAsync(bucketName, objectName + "0"); await minio.RemoveObjectAsync(bucketName, objectName + "1"); @@ -2003,8 +2003,8 @@ private async static Task ListObjects_Test4(MinioClient minio) tasks[i]= PutObject_Task(minio, bucketName, objectName + i.ToString(), null, null, 0, null, rsg.GenerateStreamFromSeed(1*MB)); } await Task.WhenAll(tasks); - - ListObjects_Test(minio, bucketName, null, 2,false).Wait(); + + ListObjects_Test(minio, bucketName, "", 2, false).Wait(); System.Threading.Thread.Sleep(5000); await minio.RemoveObjectAsync(bucketName, objectName + "0"); await minio.RemoveObjectAsync(bucketName, objectName + "1"); @@ -2040,8 +2040,8 @@ private async static Task ListObjects_Test5(MinioClient minio) } } await Task.WhenAll(tasks); - - ListObjects_Test(minio, bucketName, objectNamePrefix, numObjects,false).Wait(); + + ListObjects_Test(minio, bucketName, objectNamePrefix, numObjects, false).Wait(); System.Threading.Thread.Sleep(5000); for(int index=1; index <= numObjects; index++) { @@ -2364,7 +2364,7 @@ private async static Task PresignedPutObject_Test2(MinioClient minio) new MintLogger("PresignedPutObject_Test2",presignedPutObjectSignature,"Tests whether PresignedPutObject url retrieves object from bucket when invalid expiry is set.",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(),args).Log(); } } - + private static async Task UploadObjectAsync(string url, string filePath) { HttpWebRequest httpRequest = WebRequest.Create(url) as HttpWebRequest; @@ -2406,7 +2406,7 @@ private async static Task PresignedPostPolicy_Test1(MinioClient minio) await Setup_Test(minio, bucketName); await minio.PutObjectAsync(bucketName, objectName, - fileName); + fileName); var pairs = new List>(); string url = "https://s3.amazonaws.com/" + bucketName; Tuple> policyTuple = await minio.PresignedPostPolicyAsync(form); @@ -2477,7 +2477,7 @@ private async static Task ListIncompleteUpload_Test1(MinioClient minio) IDisposable subscription = observable.Subscribe( item => Assert.AreEqual(item.Key, objectName), - ex => Assert.Fail()); + ex => Assert.Fail()); await minio.RemoveIncompleteUploadAsync(bucketName, objectName); } @@ -2498,7 +2498,7 @@ private async static Task ListIncompleteUpload_Test2(MinioClient minio) { DateTime startTime = DateTime.Now; string bucketName = GetRandomName(15); - string prefix = "minioprefix/"; + string prefix = "minioprefix/"; string objectName = prefix + GetRandomName(10); string contentType = "gzip"; Dictionary args = new Dictionary @@ -2527,11 +2527,11 @@ private async static Task ListIncompleteUpload_Test2(MinioClient minio) } catch (OperationCanceledException) { - IObservable observable = minio.ListIncompleteUploads(bucketName,"minioprefix",false); + IObservable observable = minio.ListIncompleteUploads(bucketName, "minioprefix", false); IDisposable subscription = observable.Subscribe( item => Assert.AreEqual(item.Key, objectName), - ex => Assert.Fail()); + ex => Assert.Fail()); await minio.RemoveIncompleteUploadAsync(bucketName, objectName); } @@ -2577,11 +2577,11 @@ private async static Task ListIncompleteUpload_Test3(MinioClient minio) } catch (OperationCanceledException) { - IObservable observable = minio.ListIncompleteUploads(bucketName,prefix,true); + IObservable observable = minio.ListIncompleteUploads(bucketName, prefix, true); IDisposable subscription = observable.Subscribe( item => Assert.AreEqual(item.Key, objectName), - ex => Assert.Fail()); + ex => Assert.Fail()); await minio.RemoveIncompleteUploadAsync(bucketName, objectName); } @@ -2631,7 +2631,7 @@ private async static Task RemoveIncompleteUpload_Test(MinioClient minio) IDisposable subscription = observable.Subscribe( item => Assert.Fail(), - ex => Assert.Fail()); + ex => Assert.Fail()); } await TearDown(minio, bucketName); new MintLogger("RemoveIncompleteUpload_Test",removeIncompleteUploadSignature,"Tests whether RemoveIncompleteUpload passes.",TestStatus.PASS,(DateTime.Now - startTime), args:args).Log(); diff --git a/Minio/ApiEndpoints/BucketOperations.cs b/Minio/ApiEndpoints/BucketOperations.cs index 12f6e6ff5..cdbdf3232 100644 --- a/Minio/ApiEndpoints/BucketOperations.cs +++ b/Minio/ApiEndpoints/BucketOperations.cs @@ -72,7 +72,7 @@ public async Task MakeBucketAsync(string bucketName, string location = "us-east- location = this.Region; } } - + // Set Target URL Uri requestUrl = RequestUtil.MakeTargetURL(this.BaseUrl, this.Secure,location); SetTargetURL(requestUrl); @@ -134,20 +134,27 @@ public async Task RemoveBucketAsync(string bucketName, CancellationToken cancell /// List all objects non-recursively in a bucket with a given prefix, optionally emulating a directory /// /// Bucket to list objects from - /// Filters all objects not beginning with a given prefix - /// Set to false to emulate a directory + /// Filters all objects beginning with a given prefix + /// Set to true to recursively list all objects /// Optional cancellation token to cancel the operation /// An observable of items that client can subscribe to - public IObservable ListObjectsAsync(string bucketName, string prefix = null, bool recursive = true, CancellationToken cancellationToken = default(CancellationToken)) + public IObservable ListObjectsAsync(string bucketName, string prefix = "", bool recursive = false, CancellationToken cancellationToken = default(CancellationToken)) { return Observable.Create( async obs => { bool isRunning = true; string marker = null; + + var delimiter = "/"; + if (recursive) + { + delimiter = ""; + } + while (isRunning) { - Tuple> result = await GetObjectListAsync(bucketName, prefix, recursive, marker, cancellationToken).ConfigureAwait(false); + Tuple> result = await GetObjectListAsync(bucketName, prefix, delimiter, marker, cancellationToken).ConfigureAwait(false); Item lastItem = null; foreach (Item item in result.Item2) { @@ -171,35 +178,30 @@ public IObservable ListObjectsAsync(string bucketName, string prefix = nul /// Gets the list of objects in the bucket filtered by prefix /// /// Bucket to list objects from - /// Filters all objects not beginning with a given prefix - /// Set to false to emulate a directory + /// Filters all objects starting with a given prefix + /// Delimit the output upto this character /// marks location in the iterator sequence /// Task with a tuple populated with objects /// Optional cancellation token to cancel the operation - private async Task>> GetObjectListAsync(string bucketName, string prefix, bool recursive, string marker, CancellationToken cancellationToken = default(CancellationToken)) + private async Task>> GetObjectListAsync(string bucketName, string prefix, string delimiter, string marker, CancellationToken cancellationToken = default(CancellationToken)) { var queries = new List(); - if (!recursive) - { - queries.Add("delimiter=%2F"); - } - if (prefix != null) - { - queries.Add("prefix=" + Uri.EscapeDataString(prefix)); - } + queries.Add("delimiter="+ Uri.EscapeDataString(delimiter)); + queries.Add("prefix=" + Uri.EscapeDataString(prefix)); + queries.Add("max-keys=1000"); + if (marker != null) { queries.Add("marker=" + Uri.EscapeDataString(marker)); } - queries.Add("max-keys=1000"); string query = string.Join("&", queries); var request = await this.CreateRequest(Method.GET, bucketName, resourcePath: "?" + query) .ConfigureAwait(false); - + var response = await this.ExecuteTaskAsync(this.NoErrorHandlers, request, cancellationToken).ConfigureAwait(false); var contentBytes = System.Text.Encoding.UTF8.GetBytes(response.Content); @@ -292,7 +294,7 @@ public async Task GetBucketNotificationsAsync(string bucketN resourcePath: "?notification") .ConfigureAwait(false); BucketNotification notification = null; - + var response = await this.ExecuteTaskAsync(this.NoErrorHandlers, request, cancellationToken).ConfigureAwait(false); var contentBytes = System.Text.Encoding.UTF8.GetBytes(response.Content); using (var stream = new MemoryStream(contentBytes)) @@ -314,11 +316,11 @@ public async Task SetBucketNotificationsAsync(string bucketName, BucketNotificat var request = await this.CreateRequest(Method.PUT, bucketName, resourcePath: "?notification") .ConfigureAwait(false); - + request.XmlSerializer = new RestSharp.Serializers.DotNetXmlSerializer(); request.RequestFormat = DataFormat.Xml; request.AddBody(notification); - + IRestResponse response = await this.ExecuteTaskAsync(this.NoErrorHandlers, request, cancellationToken).ConfigureAwait(false); } @@ -335,4 +337,4 @@ public Task RemoveAllBucketNotificationsAsync(string bucketName, CancellationTok return SetBucketNotificationsAsync(bucketName, notification, cancellationToken); } } -} \ No newline at end of file +} diff --git a/Minio/ApiEndpoints/IBucketOperations.cs b/Minio/ApiEndpoints/IBucketOperations.cs index 33c8cfacc..a50a5d942 100644 --- a/Minio/ApiEndpoints/IBucketOperations.cs +++ b/Minio/ApiEndpoints/IBucketOperations.cs @@ -58,11 +58,11 @@ public interface IBucketOperations /// List all objects non-recursively in a bucket with a given prefix, optionally emulating a directory /// /// Bucket to list objects from - /// Filters all objects not beginning with a given prefix - /// Set to false to emulate a directory + /// Filter all incomplete uploads starting with this prefix + /// List incomplete uploads recursively /// /// An observable of items that client can subscribe to - IObservable ListObjectsAsync(string bucketName, string prefix = null, bool recursive = true, CancellationToken cancellationToken = default(CancellationToken)); + IObservable ListObjectsAsync(string bucketName, string prefix = "", bool recursive = false, CancellationToken cancellationToken = default(CancellationToken)); /// /// Get bucket policy @@ -80,7 +80,7 @@ public interface IBucketOperations /// Optional cancellation token to cancel the operation /// Returns Task that sets the current bucket policy Task SetPolicyAsync(String bucketName, String policyJson, CancellationToken cancellationToken = default(CancellationToken)); - + /// /// Gets the notification configuration set for this bucket /// @@ -105,8 +105,8 @@ public interface IBucketOperations /// optional cancellation token /// Task RemoveAllBucketNotificationsAsync(string bucketName, CancellationToken cancellationToken = default(CancellationToken)); - + // Task ListenBucketNotificationsAsync(string bucketName, string prefix = "", string suffix = "", List events,CancellationToken cancellationToken = default(CancellationToken)); - + } -} \ No newline at end of file +} diff --git a/Minio/ApiEndpoints/IObjectOperations.cs b/Minio/ApiEndpoints/IObjectOperations.cs index 8f98be1da..423af234f 100644 --- a/Minio/ApiEndpoints/IObjectOperations.cs +++ b/Minio/ApiEndpoints/IObjectOperations.cs @@ -94,10 +94,10 @@ public interface IObjectOperations /// /// Bucket to list all incomplepte uploads from /// prefix to list all incomplete uploads - /// option to list incomplete uploads recursively + /// Set to true to recursively list all incomplete uploads /// Optional cancellation token to cancel the operation /// A lazily populated list of incomplete uploads - IObservable ListIncompleteUploads(string bucketName, string prefix, bool recursive, CancellationToken cancellationToken = default(CancellationToken)); + IObservable ListIncompleteUploads(string bucketName, string prefix = "", bool recursive = false, CancellationToken cancellationToken = default(CancellationToken)); /// /// Remove incomplete uploads from a given bucket and objectName @@ -151,7 +151,7 @@ public interface IObjectOperations Task GetObjectAsync(string bucketName, string objectName, string filePath, ServerSideEncryption sse = null ,CancellationToken cancellationToken = default(CancellationToken)); /// - /// Presigned get url - returns a presigned url to access an object's data without credentials.URL can have a maximum expiry of + /// Presigned get url - returns a presigned url to access an object's data without credentials.URL can have a maximum expiry of /// upto 7 days or a minimum of 1 second.Additionally, you can override a set of response headers using reqParams. /// /// Bucket to retrieve object from @@ -161,7 +161,7 @@ public interface IObjectOperations Task PresignedGetObjectAsync(string bucketName, string objectName, int expiresInt, Dictionary reqParams = null); /// - /// Presigned Put url - returns a presigned url to upload an object without credentials.URL can have a maximum expiry of + /// Presigned Put url - returns a presigned url to upload an object without credentials.URL can have a maximum expiry of /// upto 7 days or a minimum of 1 second. /// /// Bucket to retrieve object from @@ -176,4 +176,4 @@ public interface IObjectOperations Task>> PresignedPostPolicyAsync(PostPolicy policy); } -} \ No newline at end of file +} diff --git a/Minio/ApiEndpoints/ObjectOperations.cs b/Minio/ApiEndpoints/ObjectOperations.cs index 624b3b540..c84901bd9 100644 --- a/Minio/ApiEndpoints/ObjectOperations.cs +++ b/Minio/ApiEndpoints/ObjectOperations.cs @@ -523,10 +523,10 @@ private async Task PutObjectAsync(string bucketName, string objectName, { var queries = new List(); queries.Add("uploads"); - if (prefix != null) - { - queries.Add("prefix=" + Uri.EscapeDataString(prefix)); - } + queries.Add("prefix=" + Uri.EscapeDataString(prefix)); + queries.Add("delimiter=" + Uri.EscapeDataString(delimiter)); + queries.Add("max-uploads=1000"); + if (keyMarker != null) { queries.Add("key-marker=" + Uri.EscapeDataString(keyMarker)); @@ -535,12 +535,6 @@ private async Task PutObjectAsync(string bucketName, string objectName, { queries.Add("upload-id-marker=" + uploadIdMarker); } - if (delimiter != null) - { - queries.Add("delimiter=" + delimiter); - } - - queries.Add("max-uploads=1000"); string query = string.Join("&", queries); @@ -571,15 +565,15 @@ private async Task PutObjectAsync(string bucketName, string objectName, /// Lists all incomplete uploads in a given bucket and prefix recursively /// /// Bucket to list all incomplepte uploads from - /// prefix to list all incomplete uploads - /// option to list incomplete uploads recursively + /// Filter all incomplete uploads starting with this prefix + /// Set to true to recursively list all incomplete uploads /// Optional cancellation token to cancel the operation /// A lazily populated list of incomplete uploads public IObservable ListIncompleteUploads(string bucketName, string prefix = "", bool recursive = true, CancellationToken cancellationToken = default(CancellationToken)) { if (recursive) { - return this.listIncompleteUploads(bucketName, prefix, null, cancellationToken); + return this.listIncompleteUploads(bucketName, prefix, "", cancellationToken); } return this.listIncompleteUploads(bucketName, prefix, "/", cancellationToken); } @@ -617,36 +611,6 @@ private IObservable listIncompleteUploads(string bucketName, string pref } - /// - /// Find uploadId of most recent unsuccessful attempt to upload object to bucket. - /// - /// - /// - /// Optional cancellation token to cancel the operation - /// - private async Task getLatestIncompleteUploadIdAsync(string bucketName, string objectName, CancellationToken cancellationToken) - { - Upload latestUpload = null; - var uploads = await this.ListIncompleteUploads(bucketName, objectName, cancellationToken: cancellationToken).ToArray(); - - foreach (Upload upload in uploads) - { - if (objectName == upload.Key && (latestUpload == null || latestUpload.Initiated.CompareTo(upload.Initiated) < 0)) - { - latestUpload = upload; - - } - } - if (latestUpload != null) - { - return latestUpload.UploadId; - } - else - { - return null; - } - - } /// /// Remove incomplete uploads from a given bucket and objectName ///