diff --git a/Minio.Functional.Tests/FunctionalTest.cs b/Minio.Functional.Tests/FunctionalTest.cs index a07f405eb..66ff1c9b4 100644 --- a/Minio.Functional.Tests/FunctionalTest.cs +++ b/Minio.Functional.Tests/FunctionalTest.cs @@ -76,7 +76,7 @@ private static String CreateFile(int size, string dataFileName = null) File.WriteAllBytes(fileName, data); return GetFilePath(fileName); } - + return GetFilePath(dataFileName); } @@ -94,8 +94,8 @@ public static String GetRandomName(int length = 5) } return "miniodotnet" + result.ToString(); } - // Return true if running in Mint mode - public static bool IsMintEnv() + // Return true if running in Mint mode + public static bool IsMintEnv() { return !String.IsNullOrEmpty(Environment.GetEnvironmentVariable("MINT_DATA_DIR")); } @@ -146,13 +146,13 @@ public static void Main(string[] args) else minioClient = new MinioClient(endPoint, accessKey, secretKey); - // Assign parameters before starting the test + // Assign parameters before starting the test string bucketName = GetRandomName(); string objectName = GetRandomName(); string destBucketName = GetRandomName(); string destObjectName = GetRandomName(); - // Set app Info + // Set app Info minioClient.SetAppInfo("app-name", "app-version"); // Set HTTP Tracing On // minioClient.SetTraceOn(new JsonNetLogger()); @@ -277,7 +277,7 @@ private static void runCoreTests(MinioClient minioClient) ListObjects_Test1(minioClient).Wait(); RemoveObject_Test1(minioClient).Wait(); CopyObject_Test1(minioClient).Wait(); - + // Test SetPolicyAsync function SetBucketPolicy_Test1(minioClient).Wait(); @@ -313,7 +313,7 @@ private async static Task BucketExists_Test(MinioClient minio) catch (MinioException ex) { new MintLogger("BucketExists_Test",bucketExistsSignature,"Tests whether BucketExists passes",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(),args).Log(); - } + } } private async static Task MakeBucket_Test1(MinioClient minio) @@ -337,7 +337,7 @@ private async static Task MakeBucket_Test1(MinioClient minio) catch (MinioException ex) { new MintLogger("MakeBucket_Test1",makeBucketSignature,"Tests whether MakeBucket passes",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message,ex.ToString(),args).Log(); - } + } } private async static Task MakeBucket_Test2(MinioClient minio) @@ -363,7 +363,7 @@ private async static Task MakeBucket_Test2(MinioClient minio) { new MintLogger("MakeBucket_Test2",makeBucketSignature,testType,TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(),args).Log(); } - + } private async static Task MakeBucket_Test3(MinioClient minio, bool aws = false) { @@ -504,7 +504,7 @@ private async static Task PutObject_Test1(MinioClient minio) } } - + private async static Task PutObject_Test2(MinioClient minio) { DateTime startTime = DateTime.Now; @@ -671,7 +671,7 @@ private async static Task PutObject_Test7(MinioClient minio) await Setup_Test(minio, bucketName); using (System.IO.MemoryStream filestream = rsg.GenerateStreamFromSeed(1 * MB)) { - + long size = -1; long file_write_size = filestream.Length; @@ -744,7 +744,7 @@ private async static Task PutGetStatEncryptedObject_Test1(MinioClient minio) }; try { - // Putobject with SSE-C encryption. + // Putobject with SSE-C encryption. await Setup_Test(minio, bucketName); Aes aesEncryption = Aes.Create(); aesEncryption.KeySize = 256; @@ -1268,7 +1268,7 @@ private async static Task CopyObject_Test5(MinioClient minio) new MintLogger("CopyObject_Test5",copyObjectSignature,"Tests whether CopyObject multi-part copy upload for large files works",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(),args).Log(); } } - + } private async static Task CopyObject_Test6(MinioClient minio) @@ -1702,7 +1702,7 @@ private async static Task GetObject_Test1(MinioClient minio) { new MintLogger("GetObject_Test1",getObjectSignature1,"Tests whether GetObject as stream works",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(),args).Log(); } - + } private async static Task GetObject_Test2(MinioClient minio) { @@ -1736,7 +1736,7 @@ private async static Task GetObject_Test2(MinioClient minio) { new MintLogger("GetObject_Test2",getObjectSignature1,"Tests for non-existent GetObject",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(),args).Log(); } - + } private async static Task GetObject_Test3(MinioClient minio) { @@ -1822,7 +1822,7 @@ private async static Task FGetObject_Test1(MinioClient minio) { new MintLogger("FGetObject_Test1",getObjectSignature3,"Tests whether FGetObject passes for small upload",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(),args).Log(); } - + } private async static Task FPutObject_Test1(MinioClient minio) @@ -1913,8 +1913,8 @@ private async static Task ListObjects_Test1(MinioClient minio) tasks[i]= PutObject_Task(minio, bucketName, objectName + i.ToString(), null, null, 0, null, rsg.GenerateStreamFromSeed(1*MB)); } await Task.WhenAll(tasks); - - ListObjects_Test(minio, bucketName, prefix, 2,false).Wait(); + + ListObjects_Test(minio, bucketName, prefix, 2, false).Wait(); System.Threading.Thread.Sleep(5000); await minio.RemoveObjectAsync(bucketName, objectName + "0"); await minio.RemoveObjectAsync(bucketName, objectName + "1"); @@ -1939,7 +1939,7 @@ private async static Task ListObjects_Test2(MinioClient minio) { await Setup_Test(minio, bucketName); - ListObjects_Test(minio, bucketName, null, 0).Wait(5000); + ListObjects_Test(minio, bucketName, "", 0).Wait(5000); await TearDown(minio, bucketName); new MintLogger("ListObjects_Test2",listObjectsSignature,"Tests whether ListObjects passes when bucket is empty",TestStatus.PASS,(DateTime.Now - startTime),args:args).Log(); } @@ -1971,7 +1971,7 @@ private async static Task ListObjects_Test3(MinioClient minio) } await Task.WhenAll(tasks); - ListObjects_Test(minio, bucketName, prefix, 2,true).Wait(); + ListObjects_Test(minio, bucketName, prefix, 2, true).Wait(); System.Threading.Thread.Sleep(5000); await minio.RemoveObjectAsync(bucketName, objectName + "0"); await minio.RemoveObjectAsync(bucketName, objectName + "1"); @@ -2003,8 +2003,8 @@ private async static Task ListObjects_Test4(MinioClient minio) tasks[i]= PutObject_Task(minio, bucketName, objectName + i.ToString(), null, null, 0, null, rsg.GenerateStreamFromSeed(1*MB)); } await Task.WhenAll(tasks); - - ListObjects_Test(minio, bucketName, null, 2,false).Wait(); + + ListObjects_Test(minio, bucketName, "", 2, false).Wait(); System.Threading.Thread.Sleep(5000); await minio.RemoveObjectAsync(bucketName, objectName + "0"); await minio.RemoveObjectAsync(bucketName, objectName + "1"); @@ -2040,8 +2040,8 @@ private async static Task ListObjects_Test5(MinioClient minio) } } await Task.WhenAll(tasks); - - ListObjects_Test(minio, bucketName, objectNamePrefix, numObjects,false).Wait(); + + ListObjects_Test(minio, bucketName, objectNamePrefix, numObjects, false).Wait(); System.Threading.Thread.Sleep(5000); for(int index=1; index <= numObjects; index++) { @@ -2364,7 +2364,7 @@ private async static Task PresignedPutObject_Test2(MinioClient minio) new MintLogger("PresignedPutObject_Test2",presignedPutObjectSignature,"Tests whether PresignedPutObject url retrieves object from bucket when invalid expiry is set.",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(),args).Log(); } } - + private static async Task UploadObjectAsync(string url, string filePath) { HttpWebRequest httpRequest = WebRequest.Create(url) as HttpWebRequest; @@ -2406,7 +2406,7 @@ private async static Task PresignedPostPolicy_Test1(MinioClient minio) await Setup_Test(minio, bucketName); await minio.PutObjectAsync(bucketName, objectName, - fileName); + fileName); var pairs = new List>(); string url = "https://s3.amazonaws.com/" + bucketName; Tuple> policyTuple = await minio.PresignedPostPolicyAsync(form); @@ -2477,7 +2477,7 @@ private async static Task ListIncompleteUpload_Test1(MinioClient minio) IDisposable subscription = observable.Subscribe( item => Assert.AreEqual(item.Key, objectName), - ex => Assert.Fail()); + ex => Assert.Fail()); await minio.RemoveIncompleteUploadAsync(bucketName, objectName); } @@ -2498,7 +2498,7 @@ private async static Task ListIncompleteUpload_Test2(MinioClient minio) { DateTime startTime = DateTime.Now; string bucketName = GetRandomName(15); - string prefix = "minioprefix/"; + string prefix = "minioprefix/"; string objectName = prefix + GetRandomName(10); string contentType = "gzip"; Dictionary args = new Dictionary @@ -2527,11 +2527,11 @@ private async static Task ListIncompleteUpload_Test2(MinioClient minio) } catch (OperationCanceledException) { - IObservable observable = minio.ListIncompleteUploads(bucketName,"minioprefix",false); + IObservable observable = minio.ListIncompleteUploads(bucketName, "minioprefix", false); IDisposable subscription = observable.Subscribe( item => Assert.AreEqual(item.Key, objectName), - ex => Assert.Fail()); + ex => Assert.Fail()); await minio.RemoveIncompleteUploadAsync(bucketName, objectName); } @@ -2577,11 +2577,11 @@ private async static Task ListIncompleteUpload_Test3(MinioClient minio) } catch (OperationCanceledException) { - IObservable observable = minio.ListIncompleteUploads(bucketName,prefix,true); + IObservable observable = minio.ListIncompleteUploads(bucketName, prefix, true); IDisposable subscription = observable.Subscribe( item => Assert.AreEqual(item.Key, objectName), - ex => Assert.Fail()); + ex => Assert.Fail()); await minio.RemoveIncompleteUploadAsync(bucketName, objectName); } @@ -2631,7 +2631,7 @@ private async static Task RemoveIncompleteUpload_Test(MinioClient minio) IDisposable subscription = observable.Subscribe( item => Assert.Fail(), - ex => Assert.Fail()); + ex => Assert.Fail()); } await TearDown(minio, bucketName); new MintLogger("RemoveIncompleteUpload_Test",removeIncompleteUploadSignature,"Tests whether RemoveIncompleteUpload passes.",TestStatus.PASS,(DateTime.Now - startTime), args:args).Log(); diff --git a/Minio/ApiEndpoints/BucketOperations.cs b/Minio/ApiEndpoints/BucketOperations.cs index 12f6e6ff5..cdbdf3232 100644 --- a/Minio/ApiEndpoints/BucketOperations.cs +++ b/Minio/ApiEndpoints/BucketOperations.cs @@ -72,7 +72,7 @@ public async Task MakeBucketAsync(string bucketName, string location = "us-east- location = this.Region; } } - + // Set Target URL Uri requestUrl = RequestUtil.MakeTargetURL(this.BaseUrl, this.Secure,location); SetTargetURL(requestUrl); @@ -134,20 +134,27 @@ public async Task RemoveBucketAsync(string bucketName, CancellationToken cancell /// List all objects non-recursively in a bucket with a given prefix, optionally emulating a directory /// /// Bucket to list objects from - /// Filters all objects not beginning with a given prefix - /// Set to false to emulate a directory + /// Filters all objects beginning with a given prefix + /// Set to true to recursively list all objects /// Optional cancellation token to cancel the operation /// An observable of items that client can subscribe to - public IObservable ListObjectsAsync(string bucketName, string prefix = null, bool recursive = true, CancellationToken cancellationToken = default(CancellationToken)) + public IObservable ListObjectsAsync(string bucketName, string prefix = "", bool recursive = false, CancellationToken cancellationToken = default(CancellationToken)) { return Observable.Create( async obs => { bool isRunning = true; string marker = null; + + var delimiter = "/"; + if (recursive) + { + delimiter = ""; + } + while (isRunning) { - Tuple> result = await GetObjectListAsync(bucketName, prefix, recursive, marker, cancellationToken).ConfigureAwait(false); + Tuple> result = await GetObjectListAsync(bucketName, prefix, delimiter, marker, cancellationToken).ConfigureAwait(false); Item lastItem = null; foreach (Item item in result.Item2) { @@ -171,35 +178,30 @@ public IObservable ListObjectsAsync(string bucketName, string prefix = nul /// Gets the list of objects in the bucket filtered by prefix /// /// Bucket to list objects from - /// Filters all objects not beginning with a given prefix - /// Set to false to emulate a directory + /// Filters all objects starting with a given prefix + /// Delimit the output upto this character /// marks location in the iterator sequence /// Task with a tuple populated with objects /// Optional cancellation token to cancel the operation - private async Task>> GetObjectListAsync(string bucketName, string prefix, bool recursive, string marker, CancellationToken cancellationToken = default(CancellationToken)) + private async Task>> GetObjectListAsync(string bucketName, string prefix, string delimiter, string marker, CancellationToken cancellationToken = default(CancellationToken)) { var queries = new List(); - if (!recursive) - { - queries.Add("delimiter=%2F"); - } - if (prefix != null) - { - queries.Add("prefix=" + Uri.EscapeDataString(prefix)); - } + queries.Add("delimiter="+ Uri.EscapeDataString(delimiter)); + queries.Add("prefix=" + Uri.EscapeDataString(prefix)); + queries.Add("max-keys=1000"); + if (marker != null) { queries.Add("marker=" + Uri.EscapeDataString(marker)); } - queries.Add("max-keys=1000"); string query = string.Join("&", queries); var request = await this.CreateRequest(Method.GET, bucketName, resourcePath: "?" + query) .ConfigureAwait(false); - + var response = await this.ExecuteTaskAsync(this.NoErrorHandlers, request, cancellationToken).ConfigureAwait(false); var contentBytes = System.Text.Encoding.UTF8.GetBytes(response.Content); @@ -292,7 +294,7 @@ public async Task GetBucketNotificationsAsync(string bucketN resourcePath: "?notification") .ConfigureAwait(false); BucketNotification notification = null; - + var response = await this.ExecuteTaskAsync(this.NoErrorHandlers, request, cancellationToken).ConfigureAwait(false); var contentBytes = System.Text.Encoding.UTF8.GetBytes(response.Content); using (var stream = new MemoryStream(contentBytes)) @@ -314,11 +316,11 @@ public async Task SetBucketNotificationsAsync(string bucketName, BucketNotificat var request = await this.CreateRequest(Method.PUT, bucketName, resourcePath: "?notification") .ConfigureAwait(false); - + request.XmlSerializer = new RestSharp.Serializers.DotNetXmlSerializer(); request.RequestFormat = DataFormat.Xml; request.AddBody(notification); - + IRestResponse response = await this.ExecuteTaskAsync(this.NoErrorHandlers, request, cancellationToken).ConfigureAwait(false); } @@ -335,4 +337,4 @@ public Task RemoveAllBucketNotificationsAsync(string bucketName, CancellationTok return SetBucketNotificationsAsync(bucketName, notification, cancellationToken); } } -} \ No newline at end of file +} diff --git a/Minio/ApiEndpoints/IBucketOperations.cs b/Minio/ApiEndpoints/IBucketOperations.cs index 33c8cfacc..a50a5d942 100644 --- a/Minio/ApiEndpoints/IBucketOperations.cs +++ b/Minio/ApiEndpoints/IBucketOperations.cs @@ -58,11 +58,11 @@ public interface IBucketOperations /// List all objects non-recursively in a bucket with a given prefix, optionally emulating a directory /// /// Bucket to list objects from - /// Filters all objects not beginning with a given prefix - /// Set to false to emulate a directory + /// Filter all incomplete uploads starting with this prefix + /// List incomplete uploads recursively /// /// An observable of items that client can subscribe to - IObservable ListObjectsAsync(string bucketName, string prefix = null, bool recursive = true, CancellationToken cancellationToken = default(CancellationToken)); + IObservable ListObjectsAsync(string bucketName, string prefix = "", bool recursive = false, CancellationToken cancellationToken = default(CancellationToken)); /// /// Get bucket policy @@ -80,7 +80,7 @@ public interface IBucketOperations /// Optional cancellation token to cancel the operation /// Returns Task that sets the current bucket policy Task SetPolicyAsync(String bucketName, String policyJson, CancellationToken cancellationToken = default(CancellationToken)); - + /// /// Gets the notification configuration set for this bucket /// @@ -105,8 +105,8 @@ public interface IBucketOperations /// optional cancellation token /// Task RemoveAllBucketNotificationsAsync(string bucketName, CancellationToken cancellationToken = default(CancellationToken)); - + // Task ListenBucketNotificationsAsync(string bucketName, string prefix = "", string suffix = "", List events,CancellationToken cancellationToken = default(CancellationToken)); - + } -} \ No newline at end of file +} diff --git a/Minio/ApiEndpoints/IObjectOperations.cs b/Minio/ApiEndpoints/IObjectOperations.cs index 8f98be1da..423af234f 100644 --- a/Minio/ApiEndpoints/IObjectOperations.cs +++ b/Minio/ApiEndpoints/IObjectOperations.cs @@ -94,10 +94,10 @@ public interface IObjectOperations /// /// Bucket to list all incomplepte uploads from /// prefix to list all incomplete uploads - /// option to list incomplete uploads recursively + /// Set to true to recursively list all incomplete uploads /// Optional cancellation token to cancel the operation /// A lazily populated list of incomplete uploads - IObservable ListIncompleteUploads(string bucketName, string prefix, bool recursive, CancellationToken cancellationToken = default(CancellationToken)); + IObservable ListIncompleteUploads(string bucketName, string prefix = "", bool recursive = false, CancellationToken cancellationToken = default(CancellationToken)); /// /// Remove incomplete uploads from a given bucket and objectName @@ -151,7 +151,7 @@ public interface IObjectOperations Task GetObjectAsync(string bucketName, string objectName, string filePath, ServerSideEncryption sse = null ,CancellationToken cancellationToken = default(CancellationToken)); /// - /// Presigned get url - returns a presigned url to access an object's data without credentials.URL can have a maximum expiry of + /// Presigned get url - returns a presigned url to access an object's data without credentials.URL can have a maximum expiry of /// upto 7 days or a minimum of 1 second.Additionally, you can override a set of response headers using reqParams. /// /// Bucket to retrieve object from @@ -161,7 +161,7 @@ public interface IObjectOperations Task PresignedGetObjectAsync(string bucketName, string objectName, int expiresInt, Dictionary reqParams = null); /// - /// Presigned Put url - returns a presigned url to upload an object without credentials.URL can have a maximum expiry of + /// Presigned Put url - returns a presigned url to upload an object without credentials.URL can have a maximum expiry of /// upto 7 days or a minimum of 1 second. /// /// Bucket to retrieve object from @@ -176,4 +176,4 @@ public interface IObjectOperations Task>> PresignedPostPolicyAsync(PostPolicy policy); } -} \ No newline at end of file +} diff --git a/Minio/ApiEndpoints/ObjectOperations.cs b/Minio/ApiEndpoints/ObjectOperations.cs index 624b3b540..c84901bd9 100644 --- a/Minio/ApiEndpoints/ObjectOperations.cs +++ b/Minio/ApiEndpoints/ObjectOperations.cs @@ -523,10 +523,10 @@ private async Task PutObjectAsync(string bucketName, string objectName, { var queries = new List(); queries.Add("uploads"); - if (prefix != null) - { - queries.Add("prefix=" + Uri.EscapeDataString(prefix)); - } + queries.Add("prefix=" + Uri.EscapeDataString(prefix)); + queries.Add("delimiter=" + Uri.EscapeDataString(delimiter)); + queries.Add("max-uploads=1000"); + if (keyMarker != null) { queries.Add("key-marker=" + Uri.EscapeDataString(keyMarker)); @@ -535,12 +535,6 @@ private async Task PutObjectAsync(string bucketName, string objectName, { queries.Add("upload-id-marker=" + uploadIdMarker); } - if (delimiter != null) - { - queries.Add("delimiter=" + delimiter); - } - - queries.Add("max-uploads=1000"); string query = string.Join("&", queries); @@ -571,15 +565,15 @@ private async Task PutObjectAsync(string bucketName, string objectName, /// Lists all incomplete uploads in a given bucket and prefix recursively /// /// Bucket to list all incomplepte uploads from - /// prefix to list all incomplete uploads - /// option to list incomplete uploads recursively + /// Filter all incomplete uploads starting with this prefix + /// Set to true to recursively list all incomplete uploads /// Optional cancellation token to cancel the operation /// A lazily populated list of incomplete uploads public IObservable ListIncompleteUploads(string bucketName, string prefix = "", bool recursive = true, CancellationToken cancellationToken = default(CancellationToken)) { if (recursive) { - return this.listIncompleteUploads(bucketName, prefix, null, cancellationToken); + return this.listIncompleteUploads(bucketName, prefix, "", cancellationToken); } return this.listIncompleteUploads(bucketName, prefix, "/", cancellationToken); } @@ -617,36 +611,6 @@ private IObservable listIncompleteUploads(string bucketName, string pref } - /// - /// Find uploadId of most recent unsuccessful attempt to upload object to bucket. - /// - /// - /// - /// Optional cancellation token to cancel the operation - /// - private async Task getLatestIncompleteUploadIdAsync(string bucketName, string objectName, CancellationToken cancellationToken) - { - Upload latestUpload = null; - var uploads = await this.ListIncompleteUploads(bucketName, objectName, cancellationToken: cancellationToken).ToArray(); - - foreach (Upload upload in uploads) - { - if (objectName == upload.Key && (latestUpload == null || latestUpload.Initiated.CompareTo(upload.Initiated) < 0)) - { - latestUpload = upload; - - } - } - if (latestUpload != null) - { - return latestUpload.UploadId; - } - else - { - return null; - } - - } /// /// Remove incomplete uploads from a given bucket and objectName ///