Skip to content

Commit

Permalink
Set prefix, delimiter params even when empty
Browse files Browse the repository at this point in the history 
We have never set values which are empty on the request
because they are perhaps not useful in the List query,
but this assumption is wrong when there are restricted
policies for a given user, because empty is actually
a valid value in IAM or Bucket policy conditions.

For example following condition would never work with our
ListObjects call and AWS cli would work fine.

            "Condition": {
                "StringEquals": {
                    "s3:prefix": [
                        "",
                        "data/",
                        "data"
                    ],
                    "s3:delimiter": [
                        "/",
                        ""
                    ]
                }
            }

The reason is empty or not prefix and delimiter should be
added to the query param in List operation, such that server
can use the value to validate the policies for the incoming
request.

Refer minio/minio-go#1064
  • Loading branch information
@harshavardhana
harshavardhana committed Jan 29, 2019
1 parent db8313d commit 0372670
Show file tree
Hide file tree
Showing 4 changed files with 43 additions and 77 deletions.
46 changes: 24 additions & 22 deletions Minio/ApiEndpoints/BucketOperations.cs
View file
Expand Up @@ -72,7 +72,7 @@ public async Task MakeBucketAsync(string bucketName, string location = "us-east-
location = this.Region;
}
}

// Set Target URL
Uri requestUrl = RequestUtil.MakeTargetURL(this.BaseUrl, this.Secure,location);
SetTargetURL(requestUrl);
Expand Down Expand Up @@ -134,20 +134,27 @@ public async Task RemoveBucketAsync(string bucketName, CancellationToken cancell
/// List all objects non-recursively in a bucket with a given prefix, optionally emulating a directory
/// </summary>
/// <param name="bucketName">Bucket to list objects from</param>
/// <param name="prefix">Filters all objects not beginning with a given prefix</param>
/// <param name="recursive">Set to false to emulate a directory</param>
/// <param name="prefix">Filters all objects beginning with a given prefix</param>
/// <param name="recursive">Set to true to recursively list all objects</param>
/// <param name="cancellationToken">Optional cancellation token to cancel the operation</param>
/// <returns>An observable of items that client can subscribe to</returns>
public IObservable<Item> ListObjectsAsync(string bucketName, string prefix = null, bool recursive = true, CancellationToken cancellationToken = default(CancellationToken))
public IObservable<Item> ListObjectsAsync(string bucketName, string prefix = "", bool recursive = false, CancellationToken cancellationToken = default(CancellationToken))
{
return Observable.Create<Item>(
async obs =>
{
bool isRunning = true;
string marker = null;
var delimiter = "/"
if (recursive)
{
delimiter = ""
}
while (isRunning)
{
Tuple<ListBucketResult, List<Item>> result = await GetObjectListAsync(bucketName, prefix, recursive, marker, cancellationToken).ConfigureAwait(false);
Tuple<ListBucketResult, List<Item>> result = await GetObjectListAsync(bucketName, prefix, delimiter, marker, cancellationToken).ConfigureAwait(false);
Item lastItem = null;
foreach (Item item in result.Item2)
{
Expand All @@ -171,35 +178,30 @@ public IObservable<Item> ListObjectsAsync(string bucketName, string prefix = nul
/// Gets the list of objects in the bucket filtered by prefix
/// </summary>
/// <param name="bucketName">Bucket to list objects from</param>
/// <param name="prefix">Filters all objects not beginning with a given prefix</param>
/// <param name="recursive">Set to false to emulate a directory</param>
/// <param name="prefix">Filters all objects starting with a given prefix</param>
/// <param name="delimiter">Delimit the output upto this character</param>
/// <param name="marker">marks location in the iterator sequence</param>
/// <returns>Task with a tuple populated with objects</returns>
/// <param name="cancellationToken">Optional cancellation token to cancel the operation</param>

private async Task<Tuple<ListBucketResult, List<Item>>> GetObjectListAsync(string bucketName, string prefix, bool recursive, string marker, CancellationToken cancellationToken = default(CancellationToken))
private async Task<Tuple<ListBucketResult, List<Item>>> GetObjectListAsync(string bucketName, string prefix, string delimiter, string marker, CancellationToken cancellationToken = default(CancellationToken))
{
var queries = new List<string>();
if (!recursive)
{
queries.Add("delimiter=%2F");
}
if (prefix != null)
{
queries.Add("prefix=" + Uri.EscapeDataString(prefix));
}
queries.Add("delimiter="+ Uri.EscapeDataString(delimiter));
queries.Add("prefix=" + Uri.EscapeDataString(prefix));
queries.Add("max-keys=1000");

if (marker != null)
{
queries.Add("marker=" + Uri.EscapeDataString(marker));
}
queries.Add("max-keys=1000");
string query = string.Join("&", queries);

var request = await this.CreateRequest(Method.GET,
bucketName,
resourcePath: "?" + query)
.ConfigureAwait(false);

var response = await this.ExecuteTaskAsync(this.NoErrorHandlers, request, cancellationToken).ConfigureAwait(false);

var contentBytes = System.Text.Encoding.UTF8.GetBytes(response.Content);
Expand Down Expand Up @@ -292,7 +294,7 @@ public async Task<BucketNotification> GetBucketNotificationsAsync(string bucketN
resourcePath: "?notification")
.ConfigureAwait(false);
BucketNotification notification = null;

var response = await this.ExecuteTaskAsync(this.NoErrorHandlers, request, cancellationToken).ConfigureAwait(false);
var contentBytes = System.Text.Encoding.UTF8.GetBytes(response.Content);
using (var stream = new MemoryStream(contentBytes))
Expand All @@ -314,11 +316,11 @@ public async Task SetBucketNotificationsAsync(string bucketName, BucketNotificat
var request = await this.CreateRequest(Method.PUT, bucketName,
resourcePath: "?notification")
.ConfigureAwait(false);

request.XmlSerializer = new RestSharp.Serializers.DotNetXmlSerializer();
request.RequestFormat = DataFormat.Xml;
request.AddBody(notification);

IRestResponse response = await this.ExecuteTaskAsync(this.NoErrorHandlers, request, cancellationToken).ConfigureAwait(false);
}

Expand All @@ -335,4 +337,4 @@ public Task RemoveAllBucketNotificationsAsync(string bucketName, CancellationTok
return SetBucketNotificationsAsync(bucketName, notification, cancellationToken);
}
}
}
}
14 changes: 7 additions & 7 deletions Minio/ApiEndpoints/IBucketOperations.cs
View file
Expand Up @@ -58,11 +58,11 @@ public interface IBucketOperations
/// List all objects non-recursively in a bucket with a given prefix, optionally emulating a directory
/// </summary>
/// <param name="bucketName">Bucket to list objects from</param>
/// <param name="prefix">Filters all objects not beginning with a given prefix</param>
/// <param name="recursive">Set to false to emulate a directory</param>
/// <param name="prefix">Filter all incomplete uploads starting with this prefix</param>
/// <param name="recursive">List incomplete uploads recursively</param>
/// <param name="cancellationToken"></param>
/// <returns>An observable of items that client can subscribe to</returns>
IObservable<Item> ListObjectsAsync(string bucketName, string prefix = null, bool recursive = true, CancellationToken cancellationToken = default(CancellationToken));
IObservable<Item> ListObjectsAsync(string bucketName, string prefix = "", bool recursive = false, CancellationToken cancellationToken = default(CancellationToken));

/// <summary>
/// Get bucket policy
Expand All @@ -80,7 +80,7 @@ public interface IBucketOperations
/// <param name="cancellationToken">Optional cancellation token to cancel the operation</param>
/// <returns> Returns Task that sets the current bucket policy</returns>
Task SetPolicyAsync(String bucketName, String policyJson, CancellationToken cancellationToken = default(CancellationToken));

/// <summary>
/// Gets the notification configuration set for this bucket
/// </summary>
Expand All @@ -105,8 +105,8 @@ public interface IBucketOperations
/// <param name="cancellationToken">optional cancellation token</param>
/// <returns></returns>
Task RemoveAllBucketNotificationsAsync(string bucketName, CancellationToken cancellationToken = default(CancellationToken));

// Task ListenBucketNotificationsAsync(string bucketName, string prefix = "", string suffix = "", List<Notification> events,CancellationToken cancellationToken = default(CancellationToken));

}
}
}
10 changes: 5 additions & 5 deletions Minio/ApiEndpoints/IObjectOperations.cs
View file
Expand Up @@ -94,10 +94,10 @@ public interface IObjectOperations
/// </summary>
/// <param name="bucketName">Bucket to list all incomplepte uploads from</param>
/// <param name="prefix">prefix to list all incomplete uploads</param>
/// <param name="recursive">option to list incomplete uploads recursively</param>
/// <param name="recursive">Set to true to recursively list all incomplete uploads</param>
/// <param name="cancellationToken">Optional cancellation token to cancel the operation</param>
/// <returns>A lazily populated list of incomplete uploads</returns>
IObservable<Upload> ListIncompleteUploads(string bucketName, string prefix, bool recursive, CancellationToken cancellationToken = default(CancellationToken));
IObservable<Upload> ListIncompleteUploads(string bucketName, string prefix = "", bool recursive = false, CancellationToken cancellationToken = default(CancellationToken));

/// <summary>
/// Remove incomplete uploads from a given bucket and objectName
Expand Down Expand Up @@ -151,7 +151,7 @@ public interface IObjectOperations
Task GetObjectAsync(string bucketName, string objectName, string filePath, ServerSideEncryption sse = null ,CancellationToken cancellationToken = default(CancellationToken));

/// <summary>
/// Presigned get url - returns a presigned url to access an object's data without credentials.URL can have a maximum expiry of
/// Presigned get url - returns a presigned url to access an object's data without credentials.URL can have a maximum expiry of
/// upto 7 days or a minimum of 1 second.Additionally, you can override a set of response headers using reqParams.
/// </summary>
/// <param name="bucketName">Bucket to retrieve object from</param>
Expand All @@ -161,7 +161,7 @@ public interface IObjectOperations
Task<string> PresignedGetObjectAsync(string bucketName, string objectName, int expiresInt, Dictionary<string,string> reqParams = null);

/// <summary>
/// Presigned Put url - returns a presigned url to upload an object without credentials.URL can have a maximum expiry of
/// Presigned Put url - returns a presigned url to upload an object without credentials.URL can have a maximum expiry of
/// upto 7 days or a minimum of 1 second.
/// </summary>
/// <param name="bucketName">Bucket to retrieve object from</param>
Expand All @@ -176,4 +176,4 @@ public interface IObjectOperations
Task<Tuple<string, Dictionary<string, string>>> PresignedPostPolicyAsync(PostPolicy policy);

}
}
}
50 changes: 7 additions & 43 deletions Minio/ApiEndpoints/ObjectOperations.cs
View file
Expand Up @@ -523,10 +523,10 @@ private async Task<string> PutObjectAsync(string bucketName, string objectName,
{
var queries = new List<string>();
queries.Add("uploads");
if (prefix != null)
{
queries.Add("prefix=" + Uri.EscapeDataString(prefix));
}
queries.Add("prefix=" + Uri.EscapeDataString(prefix));
queries.Add("delimiter=" + Uri.EscapeDataString(delimiter));
queries.Add("max-uploads=1000");

if (keyMarker != null)
{
queries.Add("key-marker=" + Uri.EscapeDataString(keyMarker));
Expand All @@ -535,12 +535,6 @@ private async Task<string> PutObjectAsync(string bucketName, string objectName,
{
queries.Add("upload-id-marker=" + uploadIdMarker);
}
if (delimiter != null)
{
queries.Add("delimiter=" + delimiter);
}

queries.Add("max-uploads=1000");

string query = string.Join("&", queries);

Expand Down Expand Up @@ -571,15 +565,15 @@ private async Task<string> PutObjectAsync(string bucketName, string objectName,
/// Lists all incomplete uploads in a given bucket and prefix recursively
/// </summary>
/// <param name="bucketName">Bucket to list all incomplepte uploads from</param>
/// <param name="prefix">prefix to list all incomplete uploads</param>
/// <param name="recursive">option to list incomplete uploads recursively</param>
/// <param name="prefix">Filter all incomplete uploads starting with this prefix</param>
/// <param name="recursive">Set to true to recursively list all incomplete uploads</param>
/// <param name="cancellationToken">Optional cancellation token to cancel the operation</param>
/// <returns>A lazily populated list of incomplete uploads</returns>
public IObservable<Upload> ListIncompleteUploads(string bucketName, string prefix = "", bool recursive = true, CancellationToken cancellationToken = default(CancellationToken))
{
if (recursive)
{
return this.listIncompleteUploads(bucketName, prefix, null, cancellationToken);
return this.listIncompleteUploads(bucketName, prefix, "", cancellationToken);
}
return this.listIncompleteUploads(bucketName, prefix, "/", cancellationToken);
}
Expand Down Expand Up @@ -617,36 +611,6 @@ private IObservable<Upload> listIncompleteUploads(string bucketName, string pref

}

/// <summary>
/// Find uploadId of most recent unsuccessful attempt to upload object to bucket.
/// </summary>
/// <param name="bucketName"></param>
/// <param name="objectName"></param>
/// <param name="cancellationToken">Optional cancellation token to cancel the operation</param>
/// <returns></returns>
private async Task<string> getLatestIncompleteUploadIdAsync(string bucketName, string objectName, CancellationToken cancellationToken)
{
Upload latestUpload = null;
var uploads = await this.ListIncompleteUploads(bucketName, objectName, cancellationToken: cancellationToken).ToArray();

foreach (Upload upload in uploads)
{
if (objectName == upload.Key && (latestUpload == null || latestUpload.Initiated.CompareTo(upload.Initiated) < 0))
{
latestUpload = upload;

}
}
if (latestUpload != null)
{
return latestUpload.UploadId;
}
else
{
return null;
}

}
/// <summary>
/// Remove incomplete uploads from a given bucket and objectName
/// </summary>
Expand Down

0 comments on commit 0372670

Please sign in to comment.