Skip to content

Commit

Permalink
Merge pull request #100 from harshavardhana/pr_out_overhaul_in_terms_…
Browse files Browse the repository at this point in the history 
…of_readability_and_definitions_are_simplified

Overhaul in terms of readability and definitions are simplified
  • Loading branch information
Harshavardhana committed Jun 16, 2015
2 parents e0c6ba5 + 34c4186 commit 115db49
Show file tree
Hide file tree
Showing 4 changed files with 99 additions and 71 deletions.
48 changes: 24 additions & 24 deletions api.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,7 @@ func New(config Config) (API, error) {
// Downloads full object with no ranges, if you need ranges use GetPartialObject
func (a api) GetObject(bucket, object string) (io.ReadCloser, ObjectStat, error) {
// get object
return a.getPartialObject(bucket, object, 0, 0)
return a.getObject(bucket, object, 0, 0)
}

// GetPartialObject retrieve partial object
Expand All @@ -215,7 +215,7 @@ func (a api) GetObject(bucket, object string) (io.ReadCloser, ObjectStat, error)
// For more information about the HTTP Range header, go to http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35.
func (a api) GetPartialObject(bucket, object string, offset, length int64) (io.ReadCloser, ObjectStat, error) {
// get partial object
return a.getPartialObject(bucket, object, offset, length)
return a.getObject(bucket, object, offset, length)
}

// completedParts is a wrapper to make parts sortable by their part number
Expand Down Expand Up @@ -277,9 +277,9 @@ func (a api) newObjectUpload(bucket, object, contentType string, size int64, dat
if err != nil {
return err
}
completeMultipartUpload.Part = append(completeMultipartUpload.Part, completePart)
completeMultipartUpload.Parts = append(completeMultipartUpload.Parts, completePart)
}
sort.Sort(completedParts(completeMultipartUpload.Part))
sort.Sort(completedParts(completeMultipartUpload.Parts))
_, err = a.completeMultipartUpload(bucket, object, uploadID, completeMultipartUpload)
if err != nil {
return err
Expand Down Expand Up @@ -308,7 +308,7 @@ func (a api) listObjectPartsRecursiveInRoutine(bucket, object, uploadID string,
}
return
}
for _, uploadedPart := range listObjectPartsResult.Part {
for _, uploadedPart := range listObjectPartsResult.Parts {
ch <- partCh{
Metadata: uploadedPart,
Err: nil,
Expand All @@ -326,7 +326,7 @@ func (a api) listObjectPartsRecursiveInRoutine(bucket, object, uploadID string,
}
return
}
for _, uploadedPart := range listObjectPartsResult.Part {
for _, uploadedPart := range listObjectPartsResult.Parts {
ch <- partCh{
Metadata: uploadedPart,
Err: nil,
Expand All @@ -350,7 +350,7 @@ func (a api) continueObjectUpload(bucket, object, uploadID string, size int64, d
var completedPart completePart
completedPart.PartNumber = part.Metadata.PartNumber
completedPart.ETag = part.Metadata.ETag
completeMultipartUpload.Part = append(completeMultipartUpload.Part, completedPart)
completeMultipartUpload.Parts = append(completeMultipartUpload.Parts, completedPart)
md5SumBytes, err := hex.DecodeString(strings.Trim(part.Metadata.ETag, "\"")) // trim off the odd double quotes
if err != nil {
return err
Expand All @@ -368,9 +368,9 @@ func (a api) continueObjectUpload(bucket, object, uploadID string, size int64, d
if err != nil {
return err
}
completeMultipartUpload.Part = append(completeMultipartUpload.Part, completedPart)
completeMultipartUpload.Parts = append(completeMultipartUpload.Parts, completedPart)
}
sort.Sort(completedParts(completeMultipartUpload.Part))
sort.Sort(completedParts(completeMultipartUpload.Parts))
_, err := a.completeMultipartUpload(bucket, object, uploadID, completeMultipartUpload)
if err != nil {
return err
Expand All @@ -379,7 +379,7 @@ func (a api) continueObjectUpload(bucket, object, uploadID string, size int64, d
}

type multiPartUploadCh struct {
Metadata upload
Metadata multiPartUpload
Err error
}

Expand All @@ -394,14 +394,14 @@ func (a api) listMultipartUploadsRecursiveInRoutine(bucket, object string, ch ch
listMultipartUploadsResult, err := a.listMultipartUploads(bucket, "", "", object, "", 1000)
if err != nil {
ch <- multiPartUploadCh{
Metadata: upload{},
Metadata: multiPartUpload{},
Err: err,
}
return
}
for _, upload := range listMultipartUploadsResult.Upload {
for _, multiPartUpload := range listMultipartUploadsResult.Uploads {
ch <- multiPartUploadCh{
Metadata: upload,
Metadata: multiPartUpload,
Err: nil,
}
}
Expand All @@ -413,14 +413,14 @@ func (a api) listMultipartUploadsRecursiveInRoutine(bucket, object string, ch ch
listMultipartUploadsResult.NextKeyMarker, listMultipartUploadsResult.NextUploadIDMarker, object, "", 1000)
if err != nil {
ch <- multiPartUploadCh{
Metadata: upload{},
Metadata: multiPartUpload{},
Err: err,
}
return
}
for _, upload := range listMultipartUploadsResult.Upload {
for _, multiPartUpload := range listMultipartUploadsResult.Uploads {
ch <- multiPartUploadCh{
Metadata: upload,
Metadata: multiPartUpload,
Err: nil,
}
}
Expand Down Expand Up @@ -721,8 +721,8 @@ func (a api) dropIncompleteUploadsInRoutine(bucket, object string, errorCh chan
errorCh <- err
return
}
for _, upload := range listMultipartUploadsResult.Upload {
err := a.abortMultipartUpload(bucket, upload.Key, upload.UploadID)
for _, multiPartUpload := range listMultipartUploadsResult.Uploads {
err := a.abortMultipartUpload(bucket, multiPartUpload.Key, multiPartUpload.UploadID)
if err != nil {
errorCh <- err
return
Expand All @@ -738,8 +738,8 @@ func (a api) dropIncompleteUploadsInRoutine(bucket, object string, errorCh chan
errorCh <- err
return
}
for _, upload := range listMultipartUploadsResult.Upload {
err := a.abortMultipartUpload(bucket, upload.Key, upload.UploadID)
for _, multiPartUpload := range listMultipartUploadsResult.Uploads {
err := a.abortMultipartUpload(bucket, multiPartUpload.Key, multiPartUpload.UploadID)
if err != nil {
errorCh <- err
return
Expand Down Expand Up @@ -774,8 +774,8 @@ func (a api) dropAllIncompleteUploadsInRoutine(bucket string, errorCh chan error
errorCh <- err
return
}
for _, upload := range listMultipartUploadsResult.Upload {
err := a.abortMultipartUpload(bucket, upload.Key, upload.UploadID)
for _, multiPartUpload := range listMultipartUploadsResult.Uploads {
err := a.abortMultipartUpload(bucket, multiPartUpload.Key, multiPartUpload.UploadID)
if err != nil {
errorCh <- err
return
Expand All @@ -791,8 +791,8 @@ func (a api) dropAllIncompleteUploadsInRoutine(bucket string, errorCh chan error
errorCh <- err
return
}
for _, upload := range listMultipartUploadsResult.Upload {
err := a.abortMultipartUpload(bucket, upload.Key, upload.UploadID)
for _, multiPartUpload := range listMultipartUploadsResult.Uploads {
err := a.abortMultipartUpload(bucket, multiPartUpload.Key, multiPartUpload.UploadID)
if err != nil {
errorCh <- err
return
Expand Down
35 changes: 20 additions & 15 deletions lowlevelapi.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,12 @@ import (
"time"
)

// lowLevelAPI container to hold unexported internal functions
type lowLevelAPI struct {
config *Config
}

// putBucketRequest wrapper creates a new PutBucket request
// putBucketRequest wrapper creates a new putBucket request
func (a lowLevelAPI) putBucketRequest(bucket, acl, location string) (*request, error) {
var r *request
var err error
Expand Down Expand Up @@ -103,11 +104,13 @@ func (a lowLevelAPI) putBucketRequest(bucket, acl, location string) (*request, e
// private - owner gets full access [DEFAULT]
// public-read - owner gets full access, others get read access
// public-read-write - owner gets full access, others get full access too
// authenticated-read - owner gets full access, authenticated users get read access
// ------------------
//
// Location valid values
// ------------------
// [ us-west-1 | us-west-2 | eu-west-1 | eu-central-1 | ap-southeast-1 | ap-northeast-1 | ap-southeast-2 | sa-east-1 ]
//
// Default - US standard
func (a lowLevelAPI) putBucket(bucket, acl, location string) error {
req, err := a.putBucketRequest(bucket, acl, location)
Expand Down Expand Up @@ -142,7 +145,7 @@ func (a lowLevelAPI) putBucketACLRequest(bucket, acl string) (*request, error) {
return req, nil
}

// putBucketACL set the permissions on an existing bucket using access control lists (ACL)
// putBucketACL set the permissions on an existing bucket using Canned ACL's
func (a lowLevelAPI) putBucketACL(bucket, acl string) error {
req, err := a.putBucketACLRequest(bucket, acl)
if err != nil {
Expand Down Expand Up @@ -246,7 +249,7 @@ func (a lowLevelAPI) getBucketLocation(bucket string) (string, error) {
return locationConstraint, nil
}

// listObjectsRequest wrapper creates a new ListObjects request
// listObjectsRequest wrapper creates a new listObjects request
func (a lowLevelAPI) listObjectsRequest(bucket, marker, prefix, delimiter string, maxkeys int) (*request, error) {
// resourceQuery - get resources properly escaped and lined up before using them in http request
resourceQuery := func() (*string, error) {
Expand Down Expand Up @@ -330,6 +333,7 @@ func (a lowLevelAPI) listObjects(bucket, marker, prefix, delimiter string, maxke
return listBucketResult, nil
}

// headBucketRequest wrapper creates a new headBucket request
func (a lowLevelAPI) headBucketRequest(bucket string) (*request, error) {
op := &operation{
HTTPServer: a.config.Endpoint,
Expand All @@ -339,7 +343,7 @@ func (a lowLevelAPI) headBucketRequest(bucket string) (*request, error) {
return newRequest(op, a.config, nil)
}

// headBucket - useful to determine if a bucket exists and you have permission to access it.
// headBucket useful to determine if a bucket exists and you have permission to access it.
func (a lowLevelAPI) headBucket(bucket string) error {
if err := invalidBucketToError(bucket); err != nil {
return err
Expand Down Expand Up @@ -379,7 +383,7 @@ func (a lowLevelAPI) headBucket(bucket string) error {
return nil
}

// deleteBucketRequest wrapper creates a new DeleteBucket request
// deleteBucketRequest wrapper creates a new deleteBucket request
func (a lowLevelAPI) deleteBucketRequest(bucket string) (*request, error) {
op := &operation{
HTTPServer: a.config.Endpoint,
Expand All @@ -389,7 +393,8 @@ func (a lowLevelAPI) deleteBucketRequest(bucket string) (*request, error) {
return newRequest(op, a.config, nil)
}

// deleteBucket - deletes the bucket named in the URI
// deleteBucket deletes the bucket named in the URI
//
// NOTE: -
// All objects (including all object versions and delete markers)
// in the bucket must be deleted before successfully attempting this request
Expand Down Expand Up @@ -459,8 +464,7 @@ func (a lowLevelAPI) putObjectRequest(bucket, object, contentType string, md5Sum
}

// putObject - add an object to a bucket
//
// You must have WRITE permissions on a bucket to add an object to it.
// NOTE: You must have WRITE permissions on a bucket to add an object to it.
func (a lowLevelAPI) putObject(bucket, object, contentType string, md5SumBytes []byte, size int64, body io.ReadSeeker) (ObjectStat, error) {
req, err := a.putObjectRequest(bucket, object, contentType, md5SumBytes, size, body)
if err != nil {
Expand All @@ -481,7 +485,7 @@ func (a lowLevelAPI) putObject(bucket, object, contentType string, md5SumBytes [
return metadata, nil
}

// getObjectRequest wrapper creates a new GetObject request
// getObjectRequest wrapper creates a new getObject request
func (a lowLevelAPI) getObjectRequest(bucket, object string, offset, length int64) (*request, error) {
encodedObject, err := urlEncodeName(object)
if err != nil {
Expand All @@ -507,12 +511,13 @@ func (a lowLevelAPI) getObjectRequest(bucket, object string, offset, length int6
return r, nil
}

// getPartialObject - retrieve object from Object Storage
// getObject - retrieve object from Object Storage
//
// Additionally this function also takes range arguments to download the specified
// range bytes of an object. Setting offset and length = 0 will download the full object.
//
// Additionally it also takes range arguments to download the specified range bytes of an object.
// Setting offset and length = 0 will download the full object.
// For more information about the HTTP Range header, go to http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35.
func (a lowLevelAPI) getPartialObject(bucket, object string, offset, length int64) (io.ReadCloser, ObjectStat, error) {
func (a lowLevelAPI) getObject(bucket, object string, offset, length int64) (io.ReadCloser, ObjectStat, error) {
if err := invalidArgumentToError(object); err != nil {
return nil, ObjectStat{}, err
}
Expand Down Expand Up @@ -620,7 +625,7 @@ func (a lowLevelAPI) headObjectRequest(bucket, object string) (*request, error)
return newRequest(op, a.config, nil)
}

// headObject - retrieves metadata from an object without returning the object itself
// headObject retrieves metadata from an object without returning the object itself
func (a lowLevelAPI) headObject(bucket, object string) (ObjectStat, error) {
if err := invalidBucketToError(bucket); err != nil {
return ObjectStat{}, err
Expand Down Expand Up @@ -681,7 +686,7 @@ func (a lowLevelAPI) headObject(bucket, object string) (ObjectStat, error) {

/// Service Operations

// listBucketRequest wrapper creates a new ListBuckets request
// listBucketRequest wrapper creates a new listBuckets request
func (a lowLevelAPI) listBucketsRequest() (*request, error) {
op := &operation{
HTTPServer: a.config.Endpoint,
Expand Down
32 changes: 20 additions & 12 deletions lowlevelcommon.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,12 @@ import (
"unicode/utf8"
)

// decoder provides a unified decoding method interface
type decoder interface {
Decode(v interface{}) error
}

// acceptTypeDecoder provide decoded value in given acceptType
func acceptTypeDecoder(body io.Reader, acceptType string, v interface{}) error {
var decoder decoder
switch {
Expand All @@ -47,22 +49,28 @@ func acceptTypeDecoder(body io.Reader, acceptType string, v interface{}) error {
return decoder.Decode(v)
}

// urlEncodedName- encode the strings from UTF-8 byte representations to HTML hex escape sequences
func urlEncodeName(objectName string) (string, error) {
// urlEncodedName encode the strings from UTF-8 byte representations to HTML hex escape sequences
//
// This is necessary since regular url.Parse() and url.Encode() functions do not support UTF-8
// non english characters cannot be parsed due to the nature in which url.Encode() is written
//
// This function on the other hand is a direct replacement for url.Encode() technique to support
// pretty much every UTF-8 character.
func urlEncodeName(name string) (string, error) {
// if object matches reserved string, no need to encode them
reservedNames := regexp.MustCompile("^[a-zA-Z0-9-_.~/]+$")
if reservedNames.MatchString(objectName) {
return objectName, nil
if reservedNames.MatchString(name) {
return name, nil
}
var encodedObjectName string
for _, s := range objectName {
var encodedName string
for _, s := range name {
if 'A' <= s && s <= 'Z' || 'a' <= s && s <= 'z' || '0' <= s && s <= '9' { // §2.3 Unreserved characters (mark)
encodedObjectName = encodedObjectName + string(s)
encodedName = encodedName + string(s)
continue
}
switch s {
case '-', '_', '.', '~', '/': // §2.3 Unreserved characters (mark)
encodedObjectName = encodedObjectName + string(s)
encodedName = encodedName + string(s)
continue
default:
len := utf8.RuneLen(s)
Expand All @@ -73,14 +81,14 @@ func urlEncodeName(objectName string) (string, error) {
utf8.EncodeRune(u, s)
for _, r := range u {
hex := hex.EncodeToString([]byte{r})
encodedObjectName = encodedObjectName + "%" + strings.ToUpper(hex)
encodedName = encodedName + "%" + strings.ToUpper(hex)
}
}
}
return encodedObjectName, nil
return encodedName, nil
}

// sum256Reader calculate sha256 sum for an input reader
// sum256Reader calculate sha256 sum for an input read seeker
func sum256Reader(reader io.ReadSeeker) ([]byte, error) {
h := sha256.New()
var err error
Expand Down Expand Up @@ -117,7 +125,7 @@ func sumHMAC(key []byte, data []byte) []byte {
return hash.Sum(nil)
}

// sumMD5Reader calculate md5 for an input reader of a given size
// sumMD5Reader calculate md5 for an input read seeker of a given size
func sumMD5Reader(body io.ReadSeeker, size int64) ([]byte, error) {
hasher := md5.New()
_, err := io.CopyN(hasher, body, size)
Expand Down

0 comments on commit 115db49

Please sign in to comment.