Merge pull request #2815 from bainsy88/issue_2814

Add code to handle pagination of parts. Fixes max layer size of 10GB bug

Signed-off-by: David van der Spek <vanderspek.david@gmail.com>

registry/storage/driver/s3-aws/s3.go

@@ -549,9 +549,9 @@ func (d *driver) Reader(ctx context.Context, path string, offset int64) (io.Read
 
 // Writer returns a FileWriter which will store the content written to it
 // at the location designated by "path" after the call to Commit.
-func (d *driver) Writer(ctx context.Context, path string, append bool) (storagedriver.FileWriter, error) {
+func (d *driver) Writer(ctx context.Context, path string, appendParam bool) (storagedriver.FileWriter, error) {
 	key := d.s3Path(path)
-	if !append {
+	if !appendParam {
 		// TODO (brianbland): cancel other uploads at this path
 		resp, err := d.S3.CreateMultipartUpload(&s3.CreateMultipartUploadInput{
 			Bucket:               aws.String(d.Bucket),
@@ -574,7 +574,7 @@ func (d *driver) Writer(ctx context.Context, path string, append bool) (storaged
 	if err != nil {
 		return nil, parseError(path, err)
 	}
-
+	var allParts []*s3.Part
 	for _, multi := range resp.Uploads {
 		if key != *multi.Key {
 			continue
@@ -587,11 +587,20 @@ func (d *driver) Writer(ctx context.Context, path string, append bool) (storaged
 		if err != nil {
 			return nil, parseError(path, err)
 		}
-		var multiSize int64
-		for _, part := range resp.Parts {
-			multiSize += *part.Size
+		allParts = append(allParts, resp.Parts...)
+		for *resp.IsTruncated {
+			resp, err = d.S3.ListParts(&s3.ListPartsInput{
+				Bucket:           aws.String(d.Bucket),
+				Key:              aws.String(key),
+				UploadId:         multi.UploadId,
+				PartNumberMarker: resp.NextPartNumberMarker,
+			})
+			if err != nil {
+				return nil, parseError(path, err)
+			}
+			allParts = append(allParts, resp.Parts...)
 		}
-		return d.newWriter(key, *multi.UploadId, resp.Parts), nil
+		return d.newWriter(key, *multi.UploadId, allParts), nil
 	}
 	return nil, storagedriver.PathNotFoundError{Path: path}
 }