• Object Storage Service

obs
  1. Help Center
  2. Object Storage Service
  3. Developer Guide (Python SDK)
  4. Object Upload
  5. Performing a Multipart Copy

Performing a Multipart Copy

As a special case of multipart upload, multipart copy implements multipart upload by copying the whole or partial object in a bucket. You can call ObsClient.copyPart to copy parts. Sample code is as follows:

# Import the module.
from com.obs.client.obs_client import ObsClient

destBucketName = 'bucketname'
destObjectKey = 'objectkey'
sourceBucketName = 'sourcebucketname'
sourceObjectKey = 'sourceobjectkey'

import platform, os, threading, multiprocessing
IS_WINDOWS = platform.system() == 'Windows' or os.name == 'nt'

def doCopyPart(lock, partETags, bucketName, objectKey, partNumber, uploadId, copySource, copySourceRange):
    if IS_WINDOWS:
        global obsClient
    else:
        obsClient = ObsClient(
            access_key_id='*** Provide your Access Key ***',    
            secret_access_key='*** Provide your Secret Key ***',    
            server='yourdomainname'
        )
    resp = obsClient.copyPart(bucketName=bucketName, objectKey=objectKey, partNumber=partNumber, uploadId=uploadId, copySource=copySource, copySourceRange=copySourceRange)
    if resp.status < 300:
        lock.acquire()
        try:
            partETags[partNumber] = resp.body.etag
            print('\tPart#' + str(partNumber) + ' done\n')
        finally:
            lock.release()

if __name__ == '__main__':
    # Create an instance of ObsClient.
    obsClient = ObsClient(
        access_key_id='*** Provide your Access Key ***',    
        secret_access_key='*** Provide your Secret Key ***',    
        server='yourdomainname'
    )
    # Initialize a multipart upload.
    resp = obsClient.initiateMultipartUpload(destBucketName, destObjectKey)
    uploadId = resp.body.uploadId
    
    # Set the part size to 100 MB.
    partSize = 100 * 1024 * 1024
    # Obtain information about the large object.
    resp = obsClient.getObjectMetadata(sourceBucketName, sourceObjectKey)
    objectSize = resp.body.contentLength
    
    # Calculate the number of parts to be copied.
    partCount = int(objectSize / partSize) if (objectSize % partSize == 0) else int(objectSize / partSize) + 1
    
    lock = threading.Lock() if IS_WINDOWS else multiprocessing.Lock()
    proc = threading.Thread if IS_WINDOWS else multiprocessing.Process
    
    partETags = dict() if IS_WINDOWS else multiprocessing.Manager().dict()
    
    processes = []
    
    #  Start copying parts concurrently.
    for i in range(partCount):
        # Start position for copying parts
        rangeStart = i * partSize
        # End position for copying parts
        rangeEnd = objectSize - 1 if (i + 1 == partCount) else rangeStart + partSize - 1
        # Part number
        partNumber = i + 1
        p = proc(target=doCopyPart, args=(lock, partETags, destBucketName, destObjectKey, partNumber, uploadId, sourceBucketName + '/' + sourceObjectKey, str(rangeStart) + '-' + str(rangeEnd)))
        p.daemon = True
        processes.append(p)
    
    for p in processes:
        p.start()
    
    for p in processes:
        p.join()
    
    # Wait until the copy is complete.
    from com.obs.models.complete_multipart_upload_request import CompletePart, CompleteMultipartUploadRequest
    
    partETags = sorted(partETags.items(), key= lambda d : d[0])
    
    parts = []
    for key, value in partETags:
        parts.append(CompletePart(partNum=key, etag=value))
    
    # Combine parts.
    resp = obsClient.completeMultipartUpload(destBucketName, destObjectKey, uploadId, CompleteMultipartUploadRequest(parts))
    if resp.status < 300:
        print('requestId:', resp.requestId)
    else:
        print('errorCode:', resp.errorCode)
        print('errorMessage:', resp.errorMessage)