public void DeleteBookData(string key)
{
var matchingFilesResponse = _amazonS3.ListObjects(new ListObjectsRequest()
{
BucketName = _bucketName,
Prefix = key
});
if (matchingFilesResponse.S3Objects.Count == 0)
{
return;
}
var deleteObjectsRequest = new DeleteObjectsRequest()
{
BucketName = UnitTestBucketName,
Objects = matchingFilesResponse.S3Objects.Select(s3Object => new KeyVersion()
{
Key = s3Object.Key
}).ToList()
};
var response = _amazonS3.DeleteObjects(deleteObjectsRequest);
Debug.Assert(response.DeleteErrors.Count == 0);
}
private Amazon.S3.Model.DeleteObjectsResponse CallAWSServiceOperation(IAmazonS3 client, Amazon.S3.Model.DeleteObjectsRequest request)
{
Utils.Common.WriteVerboseEndpointMessage(this, client.Config, "Amazon S3", "DeleteObject");
try
{
#if DESKTOP
return(client.DeleteObjects(request));
#elif CORECLR
return(client.DeleteObjectsAsync(request).GetAwaiter().GetResult());
#else
#error "Unknown build edition"
#endif
}
catch (AmazonServiceException exc)
{
var webException = exc.InnerException as System.Net.WebException;
if (webException != null)
{
throw new Exception(Utils.Common.FormatNameResolutionFailureMessage(client.Config, webException.Message), webException);
}
throw;
}
}
public void DeleteImages(int orgId, int productId)
{
var listRequest = new ListObjectsRequest
{
BucketName = imagesBucket,
Prefix = string.Format("{0}/{1}", orgId, productId)
};
using (_client = new AmazonS3Client())
{
var keys = _client.ListObjects(listRequest);
var keyVersions = new List <KeyVersion>();
keys.S3Objects.ForEach(i => keyVersions.Add(
new KeyVersion
{
Key = i.Key,
VersionId = null
}
));
var request = new DeleteObjectsRequest
{
BucketName = imagesBucket,
Objects = keyVersions
};
_client.DeleteObjects(request);
}
}
private static void DeleteBucketObjectsIncludingLocked(IAmazonS3 s3Client, string bucketName)
{
var listVersionsRequest = new ListVersionsRequest
{
BucketName = bucketName
};
ListVersionsResponse listVersionsResponse;
// Iterate through the objects in the bucket and delete them.
do
{
// List all the versions of all the objects in the bucket.
listVersionsResponse = s3Client.ListVersions(listVersionsRequest);
if (listVersionsResponse.Versions.Count == 0)
{
// If the bucket has no objects break the loop.
break;
}
var keyVersionList = new List <KeyVersion>(listVersionsResponse.Versions.Count);
for (int index = 0; index < listVersionsResponse.Versions.Count; index++)
{
keyVersionList.Add(new KeyVersion
{
Key = listVersionsResponse.Versions[index].Key,
VersionId = listVersionsResponse.Versions[index].VersionId
});
}
try
{
// Delete the current set of objects.
var deleteObjectsResponse = s3Client.DeleteObjects(new DeleteObjectsRequest
{
BucketName = bucketName,
Objects = keyVersionList,
BypassGovernanceRetention = true
});
}
catch
{
}
// Set the markers to get next set of objects from the bucket.
listVersionsRequest.KeyMarker = listVersionsResponse.NextKeyMarker;
listVersionsRequest.VersionIdMarker = listVersionsResponse.NextVersionIdMarker;
}
// Continue listing objects and deleting them until the bucket is empty.
while (listVersionsResponse.IsTruncated);
}
public static void ClenupBucket(IAmazonS3 client, string folderName)
{
try
{
var deleteObjectsRequest = new DeleteObjectsRequest();
var listObjectsRequest = new ListObjectsRequest
{
BucketName = Settings.Current.Bucket,
Prefix = folderName
};
var response = client.ListObjects(listObjectsRequest);
var containsObjectsToDelete = false;
if (response.S3Objects.Count > 0)
{
foreach (var entry in response.S3Objects)
{
if (string.IsNullOrEmpty(entry.Key))
{
continue;
}
// remove only chunk data
if (!entry.Key.Contains("sets"))
{
continue;
}
containsObjectsToDelete = true;
deleteObjectsRequest.AddKey(entry.Key);
}
if (containsObjectsToDelete)
{
deleteObjectsRequest.BucketName = Settings.Current.Bucket;
client.DeleteObjects(deleteObjectsRequest);
}
}
}
catch (AmazonS3Exception ex)
{
//status wasn't not found, so throw the exception
if (ex.StatusCode != System.Net.HttpStatusCode.NotFound)
{
throw;
}
}
}
/// <summary>
/// Deletes all objects in a bucket.
/// Based on DeleteS3BucketWithObjects, but
/// without deleting the bucket at the end.
/// </summary>
/// <param name="client">S3 Client</param>
/// <param name="bucketName">Bucket whose objects to delete</param>
public static void DeleteObjects(IAmazonS3 client, string bucketName)
{
var listVersionsRequest = new ListVersionsRequest
{
BucketName = bucketName
};
ListVersionsResponse listVersionsResponse;
do
{
// List all the versions of all the objects in the bucket.
listVersionsResponse = client.ListVersions(listVersionsRequest);
if (listVersionsResponse.Versions.Count == 0)
{
// If the bucket has no objects we're finished
return;
}
var keyVersionList = new List <KeyVersion>(listVersionsResponse.Versions.Count);
for (int index = 0; index < listVersionsResponse.Versions.Count; index++)
{
keyVersionList.Add(new KeyVersion
{
Key = listVersionsResponse.Versions[index].Key,
VersionId = listVersionsResponse.Versions[index].VersionId
});
}
// Delete the current set of objects.
client.DeleteObjects(new DeleteObjectsRequest
{
BucketName = bucketName,
Objects = keyVersionList
});
// Set the markers to get next set of objects from the bucket.
listVersionsRequest.KeyMarker = listVersionsResponse.NextKeyMarker;
listVersionsRequest.VersionIdMarker = listVersionsResponse.NextVersionIdMarker;
}
// Continue listing objects and deleting them until the bucket is empty.
while (listVersionsResponse.IsTruncated);
}
private static void MultiObjectDelete(List <KeyVersion> keys)
{
// a. multi-object delete by specifying the key names and version IDs.
DeleteObjectsRequest multiObjectDeleteRequest = new DeleteObjectsRequest
{
BucketName = "takatanet-test-bucket",
Objects = keys // This includes the object keys and null version IDs.
};
multiObjectDeleteRequest.AddKey("AWSSDKcopy2.dll", null);
try
{
DeleteObjectsResponse response = s3Client.DeleteObjects(multiObjectDeleteRequest);
//Console.WriteLine("Successfully deleted all the {0} items", response.DeletedObjects.Count);
}
catch (DeleteObjectsException e)
{
//PrintDeletionReport(e);
}
}
/// <summary>
/// Delete a folder
/// </summary>
/// <param name="prefix">prefix</param>
public void DeleteFolder(string prefix)
{
// Get all object with specified prefix
var listRequest = new ListObjectsRequest()
{
BucketName = _bucketName,
Prefix = prefix
};
var deleteRequest = new DeleteObjectsRequest
{
BucketName = _bucketName
};
do
{
var listResponse = _client.ListObjects(listRequest);
// Add all object with specified prefix to delete request.
foreach (var entry in listResponse.S3Objects)
{
deleteRequest.AddKey(entry.Key);
}
if (listResponse.IsTruncated)
{
listRequest.Marker = listResponse.NextMarker;
}
else
{
listRequest = null;
}
}while (listRequest != null);
// Delete all the object with specified prefix.
if (deleteRequest.Objects.Count > 0)
{
var deleteResponse = _client.DeleteObjects(deleteRequest);
deleteResponse.DisposeIfDisposable();
}
}
public void DeleteFolder(string folderPath)
{
try
{
folderPath = CorrectFolderPath(folderPath);
// get entire contents of source folder
ListObjectsRequest listRequest = new ListObjectsRequest()
{
BucketName = _bucketName,
Prefix = folderPath
};
ListObjectsResponse listResponse = _awsS3Client.ListObjects(listRequest);
List <KeyVersion> objectKeys = new List <KeyVersion>();
foreach (var s3Object in listResponse.S3Objects)
{
objectKeys.Add(new KeyVersion {
Key = s3Object.Key
});
}
// empty folders do not really exist in S3 so don't try to delete an empty folder
if (objectKeys.Count > 0)
{
DeleteObjectsRequest request = new DeleteObjectsRequest
{
Objects = objectKeys,
BucketName = _bucketName
};
DeleteObjectsResponse response = _awsS3Client.DeleteObjects(request);
}
}
catch (Exception ex)
{
throw new ApplicationException("S3 DeleteFolder - unexpected exception.", ex);
}
}
/// <summary>
/// Deletes an S3 bucket which contains objects.
/// An S3 bucket which contains objects cannot be deleted until all the objects
/// in it are deleted. The function deletes all the objects in the specified
/// bucket and then deletes the bucket itself.
/// </summary>
/// <param name="bucketName">The bucket to be deleted.</param>
/// <param name="s3Client">The Amazon S3 Client to use for S3 specific operations.</param>
/// <param name="deleteOptions">Options to control the behavior of the delete operation.</param>
/// <param name="updateCallback">The callback which is used to send updates about the delete operation.</param>
/// <param name="asyncCancelableResult">An IAsyncCancelableResult that can be used to poll or wait for results, or both;
/// this value is also needed when invoking EndDeleteS3BucketWithObjects. IAsyncCancelableResult can also
/// be used to cancel the operation while it's in progress.</param>
private static void DeleteS3BucketWithObjectsInternal(IAmazonS3 s3Client, string bucketName,
S3DeleteBucketWithObjectsOptions deleteOptions, Action <S3DeleteBucketWithObjectsUpdate> updateCallback,
AsyncCancelableResult asyncCancelableResult)
{
// Validations.
if (s3Client == null)
{
throw new ArgumentNullException("s3Client", "The s3Client cannot be null!");
}
if (string.IsNullOrEmpty(bucketName))
{
throw new ArgumentNullException("bucketName", "The bucketName cannot be null or empty string!");
}
var listVersionsRequest = new ListVersionsRequest
{
BucketName = bucketName
};
ListVersionsResponse listVersionsResponse;
// Iterate through the objects in the bucket and delete them.
do
{
// Check if the operation has been canceled.
if (asyncCancelableResult.IsCancelRequested)
{
// Signal that the operation is canceled.
asyncCancelableResult.SignalWaitHandleOnCanceled();
return;
}
// List all the versions of all the objects in the bucket.
listVersionsResponse = s3Client.ListVersions(listVersionsRequest);
if (listVersionsResponse.Versions.Count == 0)
{
// If the bucket has no objects break the loop.
break;
}
var keyVersionList = new List <KeyVersion>(listVersionsResponse.Versions.Count);
for (int index = 0; index < listVersionsResponse.Versions.Count; index++)
{
keyVersionList.Add(new KeyVersion
{
Key = listVersionsResponse.Versions[index].Key,
VersionId = listVersionsResponse.Versions[index].VersionId
});
}
try
{
// Delete the current set of objects.
var deleteObjectsResponse = s3Client.DeleteObjects(new DeleteObjectsRequest
{
BucketName = bucketName,
Objects = keyVersionList,
Quiet = deleteOptions.QuietMode
});
if (!deleteOptions.QuietMode)
{
// If quiet mode is not set, update the client with list of deleted objects.
InvokeS3DeleteBucketWithObjectsUpdateCallback(
updateCallback,
new S3DeleteBucketWithObjectsUpdate
{
DeletedObjects = deleteObjectsResponse.DeletedObjects
}
);
}
}
catch (DeleteObjectsException deleteObjectsException)
{
if (deleteOptions.ContinueOnError)
{
// Continue the delete operation if an error was encountered.
// Update the client with the list of objects that were deleted and the
// list of objects on which the delete failed.
InvokeS3DeleteBucketWithObjectsUpdateCallback(
updateCallback,
new S3DeleteBucketWithObjectsUpdate
{
DeletedObjects = deleteObjectsException.Response.DeletedObjects,
DeleteErrors = deleteObjectsException.Response.DeleteErrors
}
);
}
else
{
// Re-throw the exception if an error was encountered.
throw;
}
}
// Set the markers to get next set of objects from the bucket.
listVersionsRequest.KeyMarker = listVersionsResponse.NextKeyMarker;
listVersionsRequest.VersionIdMarker = listVersionsResponse.NextVersionIdMarker;
}
// Continue listing objects and deleting them until the bucket is empty.
while (listVersionsResponse.IsTruncated);
// Bucket is empty, delete the bucket.
s3Client.DeleteBucket(new DeleteBucketRequest
{
BucketName = bucketName
});
// Signal that the operation is completed.
asyncCancelableResult.SignalWaitHandleOnCompleted();
}
/// <summary>
/// Deletes the artifacts associated with an import task using the bucket name
/// and key prefix to the artifacts in Amazon S3. No check is performed to
/// determine whether the associated conversion task is in progress.
/// </summary>
/// <param name="s3Client">
/// An Amazon S3 client for the operation to use. This should have been constructed
/// using credentials that have access to the bucket containing the image file
/// artifacts and be scoped to the region containing the bucket.
/// </param>
/// <param name="bucketName">The name of the bucket containing the artifacts</param>
/// <param name="keyPrefix">The common key prefix of the artifacts</param>
/// <param name="progressCallback">Optional progress callback</param>
public static void DeleteImageArtifacts(IAmazonS3 s3Client,
string bucketName,
string keyPrefix,
CleanupProgressCallback progressCallback)
{
// build the full collection of keys to be deleted so that any progress
// indicator managed by the caller is accurate
SendProgressNotification(progressCallback, "Collating keys to image file artifacts", null);
var artifactKeys = new List <string>();
string marker = null;
do
{
var listResponse = s3Client.ListObjects(new ListObjectsRequest
{
BucketName = bucketName,
Prefix = keyPrefix,
Marker = marker
});
artifactKeys.AddRange(listResponse.S3Objects.Select(o => o.Key));
marker = listResponse.NextMarker;
} while (!string.IsNullOrEmpty(marker));
if (artifactKeys.Count == 0)
{
var msg = string.Format(CultureInfo.InvariantCulture,
"Found no image file artifacts with key prefix '{0}' to delete in bucket '{1}'.",
keyPrefix,
bucketName);
SendProgressNotification(progressCallback, msg, null);
return;
}
var deletionMsg = string.Format(CultureInfo.InvariantCulture,
"Deleting {0} image file artifacts (key prefix '{1}', bucket '{2}').",
artifactKeys.Count,
keyPrefix,
bucketName);
SendProgressNotification(progressCallback, deletionMsg, 0);
var index = 0;
var processed = 0;
do
{
var request = new DeleteObjectsRequest {
BucketName = bucketName
};
while (request.Objects.Count < 1000 && index < artifactKeys.Count)
{
request.Objects.Add(new KeyVersion {
Key = artifactKeys[index++]
});
}
s3Client.DeleteObjects(request);
processed += request.Objects.Count;
SendProgressNotification(progressCallback, deletionMsg, processed / artifactKeys.Count * 100);
} while (processed < artifactKeys.Count);
}
/// <summary>
/// Deletes an S3 bucket which contains objects.
/// An S3 bucket which contains objects cannot be deleted until all the objects
/// in it are deleted. The function deletes all the objects in the specified
/// bucket and then deletes the bucket itself.
/// </summary>
/// <param name="bucketName">The bucket to be deleted.</param>
/// <param name="s3Client">The Amazon S3 Client to use for S3 specific operations.</param>
/// <param name="deleteOptions">Options to control the behavior of the delete operation.</param>
/// <param name="updateCallback">The callback which is used to send updates about the delete operation.</param>
/// <param name="asyncCancelableResult">An IAsyncCancelableResult that can be used to poll or wait for results, or both;
/// this value is also needed when invoking EndDeleteS3BucketWithObjects. IAsyncCancelableResult can also
/// be used to cancel the operation while it's in progress.</param>
private static void DeleteS3BucketWithObjectsInternal(IAmazonS3 s3Client, string bucketName,
S3DeleteBucketWithObjectsOptions deleteOptions, Action<S3DeleteBucketWithObjectsUpdate> updateCallback,
AsyncCancelableResult asyncCancelableResult)
{
// Validations.
if (s3Client == null)
{
throw new ArgumentNullException("s3Client", "The s3Client cannot be null!");
}
if (string.IsNullOrEmpty(bucketName))
{
throw new ArgumentNullException("bucketName", "The bucketName cannot be null or empty string!");
}
var listVersionsRequest = new ListVersionsRequest
{
BucketName = bucketName
};
ListVersionsResponse listVersionsResponse;
// Iterate through the objects in the bucket and delete them.
do
{
// Check if the operation has been canceled.
if (asyncCancelableResult.IsCancelRequested)
{
// Signal that the operation is canceled.
asyncCancelableResult.SignalWaitHandleOnCanceled();
return;
}
// List all the versions of all the objects in the bucket.
listVersionsResponse = s3Client.ListVersions(listVersionsRequest);
if (listVersionsResponse.Versions.Count == 0)
{
// If the bucket has no objects break the loop.
break;
}
var keyVersionList = new List<KeyVersion>(listVersionsResponse.Versions.Count);
for (int index = 0; index < listVersionsResponse.Versions.Count; index++)
{
keyVersionList.Add(new KeyVersion
{
Key = listVersionsResponse.Versions[index].Key,
VersionId = listVersionsResponse.Versions[index].VersionId
});
}
try
{
// Delete the current set of objects.
var deleteObjectsResponse =s3Client.DeleteObjects(new DeleteObjectsRequest
{
BucketName = bucketName,
Objects = keyVersionList,
Quiet = deleteOptions.QuietMode
});
if (!deleteOptions.QuietMode)
{
// If quiet mode is not set, update the client with list of deleted objects.
InvokeS3DeleteBucketWithObjectsUpdateCallback(
updateCallback,
new S3DeleteBucketWithObjectsUpdate
{
DeletedObjects = deleteObjectsResponse.DeletedObjects
}
);
}
}
catch (DeleteObjectsException deleteObjectsException)
{
if (deleteOptions.ContinueOnError)
{
// Continue the delete operation if an error was encountered.
// Update the client with the list of objects that were deleted and the
// list of objects on which the delete failed.
InvokeS3DeleteBucketWithObjectsUpdateCallback(
updateCallback,
new S3DeleteBucketWithObjectsUpdate
{
DeletedObjects = deleteObjectsException.Response.DeletedObjects,
DeleteErrors = deleteObjectsException.Response.DeleteErrors
}
);
}
else
{
// Re-throw the exception if an error was encountered.
throw;
}
}
// Set the markers to get next set of objects from the bucket.
listVersionsRequest.KeyMarker = listVersionsResponse.NextKeyMarker;
listVersionsRequest.VersionIdMarker = listVersionsResponse.NextVersionIdMarker;
}
// Continue listing objects and deleting them until the bucket is empty.
while (listVersionsResponse.IsTruncated);
const int maxRetries = 10;
for (int retries = 1; retries <= maxRetries; retries++)
{
try
{
// Bucket is empty, delete the bucket.
s3Client.DeleteBucket(new DeleteBucketRequest
{
BucketName = bucketName
});
break;
}
catch (AmazonS3Exception e)
{
if (e.StatusCode != HttpStatusCode.Conflict || retries == maxRetries)
throw;
else
DefaultRetryPolicy.WaitBeforeRetry(retries, 5000);
}
}
// Signal that the operation is completed.
asyncCancelableResult.SignalWaitHandleOnCompleted();
}
/// <summary>
/// Deletes the artifacts associated with an import task using the bucket name
/// and key prefix to the artifacts in Amazon S3. No check is performed to
/// determine whether the associated conversion task is in progress.
/// </summary>
/// <param name="s3Client">
/// An Amazon S3 client for the operation to use. This should have been constructed
/// using credentials that have access to the bucket containing the image file
/// artifacts and be scoped to the region containing the bucket.
/// </param>
/// <param name="bucketName">The name of the bucket containing the artifacts</param>
/// <param name="keyPrefix">The common key prefix of the artifacts</param>
/// <param name="progressCallback">Optional progress callback</param>
public static void DeleteImageArtifacts(IAmazonS3 s3Client,
string bucketName,
string keyPrefix,
CleanupProgressCallback progressCallback)
{
// build the full collection of keys to be deleted so that any progress
// indicator managed by the caller is accurate
SendProgressNotification(progressCallback, "Collating keys to image file artifacts", null);
var artifactKeys = new List<string>();
string marker = null;
do
{
var listResponse = s3Client.ListObjects(new ListObjectsRequest
{
BucketName = bucketName,
Prefix = keyPrefix,
Marker = marker
});
artifactKeys.AddRange(listResponse.S3Objects.Select(o => o.Key));
marker = listResponse.NextMarker;
} while (!string.IsNullOrEmpty(marker));
if (artifactKeys.Count == 0)
{
var msg = string.Format(CultureInfo.InvariantCulture,
"Found no image file artifacts with key prefix '{0}' to delete in bucket '{1}'.",
keyPrefix,
bucketName);
SendProgressNotification(progressCallback, msg, null);
return;
}
var deletionMsg = string.Format(CultureInfo.InvariantCulture,
"Deleting {0} image file artifacts (key prefix '{1}', bucket '{2}').",
artifactKeys.Count,
keyPrefix,
bucketName);
SendProgressNotification(progressCallback, deletionMsg, 0);
var index = 0;
var processed = 0;
do
{
var request = new DeleteObjectsRequest { BucketName = bucketName };
while (request.Objects.Count < 1000 && index < artifactKeys.Count)
{
request.Objects.Add(new KeyVersion { Key = artifactKeys[index++] });
}
s3Client.DeleteObjects(request);
processed += request.Objects.Count;
SendProgressNotification(progressCallback, deletionMsg, processed/artifactKeys.Count * 100);
} while (processed < artifactKeys.Count);
}
