public void Execute(OperationOptions options)
{
_logger.Info($"Starting objects restore {(options.DryRun ? "(dry run)" : string.Empty)}...");
var listRequest = new ListVersionsRequest
{
BucketName = options.BucketName,
Prefix = options.Prefix,
MaxKeys = options.MaxKeys
};
var versions = new S3ObjectVersionsEnumerable(_s3Client, listRequest);
var deleteRequest = new DeleteObjectsRequest
{
BucketName = listRequest.BucketName,
Objects = versions.Where(_ => _.IsDeleteMarker).Select(dm =>
{
_logger.Info($"Adding to restore chunk: {dm.Key} - [{dm.VersionId}]");
return(new KeyVersion {
Key = dm.Key, VersionId = dm.VersionId
});
}).ToList()
};
if (!options.DryRun)
{
_s3Client.DeleteObjectsAsync(deleteRequest).Wait();
}
_logger.Info("Restoring completed!");
}
public static async Task <AmazonS3Client> SetupBlob(string container, string path)
{
var request = new ListVersionsRequest
{
BucketName = container,
Prefix = path
};
var resp = await _client.ListVersionsAsync(request).ConfigureAwait(false);
var toDelete = resp.Versions.Where(v => v.Key == path).Select(v => new KeyVersion
{
Key = v.Key,
VersionId = v.VersionId
}).ToList();
if (toDelete.Any())
{
var delRequest = new DeleteObjectsRequest
{
BucketName = container,
Objects = toDelete,
Quiet = true
};
await _client.DeleteObjectsAsync(delRequest).ConfigureAwait(false);
}
return(_client);
}
protected async Task <bool> IsSecretRevoked(string secretName)
{
var s3Client = GetS3Client();
string objectName = GetObjectKey(secretName);
ListVersionsRequest listRequest = new ListVersionsRequest {
BucketName = BucketName, Prefix = objectName
};
bool isSecretRevoked = false;
ListVersionsResponse listing;
do
{
logger.Info($"list secrets from marker {listRequest.KeyMarker}");
listRequest.MaxKeys = 1;
listing = await s3Client.ListVersionsAsync(listRequest);
if (null != listing.Versions && listing.Versions.Count > 0)
{
var summary = listing.Versions[0];
if (summary.IsDeleteMarker)
{
isSecretRevoked = true;
break;
}
}
listRequest.KeyMarker = listing.NextKeyMarker;
} while (listing.IsTruncated);
return(isSecretRevoked);
}
private IAsyncEnumerable <S3ObjectVersion> ListObjects(string prefix)
{
return(AsyncEnumerableEx.Create <S3ObjectVersion>(async y =>
{
var request = new ListVersionsRequest
{
BucketName = _bucket,
Prefix = prefix,
KeyMarker = null,
VersionIdMarker = null
};
while (request != null && !y.CancellationToken.IsCancellationRequested)
{
var resp = await _client.ListVersionsAsync(request, y.CancellationToken).ConfigureAwait(false);
foreach (var v in resp.Versions)
{
await y.YieldReturn(v).ConfigureAwait(false);
}
if (resp.IsTruncated)
{
request.KeyMarker = resp.NextKeyMarker;
request.VersionIdMarker = resp.NextVersionIdMarker;
}
else
{
request = null;
}
}
}));
}
private static void ListVersions()
{
try
{
ListVersionsRequest request = new ListVersionsRequest()
{
BucketName = bucketName,
MaxKeys = 10,
Delimiter = "delimiter",
Prefix = "prefix",
KeyMarker = "keyMarker"
};
ListVersionsResponse response = client.ListVersions(request);
Console.WriteLine("ListVersions response: {0}", response.StatusCode);
foreach (ObsObjectVersion objectVersion in response.Versions)
{
Console.WriteLine("ListVersions response Versions Key: {0}", objectVersion.ObjectKey);
Console.WriteLine("ListVersions response Versions VersionId: {0}", objectVersion.VersionId);
}
}
catch (ObsException ex)
{
Console.WriteLine("Exception errorcode: {0}, when list versions.", ex.ErrorCode);
Console.WriteLine("Exception errormessage: {0}", ex.ErrorMessage);
}
}
/// <summary>
/// Get a list of supported and available MySQL database major versions.
/// <br/>
/// The list is sorted by version family.
///
/// </summary>
/// <param name="request">The request object containing the details to send. Required.</param>
/// <param name="retryConfiguration">The retry configuration that will be used by to send this request. Optional.</param>
/// <param name="cancellationToken">The cancellation token to cancel this operation. Optional.</param>
/// <returns>A response object containing details about the completed operation</returns>
/// <example>Click <a href="https://docs.cloud.oracle.com/en-us/iaas/tools/dot-net-examples/latest/mysql/ListVersions.cs.html">here</a> to see an example of how to use ListVersions API.</example>
public async Task <ListVersionsResponse> ListVersions(ListVersionsRequest request, RetryConfiguration retryConfiguration = null, CancellationToken cancellationToken = default)
{
logger.Trace("Called listVersions");
Uri uri = new Uri(this.restClient.GetEndpoint(), System.IO.Path.Combine(basePathWithoutHost, "/versions".Trim('/')));
HttpMethod method = new HttpMethod("GET");
HttpRequestMessage requestMessage = Converter.ToHttpRequestMessage(uri, method, request);
requestMessage.Headers.Add("Accept", "application/json");
GenericRetrier retryingClient = Retrier.GetPreferredRetrier(retryConfiguration, this.retryConfiguration);
HttpResponseMessage responseMessage;
try
{
if (retryingClient != null)
{
responseMessage = await retryingClient.MakeRetryingCall(this.restClient.HttpSend, requestMessage, cancellationToken).ConfigureAwait(false);
}
else
{
responseMessage = await this.restClient.HttpSend(requestMessage).ConfigureAwait(false);
}
this.restClient.CheckHttpResponseMessage(requestMessage, responseMessage);
return(Converter.FromHttpResponseMessage <ListVersionsResponse>(responseMessage));
}
catch (Exception e)
{
logger.Error($"ListVersions failed with error: {e.Message}");
throw;
}
}
public static void Main(string[] args)
{
// create the AWS S3 client
AmazonS3Client s3 = AWSS3Factory.getS3Client();
bool moreRecords = true;
string nextMarker = string.Empty;
while (moreRecords)
{
ListVersionsRequest request = new ListVersionsRequest()
{
BucketName = AWSS3Factory.S3_BUCKET,
};
if (nextMarker.Length > 0)
{
request.KeyMarker = nextMarker;
}
ListVersionsResponse response = new ListVersionsResponse();
response = s3.ListVersions(request);
foreach (S3ObjectVersion theObject in response.Versions)
{
s3.DeleteObject(new DeleteObjectRequest()
{
BucketName = AWSS3Factory.S3_BUCKET,
Key = theObject.Key,
VersionId = theObject.VersionId
});
Console.WriteLine("Deleted {0}/{1}", AWSS3Factory.S3_BUCKET, theObject.Key);
}
Console.WriteLine(string.Format("Next Marker: {0} Version Count: {1}", response.NextKeyMarker, response.Versions.Count.ToString()));
if (response.IsTruncated)
{
nextMarker = response.NextKeyMarker;
}
else
{
moreRecords = false;
}
}
s3.DeleteBucket(new DeleteBucketRequest()
{
BucketName = AWSS3Factory.S3_BUCKET
});
// print bucket name for validation
Console.WriteLine(string.Format("Deleted bucket {0}", AWSS3Factory.S3_BUCKET));
Console.ReadLine();
}
public void Construct()
{
var request = new ListVersionsRequest("s3.amazon.com", "bucket", new ListVersionsOptions {
Prefix = "apples",
MaxKeys = 1000
});
Assert.Equal("https://s3.amazon.com/bucket?versions&prefix=apples&max-keys=1000", request.RequestUri.ToString());
}
public static void Main(string[] args)
{
// create the AWS S3 client
AmazonS3Client s3 = AWSS3Factory.getS3Client();
System.Net.ServicePointManager.ServerCertificateValidationCallback = ((sender, certificate, chain, sslPolicyErrors) => true);
GetBucketVersioningRequest gvr = new GetBucketVersioningRequest()
{
BucketName = AWSS3Factory.S3_BUCKET
};
Console.WriteLine(s3.GetBucketVersioning(gvr).VersioningConfig.Status);
bool moreRecords = true;
string nextMarker = string.Empty;
while (moreRecords)
{
ListVersionsRequest request = new ListVersionsRequest()
{
BucketName = AWSS3Factory.S3_BUCKET,
};
//if (nextMarker.Length > 0)
//request.KeyMarker = nextMarker;
request.VersionIdMarker = "1472739256446";
ListVersionsResponse response = new ListVersionsResponse();
response = s3.ListVersions(request);
foreach (S3ObjectVersion key in response.Versions)
{
Console.WriteLine(key.Key);
}
Console.WriteLine(string.Format("Next Marker: {0} Version Count: {1}", response.NextKeyMarker, response.Versions.Count.ToString()));
if (response.IsTruncated)
{
nextMarker = response.NextKeyMarker;
}
else
{
moreRecords = false;
}
}
// print out object key/value for validation
//Console.WriteLine(string.Format("Copied object {0}/{1} to {2}/{3}", AWSS3Factory.S3_BUCKET, key_source, AWSS3Factory.S3_BUCKET, key_target));
Console.ReadLine();
}
private async Task <List <Secret> > GetSecretVersions(string filter, AmazonS3Client s3Client)
{
var secrets = new List <Secret>();
var listRequest = new ListVersionsRequest {
BucketName = BucketName
};
if (!string.IsNullOrEmpty(filter))
{
listRequest.Prefix = filter;
}
var revokedSecrets = new List <string>();
string currentObjectKey = null;
ListVersionsResponse listResponse;
do
{
logger.Info($"list secrets form marker {listRequest.KeyMarker}");
listResponse = await s3Client.ListVersionsAsync(listRequest);
foreach (var summary in listResponse.Versions)
{
var objectKey = summary.Key;
if (objectKey.EndsWith(".isec"))
{
continue;
}
if (currentObjectKey?.CompareTo(objectKey) != 0)
{
currentObjectKey = objectKey;
if (summary.IsDeleteMarker)
{
revokedSecrets.Add(objectKey);
continue;
}
}
var secret = new Secret(objectKey.Substring(0, objectKey.LastIndexOf(".isec")))
{
Version = summary.VersionId,
LastModified = summary.LastModified
};
secrets.Add(secret);
}
listRequest.KeyMarker = listResponse.NextKeyMarker;
} while (listResponse.IsTruncated);
return(secrets);
}
public async Task <List <S3ObjectVersion> > GetS3ObjectswithVersions(AmazonS3Client client)
{
ListVersionsRequest request = new ListVersionsRequest()
{
BucketName = s3config.bucketName,
MaxKeys = 500
};
ListVersionsResponse response = await client.ListVersionsAsync(request);
return(response.Versions);
}
static void ReadBucketVersions()
{
string bucket = Common.InputString("Bucket:", null, false);
ListVersionsRequest request = new ListVersionsRequest();
request.BucketName = bucket;
ListVersionsResponse response = _S3Client.ListVersionsAsync(request).Result;
if (response != null)
{
Console.WriteLine("Success");
Console.WriteLine("Bucket name : " + response.Name);
Console.WriteLine("Max keys : " + response.MaxKeys);
Console.WriteLine("Is truncated : " + response.IsTruncated);
Console.Write("Versions : ");
if (response.Versions != null && response.Versions.Count > 0)
{
Console.WriteLine(response.Versions.Count);
foreach (S3ObjectVersion ver in response.Versions)
{
Console.Write("| " + ver.Key + "[" + ver.VersionId + "] ");
if (ver.IsDeleteMarker)
{
Console.WriteLine("(delete marker)");
}
else
{
Console.WriteLine(ver.Size + " bytes");
}
}
}
Console.Write("Common prefixes : ");
if (response.CommonPrefixes != null && response.CommonPrefixes.Count > 0)
{
Console.WriteLine(response.CommonPrefixes.Count);
foreach (string prefix in response.CommonPrefixes)
{
Console.WriteLine("| " + prefix);
}
}
else
{
Console.WriteLine("0");
}
}
else
{
Console.WriteLine("Failed");
}
}
static void GetObjectListWithAllVersions(IAmazonS3 client)
{
try
{
ListVersionsRequest request = new ListVersionsRequest()
{
BucketName = bucketName,
MaxKeys = 2
};
do
{
ListVersionsResponse response = client.ListVersions(request);
// Process response.
foreach (S3ObjectVersion entry in response.Versions)
{
Console.WriteLine("key = {0} size = {1}",
entry.Key, entry.Size);
}
// If response is truncated, set the marker to get the next
// set of keys.
if (response.IsTruncated)
{
request.KeyMarker = response.NextKeyMarker;
request.VersionIdMarker = response.NextVersionIdMarker;
}
else
{
request = null;
}
} while (request != null);
}
catch (AmazonS3Exception amazonS3Exception)
{
if (amazonS3Exception.ErrorCode != null &&
(amazonS3Exception.ErrorCode.Equals("InvalidAccessKeyId")
||
amazonS3Exception.ErrorCode.Equals("InvalidSecurity")))
{
Console.WriteLine("Check the provided AWS Credentials.");
Console.WriteLine(
"To sign up for service, go to http://aws.amazon.com/s3");
}
else
{
Console.WriteLine(
"Error occurred. Message:'{0}' when listing objects",
amazonS3Exception.Message);
}
}
}
private static void DeleteBucketObjectsIncludingLocked(IAmazonS3 s3Client, string bucketName)
{
var listVersionsRequest = new ListVersionsRequest
{
BucketName = bucketName
};
ListVersionsResponse listVersionsResponse;
// Iterate through the objects in the bucket and delete them.
do
{
// List all the versions of all the objects in the bucket.
listVersionsResponse = s3Client.ListVersions(listVersionsRequest);
if (listVersionsResponse.Versions.Count == 0)
{
// If the bucket has no objects break the loop.
break;
}
var keyVersionList = new List <KeyVersion>(listVersionsResponse.Versions.Count);
for (int index = 0; index < listVersionsResponse.Versions.Count; index++)
{
keyVersionList.Add(new KeyVersion
{
Key = listVersionsResponse.Versions[index].Key,
VersionId = listVersionsResponse.Versions[index].VersionId
});
}
try
{
// Delete the current set of objects.
var deleteObjectsResponse = s3Client.DeleteObjects(new DeleteObjectsRequest
{
BucketName = bucketName,
Objects = keyVersionList,
BypassGovernanceRetention = true
});
}
catch
{
}
// Set the markers to get next set of objects from the bucket.
listVersionsRequest.KeyMarker = listVersionsResponse.NextKeyMarker;
listVersionsRequest.VersionIdMarker = listVersionsResponse.NextVersionIdMarker;
}
// Continue listing objects and deleting them until the bucket is empty.
while (listVersionsResponse.IsTruncated);
}
/// <summary>Snippet for ListVersionsAsync</summary>
public async Task ListVersionsRequestObjectAsync()
{
// Snippet: ListVersionsAsync(ListVersionsRequest, CallSettings)
// Create client
VersionsClient versionsClient = await VersionsClient.CreateAsync();
// Initialize request argument(s)
ListVersionsRequest request = new ListVersionsRequest
{
Parent = "",
View = VersionView.Basic,
};
// Make the request
PagedAsyncEnumerable <ListVersionsResponse, gcav::Version> response = versionsClient.ListVersionsAsync(request);
// Iterate over all response items, lazily performing RPCs as required
await response.ForEachAsync((gcav::Version item) =>
{
// Do something with each item
Console.WriteLine(item);
});
// Or iterate over pages (of server-defined size), performing one RPC per page
await response.AsRawResponses().ForEachAsync((ListVersionsResponse page) =>
{
// Do something with each page of items
Console.WriteLine("A page of results:");
foreach (gcav::Version item in page)
{
// Do something with each item
Console.WriteLine(item);
}
});
// Or retrieve a single page of known size (unless it's the final page), performing as many RPCs as required
int pageSize = 10;
Page <gcav::Version> singlePage = await response.ReadPageAsync(pageSize);
// Do something with the page of items
Console.WriteLine($"A page of {pageSize} results (unless it's the final page):");
foreach (gcav::Version item in singlePage)
{
// Do something with each item
Console.WriteLine(item);
}
// Store the pageToken, for when the next page is required.
string nextPageToken = singlePage.NextPageToken;
// End snippet
}
/// <summary>Snippet for ListVersions</summary>
public void ListVersionsRequestObject()
{
// Snippet: ListVersions(ListVersionsRequest, CallSettings)
// Create client
VersionsClient versionsClient = VersionsClient.Create();
// Initialize request argument(s)
ListVersionsRequest request = new ListVersionsRequest
{
ParentAsFlowName = FlowName.FromProjectLocationAgentFlow("[PROJECT]", "[LOCATION]", "[AGENT]", "[FLOW]"),
};
// Make the request
PagedEnumerable <ListVersionsResponse, gcdcv::Version> response = versionsClient.ListVersions(request);
// Iterate over all response items, lazily performing RPCs as required
foreach (gcdcv::Version item in response)
{
// Do something with each item
Console.WriteLine(item);
}
// Or iterate over pages (of server-defined size), performing one RPC per page
foreach (ListVersionsResponse page in response.AsRawResponses())
{
// Do something with each page of items
Console.WriteLine("A page of results:");
foreach (gcdcv::Version item in page)
{
// Do something with each item
Console.WriteLine(item);
}
}
// Or retrieve a single page of known size (unless it's the final page), performing as many RPCs as required
int pageSize = 10;
Page <gcdcv::Version> singlePage = response.ReadPage(pageSize);
// Do something with the page of items
Console.WriteLine($"A page of {pageSize} results (unless it's the final page):");
foreach (gcdcv::Version item in singlePage)
{
// Do something with each item
Console.WriteLine(item);
}
// Store the pageToken, for when the next page is required.
string nextPageToken = singlePage.NextPageToken;
// End snippet
}
/// <summary>
/// Returns the list of restore points for a specific object
/// </summary>
public ObjectVersion[] GetVersions(string filename)
{
try
{
string fmtObject = $"{filename}.encrypted";
ListVersionsRequest request = new ListVersionsRequest()
{
BucketName = _bucketName,
MaxKeys = int.MaxValue,
Prefix = filename
};
ListVersionsResponse response = _client.ListVersionsAsync(request).Result;
// Be sure the object exists
if ((response.Versions == null) || (response.Versions.Count() == 0))
{
_logger.WriteLog(ErrorCodes.S3ObjectRestore_ObjectNotFound,
string.Format(ErrorResources.S3ObjectRestore_ObjectNotFound, filename),
Severity.Error, VerboseLevel.User);
return(null);
}
var list = response.Versions;
// Build a list of objects found
return(list.Select(item =>
new ObjectVersion()
{
Name = item.Key,
TimeCreated = item.LastModified,
StorageClass = item.StorageClass,
Size = item.Size,
VersionId = item.VersionId
})
.OrderByDescending(a => a.TimeCreated)
.ToArray());
}
catch (Exception ex)
{
_logger.WriteLog(ErrorCodes.S3ObjectRestore_GetVersionsException,
ErrorResources.S3ObjectRestore_GetVersionsException + Environment.NewLine + ex.Message,
Severity.Error, VerboseLevel.User);
return(null);
}
}
/// <summary>Snippet for ListVersions</summary>
/// <remarks>
/// This snippet has been automatically generated for illustrative purposes only.
/// It may require modifications to work in your environment.
/// </remarks>
public void ListVersionsRequestObject()
{
// Create client
VersionsClient versionsClient = VersionsClient.Create();
// Initialize request argument(s)
ListVersionsRequest request = new ListVersionsRequest
{
Parent = "",
View = VersionView.Basic,
};
// Make the request
PagedEnumerable <ListVersionsResponse, gcav::Version> response = versionsClient.ListVersions(request);
// Iterate over all response items, lazily performing RPCs as required
foreach (gcav::Version item in response)
{
// Do something with each item
Console.WriteLine(item);
}
// Or iterate over pages (of server-defined size), performing one RPC per page
foreach (ListVersionsResponse page in response.AsRawResponses())
{
// Do something with each page of items
Console.WriteLine("A page of results:");
foreach (gcav::Version item in page)
{
// Do something with each item
Console.WriteLine(item);
}
}
// Or retrieve a single page of known size (unless it's the final page), performing as many RPCs as required
int pageSize = 10;
Page <gcav::Version> singlePage = response.ReadPage(pageSize);
// Do something with the page of items
Console.WriteLine($"A page of {pageSize} results (unless it's the final page):");
foreach (gcav::Version item in singlePage)
{
// Do something with each item
Console.WriteLine(item);
}
// Store the pageToken, for when the next page is required.
string nextPageToken = singlePage.NextPageToken;
}
static async Task GetObjectListWithAllVersionsAsync()
{
try
{
ListVersionsRequest request = new ListVersionsRequest()
{
BucketName = bucketName,
// You can optionally specify key name prefix in the request
// if you want list of object versions of a specific object.
// For this example we limit response to return list of 2 versions.
MaxKeys = 2
};
do
{
ListVersionsResponse response = await s3Client.ListVersionsAsync(request);
// Process response.
foreach (S3ObjectVersion entry in response.Versions)
{
Console.WriteLine("key = {0} size = {1}",
entry.Key, entry.Size);
}
// If response is truncated, set the marker to get the next
// set of keys.
if (response.IsTruncated)
{
request.KeyMarker = response.NextKeyMarker;
request.VersionIdMarker = response.NextVersionIdMarker;
}
else
{
request = null;
}
} while (request != null);
}
catch (AmazonS3Exception e)
{
Console.WriteLine("Error encountered on server. Message:'{0}' when writing an object", e.Message);
}
catch (Exception e)
{
Console.WriteLine("Unknown encountered on server. Message:'{0}' when writing an object", e.Message);
}
}
private static bool CleanBucket(string bucket)
{
bool moreRecords = true;
string nextMarker = string.Empty;
while (moreRecords)
{
ListVersionsRequest request = new ListVersionsRequest()
{
BucketName = bucket,
};
if (nextMarker.Length > 0)
{
request.KeyMarker = nextMarker;
}
ListVersionsResponse response = new ListVersionsResponse();
response = client.ListVersions(request);
foreach (S3ObjectVersion theObject in response.Versions)
{
client.DeleteObject(new DeleteObjectRequest()
{
BucketName = bucket,
Key = theObject.Key,
VersionId = theObject.VersionId
});
}
if (response.IsTruncated)
{
nextMarker = response.NextKeyMarker;
}
else
{
moreRecords = false;
}
}
return(true);
}
/// <summary>
/// Deletes all objects in a bucket.
/// Based on DeleteS3BucketWithObjects, but
/// without deleting the bucket at the end.
/// </summary>
/// <param name="client">S3 Client</param>
/// <param name="bucketName">Bucket whose objects to delete</param>
public static void DeleteObjects(IAmazonS3 client, string bucketName)
{
var listVersionsRequest = new ListVersionsRequest
{
BucketName = bucketName
};
ListVersionsResponse listVersionsResponse;
do
{
// List all the versions of all the objects in the bucket.
listVersionsResponse = client.ListVersions(listVersionsRequest);
if (listVersionsResponse.Versions.Count == 0)
{
// If the bucket has no objects we're finished
return;
}
var keyVersionList = new List <KeyVersion>(listVersionsResponse.Versions.Count);
for (int index = 0; index < listVersionsResponse.Versions.Count; index++)
{
keyVersionList.Add(new KeyVersion
{
Key = listVersionsResponse.Versions[index].Key,
VersionId = listVersionsResponse.Versions[index].VersionId
});
}
// Delete the current set of objects.
client.DeleteObjects(new DeleteObjectsRequest
{
BucketName = bucketName,
Objects = keyVersionList
});
// Set the markers to get next set of objects from the bucket.
listVersionsRequest.KeyMarker = listVersionsResponse.NextKeyMarker;
listVersionsRequest.VersionIdMarker = listVersionsResponse.NextVersionIdMarker;
}
// Continue listing objects and deleting them until the bucket is empty.
while (listVersionsResponse.IsTruncated);
}
public IEnumerator <S3ObjectVersion> GetEnumerator()
{
_nextVersionMarker = null;
_lastResponse = null;
do
{
var request = new ListVersionsRequest {
BucketName = _bucket, Prefix = _folder, KeyMarker = _nextVersionMarker
};
_lastResponse = _client.ListVersionsAsync(request).Result;
foreach (var version in _lastResponse.Versions)
{
yield return(version);
}
_nextVersionMarker = _lastResponse.NextKeyMarker;
} while (_lastResponse.IsTruncated);
}
protected override void ProcessRecord()
{
base.ProcessRecord();
ListVersionsRequest request;
try
{
request = new ListVersionsRequest
{
CompartmentId = CompartmentId,
OpcRequestId = OpcRequestId
};
response = client.ListVersions(request).GetAwaiter().GetResult();
WriteOutput(response, response.Items, true);
FinishProcessing(response);
}
catch (Exception ex)
{
TerminatingErrorDuringExecution(ex);
}
}
private async Task <IEnumerable <S3ObjectVersion> > GetS3ObjectVersions(S3FileKey key)
{
var request = new ListVersionsRequest
{
BucketName = BucketNameConstructor.GetBucketName(key),
Prefix = key.Key
};
var result = new List <S3ObjectVersion>();
ListVersionsResponse response;
do
{
response = await AmazonS3.ListVersionsAsync(request);
result.AddRange(response.Versions);
request.VersionIdMarker = response.NextVersionIdMarker;
request.KeyMarker = response.NextKeyMarker;
} while (response.IsTruncated);
return(result);
}
public Task <ListVersionsResponse> ListVersionsAsync(ListVersionsRequest request, CancellationToken cancellationToken = default)
{
throw new NotImplementedException();
}
private async static Task DeleteS3BucketWithObjectsAsync(IAmazonS3 s3Client, string bucketName,
CancellationToken cancellationToken = new CancellationToken())
{
// Validations.
if (s3Client == null)
{
throw new ArgumentNullException("s3Client", "The s3Client cannot be null!");
}
if (string.IsNullOrEmpty(bucketName))
{
throw new ArgumentNullException("bucketName", "The bucketName cannot be null or empty string!");
}
var listVersionsRequest = new ListVersionsRequest
{
BucketName = bucketName
};
ListVersionsResponse listVersionsResponse;
string lastRequestId = null;
// Iterate through the objects in the bucket and delete them.
do
{
// Check if the operation has been canceled.
cancellationToken.ThrowIfCancellationRequested();
// List all the versions of all the objects in the bucket.
listVersionsResponse = await s3Client.ListVersionsAsync(listVersionsRequest).ConfigureAwait(false);
// Silverlight uses HTTP caching, so avoid an infinite loop by throwing an exception
if (string.Equals(lastRequestId, listVersionsResponse.ResponseMetadata.RequestId, StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException();
}
lastRequestId = listVersionsResponse.ResponseMetadata.RequestId;
if (listVersionsResponse.Versions.Count == 0)
{
// If the bucket has no objects break the loop.
break;
}
var keyVersionList = new List <KeyVersion>(listVersionsResponse.Versions.Count);
for (int index = 0; index < listVersionsResponse.Versions.Count; index++)
{
keyVersionList.Add(new KeyVersion
{
Key = listVersionsResponse.Versions[index].Key,
VersionId = listVersionsResponse.Versions[index].VersionId
});
}
try
{
// Delete the current set of objects.
var deleteObjectsResponse = await s3Client.DeleteObjectsAsync(new DeleteObjectsRequest
{
BucketName = bucketName,
Objects = keyVersionList,
Quiet = true
}).ConfigureAwait(false);
//if (!deleteOptions.QuietMode)
//{
// // If quiet mode is not set, update the client with list of deleted objects.
// InvokeS3DeleteBucketWithObjectsUpdateCallback(
// updateCallback,
// new S3DeleteBucketWithObjectsUpdate
// {
// DeletedObjects = deleteObjectsResponse.DeletedObjects
// }
// );
//}
}
catch //(DeleteObjectsException deleteObjectsException)
{
//if (deleteOptions.ContinueOnError)
//{
// // Continue the delete operation if an error was encountered.
// // Update the client with the list of objects that were deleted and the
// // list of objects on which the delete failed.
// InvokeS3DeleteBucketWithObjectsUpdateCallback(
// updateCallback,
// new S3DeleteBucketWithObjectsUpdate
// {
// DeletedObjects = deleteObjectsException.Response.DeletedObjects,
// DeleteErrors = deleteObjectsException.Response.DeleteErrors
// }
// );
//}
//else
//{
// // Re-throw the exception if an error was encountered.
// throw;
//}
throw;
}
// Set the markers to get next set of objects from the bucket.
listVersionsRequest.KeyMarker = listVersionsResponse.NextKeyMarker;
listVersionsRequest.VersionIdMarker = listVersionsResponse.NextVersionIdMarker;
}
// Continue listing objects and deleting them until the bucket is empty.
while (listVersionsResponse.IsTruncated);
const int maxRetries = 10;
for (int retries = 1; retries <= maxRetries; retries++)
{
try
{
// Bucket is empty, delete the bucket.
await s3Client.DeleteBucketAsync(new DeleteBucketRequest
{
BucketName = bucketName
}).ConfigureAwait(false);
break;
}
catch (AmazonS3Exception e)
{
if (e.StatusCode != HttpStatusCode.Conflict || retries == maxRetries)
{
throw;
}
else
{
DefaultRetryPolicy.WaitBeforeRetry(retries, 5000);
}
}
}
//// Signal that the operation is completed.
//asyncCancelableResult.SignalWaitHandleOnCompleted();
}
public S3ObjectVersionsEnumerable(IAmazonS3 s3Client, ListVersionsRequest request)
{
_s3Client = s3Client;
_request = request;
}
/// <summary>
/// Deletes an S3 bucket which contains objects.
/// An S3 bucket which contains objects cannot be deleted until all the objects
/// in it are deleted. The function deletes all the objects in the specified
/// bucket and then deletes the bucket itself.
/// </summary>
/// <param name="bucketName">The bucket to be deleted.</param>
/// <param name="s3Client">The Amazon S3 Client to use for S3 specific operations.</param>
/// <param name="deleteOptions">Options to control the behavior of the delete operation.</param>
/// <param name="updateCallback">The callback which is used to send updates about the delete operation.</param>
/// <param name="asyncCancelableResult">An IAsyncCancelableResult that can be used to poll or wait for results, or both;
/// this value is also needed when invoking EndDeleteS3BucketWithObjects. IAsyncCancelableResult can also
/// be used to cancel the operation while it's in progress.</param>
private static void DeleteS3BucketWithObjectsInternal(IAmazonS3 s3Client, string bucketName,
S3DeleteBucketWithObjectsOptions deleteOptions, Action <S3DeleteBucketWithObjectsUpdate> updateCallback,
AsyncCancelableResult asyncCancelableResult)
{
// Validations.
if (s3Client == null)
{
throw new ArgumentNullException("s3Client", "The s3Client cannot be null!");
}
if (string.IsNullOrEmpty(bucketName))
{
throw new ArgumentNullException("bucketName", "The bucketName cannot be null or empty string!");
}
var listVersionsRequest = new ListVersionsRequest
{
BucketName = bucketName
};
ListVersionsResponse listVersionsResponse;
// Iterate through the objects in the bucket and delete them.
do
{
// Check if the operation has been canceled.
if (asyncCancelableResult.IsCancelRequested)
{
// Signal that the operation is canceled.
asyncCancelableResult.SignalWaitHandleOnCanceled();
return;
}
// List all the versions of all the objects in the bucket.
listVersionsResponse = s3Client.ListVersions(listVersionsRequest);
if (listVersionsResponse.Versions.Count == 0)
{
// If the bucket has no objects break the loop.
break;
}
var keyVersionList = new List <KeyVersion>(listVersionsResponse.Versions.Count);
for (int index = 0; index < listVersionsResponse.Versions.Count; index++)
{
keyVersionList.Add(new KeyVersion
{
Key = listVersionsResponse.Versions[index].Key,
VersionId = listVersionsResponse.Versions[index].VersionId
});
}
try
{
// Delete the current set of objects.
var deleteObjectsResponse = s3Client.DeleteObjects(new DeleteObjectsRequest
{
BucketName = bucketName,
Objects = keyVersionList,
Quiet = deleteOptions.QuietMode
});
if (!deleteOptions.QuietMode)
{
// If quiet mode is not set, update the client with list of deleted objects.
InvokeS3DeleteBucketWithObjectsUpdateCallback(
updateCallback,
new S3DeleteBucketWithObjectsUpdate
{
DeletedObjects = deleteObjectsResponse.DeletedObjects
}
);
}
}
catch (DeleteObjectsException deleteObjectsException)
{
if (deleteOptions.ContinueOnError)
{
// Continue the delete operation if an error was encountered.
// Update the client with the list of objects that were deleted and the
// list of objects on which the delete failed.
InvokeS3DeleteBucketWithObjectsUpdateCallback(
updateCallback,
new S3DeleteBucketWithObjectsUpdate
{
DeletedObjects = deleteObjectsException.Response.DeletedObjects,
DeleteErrors = deleteObjectsException.Response.DeleteErrors
}
);
}
else
{
// Re-throw the exception if an error was encountered.
throw;
}
}
// Set the markers to get next set of objects from the bucket.
listVersionsRequest.KeyMarker = listVersionsResponse.NextKeyMarker;
listVersionsRequest.VersionIdMarker = listVersionsResponse.NextVersionIdMarker;
}
// Continue listing objects and deleting them until the bucket is empty.
while (listVersionsResponse.IsTruncated);
// Bucket is empty, delete the bucket.
s3Client.DeleteBucket(new DeleteBucketRequest
{
BucketName = bucketName
});
// Signal that the operation is completed.
asyncCancelableResult.SignalWaitHandleOnCompleted();
}
/// <summary>
/// 列举桶内多版本对象。
/// </summary>
/// <param name="request">列举多版本对象的请求参数。</param>
/// <returns>列举多版本对象响应结果。</returns>
public ListVersionsResponse ListVersions(ListVersionsRequest request)
{
return(this.DoRequest <ListVersionsRequest, ListVersionsResponse>(request));
}
public void ObjectSamples()
{
{
#region ListObjects Sample
// Create a client
AmazonS3Client client = new AmazonS3Client();
// List all objects
ListObjectsRequest listRequest = new ListObjectsRequest
{
BucketName = "SampleBucket",
};
ListObjectsResponse listResponse;
do
{
// Get a list of objects
listResponse = client.ListObjects(listRequest);
foreach (S3Object obj in listResponse.S3Objects)
{
Console.WriteLine("Object - " + obj.Key);
Console.WriteLine(" Size - " + obj.Size);
Console.WriteLine(" LastModified - " + obj.LastModified);
Console.WriteLine(" Storage class - " + obj.StorageClass);
}
// Set the marker property
listRequest.Marker = listResponse.NextMarker;
} while (listResponse.IsTruncated);
#endregion
}
{
#region GetObject Sample
// Create a client
AmazonS3Client client = new AmazonS3Client();
// Create a GetObject request
GetObjectRequest request = new GetObjectRequest
{
BucketName = "SampleBucket",
Key = "Item1"
};
// Issue request and remember to dispose of the response
using (GetObjectResponse response = client.GetObject(request))
{
using (StreamReader reader = new StreamReader(response.ResponseStream))
{
string contents = reader.ReadToEnd();
Console.WriteLine("Object - " + response.Key);
Console.WriteLine(" Version Id - " + response.VersionId);
Console.WriteLine(" Contents - " + contents);
}
}
#endregion
}
{
#region GetObjectMetadata Sample
// Create a client
AmazonS3Client client = new AmazonS3Client();
// Create a GetObjectMetadata request
GetObjectMetadataRequest request = new GetObjectMetadataRequest
{
BucketName = "SampleBucket",
Key = "Item1"
};
// Issue request and view the response
GetObjectMetadataResponse response = client.GetObjectMetadata(request);
Console.WriteLine("Content Length - " + response.ContentLength);
Console.WriteLine("Content Type - " + response.Headers.ContentType);
if (response.Expiration != null)
{
Console.WriteLine("Expiration Date - " + response.Expiration.ExpiryDate);
Console.WriteLine("Expiration Rule Id - " + response.Expiration.RuleId);
}
#endregion
}
{
#region PutObject Sample 1
// Create a client
AmazonS3Client client = new AmazonS3Client();
// Create a PutObject request
PutObjectRequest request = new PutObjectRequest
{
BucketName = "SampleBucket",
Key = "Item1",
ContentBody = "This is sample content..."
};
// Put object
PutObjectResponse response = client.PutObject(request);
#endregion
}
{
#region PutObject Sample 2
// Create a client
AmazonS3Client client = new AmazonS3Client();
// Create a PutObject request
PutObjectRequest request = new PutObjectRequest
{
BucketName = "SampleBucket",
Key = "Item1",
FilePath = "contents.txt"
};
// Put object
PutObjectResponse response = client.PutObject(request);
#endregion
}
{
#region PutObject Sample 3
// Create a client
AmazonS3Client client = new AmazonS3Client();
// Create a PutObject request
PutObjectRequest request = new PutObjectRequest
{
BucketName = "SampleBucket",
Key = "Item1",
};
using (FileStream stream = new FileStream("contents.txt", FileMode.Open))
{
request.InputStream = stream;
// Put object
PutObjectResponse response = client.PutObject(request);
}
#endregion
}
{
#region DeleteObject Sample
// Create a client
AmazonS3Client client = new AmazonS3Client();
// Create a DeleteObject request
DeleteObjectRequest request = new DeleteObjectRequest
{
BucketName = "SampleBucket",
Key = "Item1"
};
// Issue request
client.DeleteObject(request);
#endregion
}
{
#region DeleteObjects Sample
// Create a client
AmazonS3Client client = new AmazonS3Client();
// Create a DeleteObject request
DeleteObjectsRequest request = new DeleteObjectsRequest
{
BucketName = "SampleBucket",
Objects = new List <KeyVersion>
{
new KeyVersion()
{
Key = "Item1"
},
// Versioned item
new KeyVersion()
{
Key = "Item2", VersionId = "Rej8CiBxcZKVK81cLr39j27Y5FVXghDK",
},
// Item in subdirectory
new KeyVersion()
{
Key = "Logs/error.txt"
}
}
};
try
{
// Issue request
DeleteObjectsResponse response = client.DeleteObjects(request);
}
catch (DeleteObjectsException doe)
{
// Catch error and list error details
DeleteObjectsResponse errorResponse = doe.Response;
foreach (DeletedObject deletedObject in errorResponse.DeletedObjects)
{
Console.WriteLine("Deleted item " + deletedObject.Key);
}
foreach (DeleteError deleteError in errorResponse.DeleteErrors)
{
Console.WriteLine("Error deleting item " + deleteError.Key);
Console.WriteLine(" Code - " + deleteError.Code);
Console.WriteLine(" Message - " + deleteError.Message);
}
}
#endregion
}
{
#region CopyObject Sample
// Create a client
AmazonS3Client client = new AmazonS3Client();
// Create a CopyObject request
CopyObjectRequest request = new CopyObjectRequest
{
SourceBucket = "SampleBucket",
SourceKey = "Item1",
DestinationBucket = "AnotherBucket",
DestinationKey = "Copy1",
CannedACL = S3CannedACL.PublicRead
};
// Issue request
client.CopyObject(request);
#endregion
}
{
#region CopyObject Sample
// Create a client
AmazonS3Client client = new AmazonS3Client();
// Create a CopyObject request
CopyObjectRequest request = new CopyObjectRequest
{
SourceBucket = "SampleBucket",
SourceKey = "Item1",
DestinationBucket = "AnotherBucket",
DestinationKey = "Copy1",
CannedACL = S3CannedACL.PublicRead
};
// Issue request
client.CopyObject(request);
#endregion
}
{
#region ListVersions Sample
// Create a client
AmazonS3Client client = new AmazonS3Client();
// Turn versioning on for a bucket
client.PutBucketVersioning(new PutBucketVersioningRequest
{
BucketName = "SampleBucket",
VersioningConfig = new S3BucketVersioningConfig {
Status = "Enable"
}
});
// Populate bucket with multiple items, each with multiple versions
PopulateBucket(client, "SampleBucket");
// Get versions
ListVersionsRequest request = new ListVersionsRequest
{
BucketName = "SampleBucket"
};
// Make paged ListVersions calls
ListVersionsResponse response;
do
{
response = client.ListVersions(request);
// View information about versions
foreach (var version in response.Versions)
{
Console.WriteLine("Key = {0}, Version = {1}, IsLatest = {2}, LastModified = {3}, Size = {4}",
version.Key,
version.VersionId,
version.IsLatest,
version.LastModified,
version.Size);
}
request.KeyMarker = response.NextKeyMarker;
request.VersionIdMarker = response.NextVersionIdMarker;
} while (response.IsTruncated);
#endregion
}
{
#region Multipart Upload Sample
int MB = (int)Math.Pow(2, 20);
// Create a client
AmazonS3Client client = new AmazonS3Client();
// Define input stream
Stream inputStream = Create13MBDataStream();
// Initiate multipart upload
InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest
{
BucketName = "SampleBucket",
Key = "Item1"
};
InitiateMultipartUploadResponse initResponse = client.InitiateMultipartUpload(initRequest);
// Upload part 1
UploadPartRequest uploadRequest = new UploadPartRequest
{
BucketName = "SampleBucket",
Key = "Item1",
UploadId = initResponse.UploadId,
PartNumber = 1,
PartSize = 5 * MB,
InputStream = inputStream
};
UploadPartResponse up1Response = client.UploadPart(uploadRequest);
// Upload part 2
uploadRequest = new UploadPartRequest
{
BucketName = "SampleBucket",
Key = "Item1",
UploadId = initResponse.UploadId,
PartNumber = 2,
PartSize = 5 * MB,
InputStream = inputStream
};
UploadPartResponse up2Response = client.UploadPart(uploadRequest);
// Upload part 3
uploadRequest = new UploadPartRequest
{
BucketName = "SampleBucket",
Key = "Item1",
UploadId = initResponse.UploadId,
PartNumber = 3,
InputStream = inputStream
};
UploadPartResponse up3Response = client.UploadPart(uploadRequest);
// List parts for current upload
ListPartsRequest listPartRequest = new ListPartsRequest
{
BucketName = "SampleBucket",
Key = "Item1",
UploadId = initResponse.UploadId
};
ListPartsResponse listPartResponse = client.ListParts(listPartRequest);
Debug.Assert(listPartResponse.Parts.Count == 3);
// Complete the multipart upload
CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest
{
BucketName = "SampleBucket",
Key = "Item1",
UploadId = initResponse.UploadId,
PartETags = new List <PartETag>
{
new PartETag {
ETag = up1Response.ETag, PartNumber = 1
},
new PartETag {
ETag = up2Response.ETag, PartNumber = 2
},
new PartETag {
ETag = up3Response.ETag, PartNumber = 3
}
}
};
CompleteMultipartUploadResponse compResponse = client.CompleteMultipartUpload(compRequest);
#endregion
}
}
