public async Task <CloudPlatformResponse> PostBlobAsync(CloudPlatformRequest request)
{
AmazonWebServiceResponse response;
if (request == null)
{
throw new ArgumentException("Request is null");
}
using (var client = new AmazonS3Client())
{
response = await client.PutObjectAsync(new PutObjectRequest()
{
BucketName = request.Source,
InputStream = request.DataStream,
AutoCloseStream = false,
Key = request.Key
});
var insertionTime = DateTime.UtcNow;
if (!response.ResponseMetadata.Metadata.ContainsKey(ServerlessBenchmark.Constants.InsertionTime))
{
response.ResponseMetadata.Metadata.Add(ServerlessBenchmark.Constants.InsertionTime, insertionTime.ToString("o"));
}
}
return(AwsCloudPlatformResponse.PopulateFrom(response));
}
public CloudPlatformResponse DeleteMessages(CloudPlatformRequest request)
{
var operationContext = new OperationContext();
var response = new CloudPlatformResponse();
var client = storageAccount.CreateCloudQueueClient();
var queue = client.GetQueueReference(request.Source);
queue.FetchAttributes();
var queueLength = queue.ApproximateMessageCount;
var messages = new List <CloudQueueMessage>();
var deletedMessages = new List <CloudQueueMessage>();
do
{
messages.AddRange(queue.GetMessages(Constants.MaxDequeueAmount));
foreach (var cloudQueueMessage in messages)
{
queue.DeleteMessage(cloudQueueMessage, null, operationContext);
if (operationContext.LastResult.HttpStatusCode == 204)
{
deletedMessages.Add(cloudQueueMessage);
}
}
messages.RemoveAll(cloudQueueMessage => deletedMessages.Contains(cloudQueueMessage));
deletedMessages.Clear();
queue.FetchAttributes();
queueLength = queue.ApproximateMessageCount;
} while (queueLength > 0);
var successfulPost = operationContext.RequestResults.All(cxt => cxt.HttpStatusCode == 204);
response.HttpStatusCode = successfulPost ? HttpStatusCode.OK : HttpStatusCode.Conflict;
return(response);
}
public async Task <CloudPlatformResponse> DequeueMessagesAsync(CloudPlatformRequest request)
{
var cResponse = new CloudPlatformResponse();
try
{
ReceiveMessageResponse response;
using (var client = new AmazonSQSClient())
{
var queueUrl = client.GetQueueUrl(request.Source).QueueUrl;
response = await client.ReceiveMessageAsync(queueUrl);
if (response.HttpStatusCode == HttpStatusCode.OK)
{
cResponse.Data = response.Messages;
foreach (var message in response.Messages)
{
await client.DeleteMessageAsync(new DeleteMessageRequest(queueUrl, message.ReceiptHandle));
}
}
else
{
cResponse.ErrorDetails.Add("unknown", 1);
}
}
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
return(cResponse);
}
public CloudPlatformResponse DeleteBlobs(CloudPlatformRequest request)
{
var response = new CloudPlatformResponse();
var blobs = ListBlobs(request).Data as List <CloudBlockBlob> ?? new List <CloudBlockBlob>();
Parallel.ForEach(blobs, blob => blob.Delete());
response.HttpStatusCode = HttpStatusCode.OK;
return(response);
}
public async Task <CloudPlatformResponse> PostBlobAsync(CloudPlatformRequest request)
{
var uploadContext = new OperationContext();
var response = new CloudPlatformResponse();
var blobContainer = BlobClient.GetContainerReference(request.Source);
var blob = blobContainer.GetBlockBlobReference(request.Key);
await blob.UploadFromStreamAsync(request.DataStream, null, null, uploadContext);
var successfulPost = uploadContext.RequestResults.All(uploadRequest => uploadRequest.HttpStatusCode == 201);
response.HttpStatusCode = successfulPost ? HttpStatusCode.OK : HttpStatusCode.Conflict;
return(response);
}
public CloudPlatformResponse PostBlob(CloudPlatformRequest request)
{
OperationContext uploadContext = new OperationContext();
var response = new CloudPlatformResponse();
var client = storageAccount.CreateCloudBlobClient();
var blobContainer = client.GetContainerReference(request.Source);
var blob = blobContainer.GetBlockBlobReference(request.Key);
blob.UploadFromStream(request.DataStream, operationContext: uploadContext);
var successfulPost = uploadContext.RequestResults.All(uploadRequest => uploadRequest.HttpStatusCode == 200);
response.HttpStatusCode = successfulPost ? HttpStatusCode.OK : HttpStatusCode.Conflict;
return(response);
}
public CloudPlatformResponse ListBlobs(CloudPlatformRequest request)
{
var requestContext = new OperationContext();
var response = new CloudPlatformResponse();
var client = storageAccount.CreateCloudBlobClient();
var blobContainer = client.GetContainerReference(request.Source);
var blobs = blobContainer.ListBlobs(operationContext: requestContext).Select(blob => (CloudBlockBlob)blob).ToList();
response.Data = blobs;
var successfulPost = requestContext.RequestResults.All(r => r.HttpStatusCode == 200);
response.HttpStatusCode = successfulPost ? HttpStatusCode.OK : HttpStatusCode.Conflict;
response.TimeStamp = requestContext.EndTime;
return(response);
}
public CloudPlatformResponse DeleteBlobs(CloudPlatformRequest request)
{
var blobs = (IEnumerable <S3Object>)ListBlobs(request).Data;
AmazonWebServiceResponse response = null;
const int deleteLimit = 1000;
if (blobs.Any())
{
using (var client = new AmazonS3Client(new AmazonS3Config()))
{
try
{
var keys = blobs.Select(blob => new KeyVersion {
Key = blob.Key
}).ToList();
int num = keys.Count;
do
{
Console.WriteLine("Deleting Blobs - Remaining: {0}", num);
num -= (num < deleteLimit ? num : deleteLimit);
var takenKeys = keys.Take(deleteLimit).ToList();
response = client.DeleteObjects(new DeleteObjectsRequest()
{
BucketName = request.Source,
Objects = takenKeys
});
keys = keys.Except(takenKeys).ToList();
} while (num > 0);
}
catch (DeleteObjectsException e)
{
DeleteObjectsResponse errorResponse = e.Response;
foreach (DeleteError deleteError in errorResponse.DeleteErrors)
{
Console.WriteLine("Error deleting item " + deleteError.Key);
Console.WriteLine(" Code - " + deleteError.Code);
Console.WriteLine(" Message - " + deleteError.Message);
}
}
}
}
return(AwsCloudPlatformResponse.PopulateFrom(response));
}
public CloudPlatformResponse DeleteMessages(CloudPlatformRequest request)
{
var sqsClient = new AmazonSQSClient();
var cpResponse = new CloudPlatformResponse();
try
{
var queueUrl = sqsClient.GetQueueUrl(request.Source).QueueUrl;
var response = sqsClient.PurgeQueue(queueUrl);
cpResponse.HttpStatusCode = response.HttpStatusCode;
}
catch (PurgeQueueInProgressException)
{
//retry
Thread.Sleep(TimeSpan.FromSeconds(60));
cpResponse.HttpStatusCode = HttpStatusCode.InternalServerError;
}
return(cpResponse);
}
public async Task <CloudPlatformResponse> DequeueMessagesAsync(CloudPlatformRequest request)
{
var operationContext = new OperationContext();
var response = new CloudPlatformResponse();
var messages = new List <CloudQueueMessage>();
var queue = QueueClient.GetQueueReference(request.Source);
queue.FetchAttributes();
var queueLength = queue.ApproximateMessageCount;
do
{
messages.AddRange(await queue.GetMessagesAsync(Constants.MaxDequeueAmount));
} while (messages.Count < queueLength);
var messagesString = messages.Select(message => message.AsString);
var successfulPost = operationContext.RequestResults.All(cxt => cxt.HttpStatusCode == 200);
response.HttpStatusCode = successfulPost ? HttpStatusCode.OK : HttpStatusCode.Conflict;
response.Data = messagesString;
return(response);
}
public async Task <CloudPlatformResponse> EnqueueMessagesAsync(CloudPlatformRequest request)
{
var cResponse = new CloudPlatformResponse();
try
{
SendMessageBatchResponse response;
var messages = (IEnumerable <string>)request.Data[ServerlessBenchmark.Constants.Message];
var batchMessageEntry =
messages.Select(message => new SendMessageBatchRequestEntry(Guid.NewGuid().ToString("N"), message))
.ToList();
using (var client = new AmazonSQSClient())
{
response = await client.SendMessageBatchAsync(client.GetQueueUrl(request.Source).QueueUrl, batchMessageEntry);
}
if (response.Failed.Any())
{
var groupedFailures = response.Failed.GroupBy(failure => failure.Message);
foreach (var group in groupedFailures)
{
cResponse.ErrorDetails.Add(group.Key, group.Count());
}
cResponse.HttpStatusCode = HttpStatusCode.InternalServerError;
}
else
{
cResponse.HttpStatusCode = HttpStatusCode.OK;
}
}
catch (InvalidCastException)
{
Console.WriteLine("Data needs to be IEnumberable of strings");
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
return(cResponse);
}
public CloudPlatformResponse ListBlobs(CloudPlatformRequest request)
{
var response = new CloudPlatformResponse();
var blobs = new List <S3Object>();
var listBlobRequest = new ListObjectsRequest()
{
BucketName = request.Source
};
using (var client = new AmazonS3Client())
{
ListObjectsResponse listResponse;
do
{
listResponse = client.ListObjects(listBlobRequest);
blobs.AddRange(listResponse.S3Objects);
listBlobRequest.Marker = listResponse.NextMarker;
} while (listResponse.IsTruncated);
}
response.Data = blobs;
return(response);
}
public async Task <CloudPlatformResponse> PostMessagesAsync(CloudPlatformRequest request)
{
IEnumerable <PublishResponse> completedPublishJobs;
var topic = request.Data[Constants.Topic] as string;
var messages = request.Data[Constants.Message] as IEnumerable <string>;
var pendingPublishJobs = messages.Select(message => PublishMessage(topic, message, request.RetryAttempts)).ToList();
var completedPublishJobsTasks = Task.WhenAll(pendingPublishJobs);
var timeout = request.TimeoutMilliseconds ?? (int)TimeSpan.FromSeconds(60).TotalMilliseconds;
var timeoutTask = Task.Delay(timeout);
var aggregateTasks = new List <Task>()
{
{ completedPublishJobsTasks },
{ timeoutTask }
};
var doneTask = await Task.WhenAny(aggregateTasks);
if (doneTask.Id == timeoutTask.Id)
{
Console.WriteLine("--ERROR-- Reached timeout {0}ms", timeout);
throw new Exception();
}
else
{
//the only other task that we were waiting for were the publish jobs
completedPublishJobs = ((Task <PublishResponse[]>)doneTask).Result;
}
var areAnyFailed = completedPublishJobs.Any(publishJob => publishJob.HttpStatusCode != HttpStatusCode.OK);
var requestResponse = new CloudPlatformResponse()
{
HttpStatusCode = areAnyFailed ? HttpStatusCode.Conflict : HttpStatusCode.OK,
TimeStamp = DateTime.Now
};
return(requestResponse);
}
public async Task <CloudPlatformResponse> EnqueueMessagesAsync(CloudPlatformRequest request)
{
var response = new CloudPlatformResponse();
var messages = request.Data[Constants.Message] as IEnumerable <string>;
var queue = QueueClient.GetQueueReference(request.Source);
var tasks = new List <Task>();
var operationResultsByTask = new Dictionary <int, OperationContext>();
foreach (var message in messages)
{
var operationContext = new OperationContext();
var t = queue.AddMessageAsync(new CloudQueueMessage(message), null, null, null, operationContext);
tasks.Add(t);
operationResultsByTask.Add(t.Id, operationContext);
}
await Task.WhenAll(tasks);
var operationResults = operationResultsByTask.Values;
var successfulPost = operationResults.All(operationContext => operationContext.LastResult.HttpStatusCode == 201);
response.HttpStatusCode = successfulPost ? HttpStatusCode.OK : HttpStatusCode.Conflict;
return(response);
}
public CloudPlatformResponse GetMessages(CloudPlatformRequest request)
{
throw new NotImplementedException();
}
public Task <CloudPlatformResponse> PostMessagesAsync(CloudPlatformRequest request)
{
throw new NotImplementedException("Use this for event hub");
}
public Task <CloudPlatformResponse> PostBlobsAsync(CloudPlatformRequest request)
{
throw new NotImplementedException();
}
public CloudPlatformResponse GetMessages(CloudPlatformRequest request)
{
throw new NotImplementedException("Use this for event hub");
}
