private async Task RunAsync(CancellationToken cancellationToken)
{
IQueueProcessor queueProcessor = null;
while (!cancellationToken.IsCancellationRequested)
{
Trace.TraceInformation("Working");
await Task.Delay(10, cancellationToken);
try
{
// Retrieve and process a new message from the queue.
_receivedMessage = _urbanWaterQueue.GetMessage();
if (_receivedMessage != null)
{
if (_receivedMessage.DequeueCount > 5)
{
EventSourceWriter.Log.MessageMethod("Worker Role Processing dequeued message after 5 failure Attempts " + _receivedMessage.Id);
_urbanWaterQueue.DeleteMessage(_receivedMessage);
return;
}
EventSourceWriter.Log.MessageMethod("Worker Role Processing recived message from Queue " + _receivedMessage.Id);
var messageParts = _receivedMessage.AsString.Split(',');
var strType = messageParts[1];
var strFileName = messageParts[0];
var strContainer = messageParts[1];
var strConnectionString = messageParts[3];
// Retrieve storage account from connection string.
var account = Microsoft.WindowsAzure.Storage.CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting(strConnectionString));
// Create the blob client.
var blobClient = account.CreateCloudBlobClient();
// Retrieve reference to a previously created container.
var container = blobClient.GetContainerReference(strContainer);
// Retrieve reference to a blob file.
var blockBlob = container.GetBlockBlobReference(strFileName);
var blobs = container.ListBlobs();
while (blobs.Any() && blobs.FirstOrDefault() is CloudBlobDirectory)
{
var directory = (CloudBlobDirectory)blobs.FirstOrDefault();
blobs = directory.ListBlobs();
if (blobs.FirstOrDefault().GetType() == typeof(CloudBlockBlob))
{
blockBlob = directory.GetBlockBlobReference(strFileName);
}
}
switch (strType)
{
case "sw-sagem":
{
queueProcessor = new WaterMeterQueueXMLProcessor();
break;
}
case "aqualiameterdata":
{
queueProcessor = new WaterMeterCSVProcessor_Aqualia();
break;
}
case "ovodmeterdata":
{
queueProcessor = new WaterMeterExcelProcessor_Ovod();
break;
}
case "Weatherdata":
{
queueProcessor = new WeatherQueueProcessor();
break;
}
}
queueProcessor?.ProcessQueue(blockBlob, _receivedMessage, _urbanWaterQueue);
}
}
catch (Exception e)
{
EventSourceWriter.Log.MessageMethod("Exception thrown in processor Worker role " + e.Message);
}
}
}
private async Task ProcessMessages(TimeSpan?visibilityTimeout = null, CancellationToken cancellationToken = default(CancellationToken))
{
var runningTasks = new ConcurrentDictionary <Task, Task>();
var semaphore = new SemaphoreSlimEx(_minConcurrentTasks, _minConcurrentTasks, _maxConcurrentTasks);
// Define the task pump
var pumpTask = Task.Run(async() =>
{
while (!cancellationToken.IsCancellationRequested)
{
await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
var runningTask = Task.Run(async() =>
{
if (cancellationToken.IsCancellationRequested)
{
return(false);
}
CloudQueueMessage message = null;
try
{
message = await _cloudQueue.GetMessageAsync(visibilityTimeout, null, null, cancellationToken);
}
catch (Exception e)
{
_logger.ErrorException("An error occured when attempting to get a message from the queue", e);
}
if (message == null)
{
try
{
// The queue is empty
OnQueueEmpty?.Invoke(cancellationToken);
}
#pragma warning disable RECS0022 // A catch clause that catches System.Exception and has an empty body
catch
#pragma warning restore RECS0022 // A catch clause that catches System.Exception and has an empty body
{
// Intentionally left empty. We ignore errors from OnQueueEmpty.
}
// False indicates that no message was processed
return(false);
}
else
{
try
{
// Process the message
OnMessage?.Invoke(message, cancellationToken);
// Delete the processed message from the queue
await _cloudQueue.DeleteMessageAsync(message);
}
catch (Exception ex)
{
var isPoison = (message.DequeueCount > _maxDequeueCount);
OnError?.Invoke(message, ex, isPoison);
if (isPoison)
{
await _cloudQueue.DeleteMessageAsync(message);
}
}
// True indicates that a message was processed
return(true);
}
}, CancellationToken.None);
runningTasks.TryAdd(runningTask, runningTask);
runningTask.ContinueWith(async t =>
{
// Decide if we need to scale up or down
if (!cancellationToken.IsCancellationRequested)
{
if (await t)
{
// The queue is not empty, therefore increase the number of concurrent tasks
semaphore.TryIncrease();
}
else
{
// The queue is empty, therefore reduce the number of concurrent tasks
semaphore.TryDecrease();
}
}
// Complete the task
semaphore.Release();
Task taskToBeRemoved;
runningTasks.TryRemove(t, out taskToBeRemoved);
}, TaskContinuationOptions.ExecuteSynchronously)
.IgnoreAwait();
}
});
// Run the task pump until canceled
await pumpTask.UntilCancelled().ConfigureAwait(false);
// Task pump has been canceled, wait for the currently running tasks to complete
await Task.WhenAll(runningTasks.Values).UntilCancelled().ConfigureAwait(false);
}
public virtual async Task AbandonMessageAsync(MessageData message, SessionBase session)
{
CloudQueueMessage queueMessage = message.OriginalQueueMessage;
TaskMessage taskMessage = message.TaskMessage;
OrchestrationInstance instance = taskMessage.OrchestrationInstance;
// Exponentially backoff a given queue message until a maximum visibility delay of 10 minutes.
// Once it hits the maximum, log the message as a poison message.
const int maxSecondsToWait = 600;
int numSecondsToWait = Math.Min((int)Math.Pow(2, queueMessage.DequeueCount), maxSecondsToWait);
if (numSecondsToWait == maxSecondsToWait)
{
AnalyticsEventSource.Log.PoisonMessageDetected(
this.storageAccountName,
this.settings.TaskHubName,
queueMessage.Id,
instance.InstanceId,
instance.ExecutionId,
this.storageQueue.Name,
queueMessage.DequeueCount,
Utils.ExtensionVersion);
}
TimeSpan visibilityDelay = TimeSpan.FromSeconds(numSecondsToWait);
AnalyticsEventSource.Log.AbandoningMessage(
this.storageAccountName,
this.settings.TaskHubName,
taskMessage.Event.EventType.ToString(),
Utils.GetTaskEventId(taskMessage.Event),
queueMessage.Id,
instance.InstanceId,
instance.ExecutionId,
this.storageQueue.Name,
message.SequenceNumber,
(int)visibilityDelay.TotalSeconds,
Utils.ExtensionVersion);
try
{
// We "abandon" the message by settings its visibility timeout using an exponential backoff algorithm.
// This allows it to be reprocessed on this node or another node at a later time, hopefully successfully.
await this.storageQueue.UpdateMessageAsync(
queueMessage,
visibilityDelay,
MessageUpdateFields.Visibility,
this.QueueRequestOptions,
session.StorageOperationContext);
this.stats.MessagesUpdated.Increment();
}
catch (Exception e)
{
// Message may have been processed and deleted already.
this.HandleMessagingExceptions(e, message, $"Caller: {nameof(AbandonMessageAsync)}");
}
finally
{
this.stats.StorageRequests.Increment();
}
}
public override void Run()
{
var storageAccount = CloudStorageAccount.Parse(RoleEnvironment.GetConfigurationSettingValue("DataConnectionString"));
CloudBlobClient blobStorage = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobStorage.GetContainerReference("photogallery");
CloudQueueClient queueStorage = storageAccount.CreateCloudQueueClient();
CloudQueue queue = queueStorage.GetQueueReference("thumbnailmaker");
Trace.TraceInformation("Creating container and queue...");
// If the Start() method throws an exception, the role recycles.
// If this sample is run locally and the development storage tool has not been started, this
// can cause a number of exceptions to be thrown because roles are restarted repeatedly.
// Lets try to create the queue and the container and check whether the storage services are running
// at all.
bool containerAndQueueCreated = false;
while (!containerAndQueueCreated)
{
try
{
container.CreateIfNotExist();
var permissions = container.GetPermissions();
permissions.PublicAccess = BlobContainerPublicAccessType.Container;
container.SetPermissions(permissions);
permissions = container.GetPermissions();
queue.CreateIfNotExist();
containerAndQueueCreated = true;
}
catch (StorageClientException e)
{
if (e.ErrorCode == StorageErrorCode.TransportError)
{
Trace.TraceError(string.Format("Connect failure! The most likely reason is that the local " +
"Development Storage tool is not running or your storage account configuration is incorrect. " +
"Message: '{0}'", e.Message));
System.Threading.Thread.Sleep(5000);
}
else
{
throw;
}
}
}
Trace.TraceInformation("Listening for queue messages...");
// Now that the queue and the container have been created in the above initialization process, get messages
// from the queue and process them individually.
while (true)
{
try
{
CloudQueueMessage msg = queue.GetMessage();
if (msg != null)
{
string path = msg.AsString;
string thumbnailName = System.IO.Path.GetFileNameWithoutExtension(path) + ".jpg";
Trace.TraceInformation(string.Format("Dequeued '{0}'", path));
CloudBlockBlob content = container.GetBlockBlobReference(path);
CloudBlockBlob thumbnail = container.GetBlockBlobReference("thumbnails/" + thumbnailName);
MemoryStream image = new MemoryStream();
content.DownloadToStream(image);
image.Seek(0, SeekOrigin.Begin);
thumbnail.Properties.ContentType = "image/jpeg";
thumbnail.UploadFromStream(CreateThumbnail(image));
Trace.TraceInformation(string.Format("Done with '{0}'", path));
queue.DeleteMessage(msg);
}
else
{
System.Threading.Thread.Sleep(1000);
}
}
catch (Exception e)
{
// Explicitly catch all exceptions of type StorageException here because we should be able to
// recover from these exceptions next time the queue message, which caused this exception,
// becomes visible again.
System.Threading.Thread.Sleep(5000);
Trace.TraceError(string.Format("Exception when processing queue item. Message: '{0}'", e.Message));
}
}
}
public static void Run([QueueTrigger(TriggerQueueName)] CloudQueueMessage message,
[Table(TableName, PartitionKey, RowKey)] Poco entity)
{
entity.Value = message.AsString;
}
public static void Run([QueueTrigger(TriggerQueueName)] CloudQueueMessage message,
[Table(TableName, PartitionKey, RowKey)] PocoWithByteArrayValue entity)
{
entity.Value = message.AsBytes;
}
public static void Run([QueueTrigger(QueueName)] CloudQueueMessage message, DateTimeOffset insertionTime)
{
TaskSource.TrySetResult(insertionTime);
}
public void ProcessQueue(CloudBlockBlob blob,
CloudQueueMessage receivedMessage,
CloudQueue urbanWaterQueue)
{
EventSourceWriter.Log.MessageMethod(
$"WaterMeterExcelProcessor_Ovod Message Processing Started. MessageId: {receivedMessage.Id}, BLOB name: {blob.Name}");
var currentContext = new SQLAzureDataContext();
var missingMeterIds = new List <string>();
var processedMeterIds = new List <string>();
var stream = new MemoryStream();
blob.DownloadToStream(stream);
using (var spreadsheetDocument = SpreadsheetDocument.Open(stream, false))
{
var workbookPart = spreadsheetDocument.WorkbookPart;
var worksheetPart = workbookPart.WorksheetParts.First();
var sheetData = worksheetPart.Worksheet.Elements <SheetData>().First();
foreach (var r in sheetData.Elements <Row>())
{
if (r.RowIndex != 1)
{
var creationDateTime = DateTime.Now;
string meterIdentity = null;
string reading = null;
string customer = null;
var column = 1;
foreach (var c in r.Elements <Cell>())
{
switch (column)
{
case 1:
customer = c.InnerText;
break;
case 2:
meterIdentity = c.InnerText;
break;
case 3:
creationDateTime = DateTime.ParseExact(c.InnerText, "MM/dd/yyyy hh:mm tt", null).ToUniversalTime();
break;
case 4:
reading = c.InnerText;
break;
default:
break;
}
column = column + 1;
}
try
{
if (string.IsNullOrEmpty(meterIdentity))
{
meterIdentity = currentContext.Customers.FirstOrDefault(x => x.CustomerNumber == customer).Meter.MeterIdentity;
}
var meterSet = currentContext.Meters.Where(x => x.MeterIdentity == meterIdentity);
if (!meterSet.Any())
{
if (!missingMeterIds.Contains(meterIdentity))
{
missingMeterIds.Add(meterIdentity);
}
}
else
{
if (!processedMeterIds.Contains(meterIdentity))
{
processedMeterIds.Add(meterIdentity);
}
}
}
catch (Exception e)
{
EventSourceWriter.Log.MessageMethod($"EXCEPTION: WaterMeterExcelProcessor_Ovod. {e.Message}");
}
try
{
var firstOrDefault = currentContext.Meters.FirstOrDefault(x => x.MeterIdentity == meterIdentity);
if (firstOrDefault != null)
{
var thisDma =
firstOrDefault.DMA;
var sm = new MeterReadingEntity
{
PartitionKey = meterIdentity,
CreatedOn = creationDateTime,
RowKey = creationDateTime.Ticks.ToString(),
Reading = reading,
Encrypted = false,
DMA = thisDma.Identifier
};
const string strWriteConnectionString = "MKWDNConnectionString";
var blAttempt = WriteMessageMeterDataToDataTable(sm, strWriteConnectionString,
thisDma.Site.TableName);
if (!blAttempt)
{
EventSourceWriter.Log.MessageMethod(
$"ERROR writing to BLOB storage by WaterMeterExcelProcessor_Ovod. Received message Id: {receivedMessage.Id}BLOB name: {blob.Name}, PartitionKey: {sm.PartitionKey}");
}
}
}
catch (Exception e)
{
EventSourceWriter.Log.MessageMethod(
$"ERROR writing to BLOB storage in WaterMeterExcelProcessor_Ovod. Received message Id: {receivedMessage.Id}BLOB name: {blob.Name}, Exception: {e.Message}");
}
}
}
}
EventSourceWriter.Log.MessageMethod("WaterMeterExcelProcessor_Ovod Message Processing Complete. MessageId: " + receivedMessage.Id);
urbanWaterQueue.DeleteMessage(receivedMessage);
var newEvent = new Event
{
BusDispatched = false,
EventDateTime = DateTime.Now,
EventType = (int)EventTypes.NewMeteringData
};
var processedMetercsv = processedMeterIds.Aggregate(string.Empty, (current, meterId) => current + (meterId + ", "));
newEvent.Description = processedMetercsv;
currentContext.Events.InsertOnSubmit(newEvent);
if (missingMeterIds.Count > 0)
{
var newMeterMissingEvent = new Event
{
BusDispatched = false,
EventDateTime = DateTime.Now,
EventType = (int)EventTypes.MissingMeter
};
var metercsv = missingMeterIds.Aggregate(string.Empty, (current, meterId) => current + (meterId + ", "));
newMeterMissingEvent.Description = metercsv;
currentContext.Events.InsertOnSubmit(newMeterMissingEvent);
}
currentContext.SubmitChanges();
}
public static void Run([QueueTrigger(QueueName)] CloudQueueMessage message, string queueTrigger)
{
TaskSource.TrySetResult(queueTrigger);
}
public static void Run([QueueTrigger(QueueName)] CloudQueueMessage message, int dequeueCount)
{
TaskSource.TrySetResult(dequeueCount);
}
public override Task CompleteProcessingMessageAsync(CloudQueueMessage message, FunctionResult result, CancellationToken cancellationToken)
{
CompleteProcessingCount++;
return(base.CompleteProcessingMessageAsync(message, result, cancellationToken));
}
public override Task <bool> BeginProcessingMessageAsync(CloudQueueMessage message, CancellationToken cancellationToken)
{
BeginProcessingCount++;
return(base.BeginProcessingMessageAsync(message, cancellationToken));
}
/// <summary>
/// Send back to watcher a "Posion messsage" and delete from in queue
/// </summary>
/// <param name="poisonMessage">the poison message</param>
/// <returns>Sueccess or not</returns>
private bool SendPoisonMessage(CloudQueueMessage poisonMessage)
{
bool sw = false;
try
{
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(_Configuration.ProcessConfigConn);
CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient();
CloudQueue poisonQueue = queueClient.GetQueueReference(_Configuration.poisonQueue);
poisonQueue.CreateIfNotExists();
MediaButler.Common.ButlerRequest myButlerRequest = Newtonsoft.Json.JsonConvert.DeserializeObject <ButlerRequest>(poisonMessage.AsString);
MediaButler.Common.ButlerResponse myButlerResponse = new Common.ButlerResponse();
myButlerResponse.MezzanineFiles = myButlerRequest.MezzanineFiles;
////Add to Mezzamine Files the control File URL if it exist
//Becouse it is needed to move/delete the control file from processing to succes or fail
if (!string.IsNullOrEmpty(myButlerRequest.ControlFileUri))
{
myButlerResponse.MezzanineFiles.Add(myButlerRequest.ControlFileUri);
}
myButlerResponse.TimeStampProcessingCompleted = DateTime.Now.ToString();
myButlerResponse.TimeStampProcessingStarted = DateTime.Now.ToString();
myButlerResponse.WorkflowName = myButlerRequest.WorkflowName;
myButlerResponse.MessageId = myButlerRequest.MessageId;
myButlerResponse.TimeStampRequestSubmitted = myButlerRequest.TimeStampUTC;
myButlerResponse.StorageConnectionString = myButlerRequest.StorageConnectionString;
myButlerResponse.Log = "Poison Message";
//Lookin for Errors in Table Status
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
CloudTable table = tableClient.GetTableReference(MediaButler.Common.Configuration.ButlerWorkflowStatus);
TableOperation retrieveOperation =
TableOperation.Retrieve <MediaButler.Common.workflow.ProcessSnapShot>(myButlerRequest.WorkflowName, myButlerRequest.MessageId.ToString());
TableResult retrievedResult = table.Execute(retrieveOperation);
if (retrievedResult.Result != null)
{
//we have process info
var status = (MediaButler.Common.workflow.ProcessSnapShot)retrievedResult.Result;
try
{
var request = Newtonsoft.Json.JsonConvert.DeserializeObject <MediaButler.Common.workflow.ChainRequest>(status.jsonContext);
foreach (var error in request.Exceptions)
{
myButlerResponse.Log += "\r\n" + error.Message;
}
}
catch (Exception X)
{
Trace.TraceWarning("Unable to load Error LOG in response.log on poison message");
myButlerResponse.Log += "\r\n" + X.Message;
myButlerResponse.Log += "\r\n" + status.jsonContext;
}
//Delete register from Status table
TableOperation insertOperation = TableOperation.Delete(status);
table.Execute(insertOperation);
}
//Send Poison Mesagge
CloudQueueMessage poison = new CloudQueueMessage(Newtonsoft.Json.JsonConvert.SerializeObject(myButlerResponse));
poisonQueue.AddMessage(poison);
sw = true;
}
catch (Exception X)
{
string txt = string.Format("[{0}] at {1} has an error: {2}", this.GetType().FullName, "GetNewMessage", X.Message);
Trace.TraceError(txt);
}
return(sw);
}
/// <summary>
/// Check if the incoming message was dequeue more than N time. N is defines in configurtion
/// </summary>
/// <param name="theMessage">incoming message</param>
/// <returns>Is or not poisson message</returns>
private bool CheckPoison(CloudQueueMessage theMessage)
{
return(_Configuration.MaxDequeueCount < theMessage.DequeueCount);
}
public static void Run([QueueTrigger(QueueName)] CloudQueueMessage message, DateTimeOffset nextVisibleTime)
{
TaskSource.TrySetResult(nextVisibleTime);
}
static void Main(string[] args)
{
Console.WriteLine("Queue encryption sample");
// Retrieve storage account information from connection string
// How to create a storage connection string - https://azure.microsoft.com/en-us/documentation/articles/storage-configure-connection-string/
CloudStorageAccount storageAccount = EncryptionShared.Utility.CreateStorageAccountFromConnectionString();
CloudQueueClient client = storageAccount.CreateCloudQueueClient();
CloudQueue queue = client.GetQueueReference(DemoQueue + Guid.NewGuid().ToString("N"));
try
{
queue.Create();
// Create the IKey used for encryption.
RsaKey key = new RsaKey("private:key1");
// Create the encryption policy to be used for insert and update.
QueueEncryptionPolicy insertPolicy = new QueueEncryptionPolicy(key, null);
// Set the encryption policy on the request options.
QueueRequestOptions insertOptions = new QueueRequestOptions()
{
EncryptionPolicy = insertPolicy
};
string messageStr = Guid.NewGuid().ToString();
CloudQueueMessage message = new CloudQueueMessage(messageStr);
// Add message
Console.WriteLine("Inserting the encrypted message.");
queue.AddMessage(message, null, null, insertOptions, null);
// For retrieves, a resolver can be set up that will help pick the key based on the key id.
LocalResolver resolver = new LocalResolver();
resolver.Add(key);
QueueEncryptionPolicy retrPolicy = new QueueEncryptionPolicy(null, resolver);
QueueRequestOptions retrieveOptions = new QueueRequestOptions()
{
EncryptionPolicy = retrPolicy
};
// Retrieve message
Console.WriteLine("Retrieving the encrypted message.");
CloudQueueMessage retrMessage = queue.GetMessage(null, retrieveOptions, null);
// Update message
Console.WriteLine("Updating the encrypted message.");
string updatedMessage = Guid.NewGuid().ToString("N");
retrMessage.SetMessageContent(updatedMessage);
queue.UpdateMessage(retrMessage, TimeSpan.FromSeconds(0), MessageUpdateFields.Content | MessageUpdateFields.Visibility, insertOptions, null);
// Retrieve updated message
Console.WriteLine("Retrieving the updated encrypted message.");
retrMessage = queue.GetMessage(null, retrieveOptions, null);
Console.WriteLine("Press enter key to exit");
Console.ReadLine();
}
finally
{
queue.DeleteIfExists();
}
}
public static void Run([QueueTrigger(QueueName)] CloudQueueMessage message, string popReceipt)
{
TaskSource.TrySetResult(popReceipt);
}
public static void Run([QueueTrigger(TriggerQueueName)] CloudQueueMessage ignore,
[Table(TableName, PartitionKey, RowKey)] DynamicTableEntity entity)
{
TaskSource.TrySetResult(entity);
}
public void Send(MessageBusDto msg)
{
CloudQueueMessage queueMsg = new CloudQueueMessage(JsonConvert.SerializeObject(msg));
queue.AddMessageAsync(queueMsg).Wait();
}
public static void Run([QueueTrigger(TriggerQueueName)] CloudQueueMessage ignore,
[Table(TableName, PartitionKey, RowKey)] PocoWithByteArrayValue entity)
{
TaskSource.TrySetResult(entity);
}
public void PushMessage(string text)
{
var message = new CloudQueueMessage(text);
Queue.AddMessage(message);
}
public static void Run([QueueTrigger(TriggerQueueName)] CloudQueueMessage ignore,
[Table(TableName, PartitionKey, RowKey)] PocoWithKeys entity)
{
entity.RowKey = Guid.NewGuid().ToString();
}
public static async Task <IActionResult> RunAsync(
[HttpTrigger(AuthorizationLevel.Anonymous,
"DELETE", "GET", "HEAD", "OPTIONS", "POST", "PUT", "TRACE",
Route = "{*path}")] HttpRequestMessage httpRequest,
[Inject] Client client,
[Config("InitialRequestTimeout")] TimeSpan initialRequestTimeout,
[Queue(queueName: "requests", Connection = "RequestsQueueConnection")] CloudQueue queue,
[Config("InitialRetryDelay")] TimeSpan initialRetryDelay,
[Config("MaxRetryDelay")] TimeSpan maxRetryDelay,
ILogger logger)
{
logger.LogInformation($"Received request {httpRequest.RequestUri}");
// remove "/r/" from the path and query part of URL
var targetUri = httpRequest.RequestUri.PathAndQuery.Substring(3);
if (!Uri.TryCreate(targetUri, UriKind.Absolute, out var uri))
{
logger.LogInformation($"Rejecting request to {targetUri} because it is not valid absolute Uri");
return(new BadRequestObjectResult($"Unsupported request to {uri}"));
}
HttpRequestMessage requestMessage;
try
{
requestMessage = new HttpRequestMessage(httpRequest.Method, uri);
}
catch (ArgumentException ex)
{
logger.LogInformation($"Rejecting request to {targetUri} due to {ex.Message}");
return(new BadRequestObjectResult($"Unsupported request to {uri}"));
}
if (httpRequest.Method != HttpMethod.Get && httpRequest.Method != HttpMethod.Head)
{
requestMessage.Content = httpRequest.Content;
}
;
var requestResult = await client.SendAsync(requestMessage, initialRequestTimeout)
.ConfigureAwait(false);
switch (requestResult)
{
case RequestResult.Ok:
logger.LogInformation($"Succeeded to relay to {uri}");
return(new OkResult());
case RequestResult.Invalid:
logger.LogInformation($"Rejecting invalid request to {uri}");
return(new BadRequestResult());
}
var request = new RequestMessage()
{
Destination = targetUri,
Method = httpRequest.Method.Method,
DelayInSeconds = initialRetryDelay.TotalSeconds,
};
if (httpRequest.Content != null)
{
request.Content = await httpRequest.Content.ReadAsByteArrayAsync().ConfigureAwait(false);
}
var serializedRequest = JsonConvert.SerializeObject(request);
// 48KB is a limit for byte array queue messages
// https://docs.microsoft.com/en-us/azure/service-bus-messaging/service-bus-azure-and-service-bus-queues-compared-contrasted#capacity-and-quotas
// reserving 1KB for other properties of the message, like TTL
if (Encoding.Unicode.GetByteCount(serializedRequest) > 47 * KB)
{
}
var message = new CloudQueueMessage(serializedRequest);
await queue.AddMessageAsync(message,
timeToLive : maxRetryDelay,
initialVisibilityDelay : initialRetryDelay,
options : queue.ServiceClient.DefaultRequestOptions,
operationContext : null)
.ConfigureAwait(false);
logger.LogInformation($"Postponed request for {initialRetryDelay.TotalSeconds}s to {uri}");
return(new OkResult());
}
async Task <MessageData> AddMessageAsync(TaskMessage taskMessage, OrchestrationInstance sourceInstance, SessionBase session)
{
MessageData data;
try
{
// We transfer to a new trace activity ID every time a new outbound queue message is created.
Guid outboundTraceActivityId = Guid.NewGuid();
data = new MessageData(
taskMessage,
outboundTraceActivityId,
this.storageQueue.Name,
session?.GetCurrentEpisode(),
sourceInstance);
data.SequenceNumber = Interlocked.Increment(ref messageSequenceNumber);
string rawContent = await this.messageManager.SerializeMessageDataAsync(data);
CloudQueueMessage queueMessage = new CloudQueueMessage(rawContent);
AnalyticsEventSource.Log.SendingMessage(
outboundTraceActivityId,
this.storageAccountName,
this.settings.TaskHubName,
taskMessage.Event.EventType.ToString(),
Utils.GetTaskEventId(taskMessage.Event),
sourceInstance.InstanceId,
sourceInstance.ExecutionId,
Encoding.Unicode.GetByteCount(rawContent),
data.QueueName /* PartitionId */,
taskMessage.OrchestrationInstance.InstanceId,
taskMessage.OrchestrationInstance.ExecutionId,
data.SequenceNumber,
data.Episode.GetValueOrDefault(-1),
Utils.ExtensionVersion);
await this.storageQueue.AddMessageAsync(
queueMessage,
null /* timeToLive */,
GetVisibilityDelay(taskMessage),
this.QueueRequestOptions,
session?.StorageOperationContext);
this.stats.MessagesSent.Increment();
// Wake up the queue polling thread
this.backoffHelper.Reset();
}
catch (StorageException e)
{
AnalyticsEventSource.Log.MessageFailure(
this.storageAccountName,
this.settings.TaskHubName,
string.Empty /* MessageId */,
sourceInstance.InstanceId,
sourceInstance.ExecutionId,
this.storageQueue.Name,
taskMessage.Event.EventType.ToString(),
Utils.GetTaskEventId(taskMessage.Event),
e.ToString(),
Utils.ExtensionVersion);
throw;
}
finally
{
this.stats.StorageRequests.Increment();
}
return(data);
}
static async Task <int> Main(string[] args)
{
var builder = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.local.json", optional: true, reloadOnChange: true);
IConfigurationRoot configuration = builder.Build();
dbConnectionString = configuration.GetConnectionString("PlaceboDatabase");
storageConnectionString = configuration["storageConnectionString"];
trainingQueueName = configuration["TrainingQueueName"];
var shortDBConnectionString = dbConnectionString.Substring(0, 50);
var shortStorageConnectionString = storageConnectionString.Substring(0, 50);
Console.WriteLine($"Database connection string = {shortDBConnectionString}");
Console.WriteLine($"Storage connection string = {shortStorageConnectionString}");
Console.WriteLine($"Training Queue Name = {trainingQueueName}");
var rootCommand = new RootCommand
{
new Option <string>(
"--documentFormat",
description: "The document format for which the training assets should be uploaded"),
new Option <string>(
"--localPath",
description: "The local folder containing the training assets"),
new Option <string>(
"--blobContainer",
description: "The name of the blob container where the assets should be uploaded"),
new Option <string>(
"--blobContainerFolder",
getDefaultValue: () => null,
description: "The anme of a folder within the blob container where the assets should be uploaded"),
};
rootCommand.Description = "This command uploads a set of model training assets for a document format (e.g. phoenix) from a local directory to Azure blob storage. " +
"This triggers a model training run in azure. A new model is created based on the assets and a record of the new model is kept in the ModelTraining table in the " +
"database. This new model becomes the latest model for that document format and is then used by the rcognizer component while processing future documents";
try
{
rootCommand.Handler = CommandHandler.Create <string, string, string, string>(async(documentFormat, localPath, blobContainer, blobContainerFolder) =>
{
try
{
Console.WriteLine($"The value for --documentFormat is: {documentFormat}");
if (string.IsNullOrEmpty(documentFormat))
{
throw new Exception($"--documentFormat {documentFormat} must be provided");
}
Console.WriteLine($"The value for --localPath is: {localPath}");
if (string.IsNullOrEmpty(localPath))
{
throw new Exception($"--localPath {localPath} must be provided");
}
Console.WriteLine($"The value for --blobContainer is: {blobContainer}");
if (string.IsNullOrEmpty(blobContainer))
{
throw new Exception($"--blobContainer {blobContainer} must be provided");
}
Console.WriteLine($"The value for --blobContainerFolder is: {blobContainerFolder}");
if (string.IsNullOrEmpty(blobContainerFolder))
{
throw new Exception($"--blobContainerFolder {blobContainerFolder} must be provided");
}
if (!Directory.Exists(localPath))
{
throw new Exception($"--localPath {localPath} does not exist or is not a directory");
}
// Get hold of the storage account
CloudStorageAccount storageAccount = null;
try
{
storageAccount = CloudStorageAccount.Parse(storageConnectionString);
var targetBlobClient = storageAccount.CreateCloudBlobClient();
var targetContainer = targetBlobClient.GetContainerReference(blobContainer);
await targetContainer.CreateIfNotExistsAsync();
var directory = targetContainer.GetDirectoryReference(blobContainerFolder);
BlobResultSegment resultSegment = null;
BlobContinuationToken continuationToken = null;
do
{
resultSegment = await directory.ListBlobsSegmentedAsync(true, BlobListingDetails.All, 50, continuationToken, null, null);
if (resultSegment.Results.Count() > 0)
{
Console.WriteLine($"Container already contains {resultSegment.Results.Count()} blobs - they will be deleted");
}
foreach (var blob in resultSegment.Results)
{
try
{
var blobToDelete = directory.GetBlockBlobReference(blob.Uri.ToString());
await blobToDelete.DeleteIfExistsAsync();
Console.WriteLine($"Deleted blob: {blobToDelete.Name}");
}
catch (Exception e)
{
Console.WriteLine("Unable to delete blob {blob.Uri.ToString()}");
}
}
// Get the continuation token. If not null, get the next segment.
continuationToken = resultSegment.ContinuationToken;
} while (continuationToken != null);
string[] fileEntries = Directory.GetFiles(localPath);
Stopwatch innnerTimer = new Stopwatch();
Stopwatch outerTimer = new Stopwatch();
outerTimer.Start();
int i = 0;
foreach (string fileName in fileEntries)
{
FileInfo f = new FileInfo(fileName);
innnerTimer.Reset();
innnerTimer.Start();
using FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read);
var blobToUpload = directory.GetBlockBlobReference(f.Name);
await blobToUpload.UploadFromStreamAsync(fs);
i++;
innnerTimer.Stop();
Console.WriteLine($"Uploaded file {f.Name} to container {targetContainer.Name} in {innnerTimer.ElapsedMilliseconds} ms");
}
outerTimer.Stop();
Console.WriteLine($"Uploaded {i} files to container {targetContainer.Name} in {outerTimer.ElapsedMilliseconds} ms");
var policy = new SharedAccessBlobPolicy
{
Permissions = SharedAccessBlobPermissions.List | SharedAccessBlobPermissions.Read,
SharedAccessStartTime = DateTime.UtcNow.AddMinutes(-15),
SharedAccessExpiryTime = DateTime.UtcNow.AddMinutes(120)
};
var targetContainerToken = targetContainer.GetSharedAccessSignature(policy);
var targetContainerSAS = string.Format("{0}{1}", targetContainer.Uri, targetContainerToken);
Console.WriteLine($"targetContainerSAS={targetContainerSAS}");
TrainingRequestMessage trainingRequestMessage = new TrainingRequestMessage
{
BlobFolderName = blobContainerFolder,
BlobSasUrl = targetContainerSAS,
DocumentFormat = documentFormat,
IncludeSubFolders = "false",
UseLabelFile = "true"
};
CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient();
// Retrieve a reference to a container.
CloudQueue queue = queueClient.GetQueueReference(trainingQueueName);
// Create the queue if it doesn't already exist
await queue.CreateIfNotExistsAsync();
CloudQueueMessage message = new CloudQueueMessage(JsonConvert.SerializeObject(trainingRequestMessage));
await queue.AddMessageAsync(message);
}
catch (Exception e)
{
throw;
}
Console.WriteLine("done.");
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
}
);
}
catch (Exception e)
{
Console.WriteLine(e.Message);
return(-1);
}
return(rootCommand.InvokeAsync(args).Result);
}
public async Task DeleteMessage(CloudQueue queue, CloudQueueMessage message)
{
await queue.DeleteMessageAsync(message);
}
public async Task EnqueueMapping(Pex2AplosMappingModel mapping, CancellationToken cancellationToken)
{
var message = new CloudQueueMessage(JsonConvert.SerializeObject(mapping));
await Queue.AddMessageAsync(message, cancellationToken);
}
public static async Task Run([TimerTrigger("0 25 * * * *")] TimerInfo myTimer, TraceWriter log)
{
Startup.Init();
log.Info($"C# Timer trigger function executed at: {DateTime.Now}");
BlogToFetchQueueAdapter blogToFetchQueueAdapter = new BlogToFetchQueueAdapter();
blogToFetchQueueAdapter.Init();
BlogInfoTableAdapter blogInfoTableAdapter = new BlogInfoTableAdapter();
blogInfoTableAdapter.Init();
PostsGetter postsGetter = new PostsGetter();
Stopwatch stopwatch = Stopwatch.StartNew();
bool success; // basically means that can the message be deleted, or if it needs to be left in queue to be resumed later
do
{
//TODO: error handling, if there is error from e.g. postsGetter
CloudQueueMessage message = await blogToFetchQueueAdapter.GetNextMessage();
if (message == null)
{
return;
}
BlogToFetch blogToFetch = JsonConvert.DeserializeObject <BlogToFetch>(message.AsString);
BlogEntity blogEntity = await blogInfoTableAdapter.GetBlog(blogToFetch.Blogname);
long timeoutLeft = 270 - stopwatch.ElapsedMilliseconds / 1000;
if (timeoutLeft < 10)
{
return;
}
success = false;
GetPostsResult result = null;
if (blogToFetch.NewerThan.HasValue)
{
result = await postsGetter.GetNewerPosts(log, blogToFetch.Blogname, blogToFetch.NewerThan.Value, timeoutLeft);
if (result.Success)
{
success = true;
}
}
int offset = 0;
if (blogEntity?.FetchedUntilOffset != null && !blogToFetch.UpdateNpf)
{
offset = blogEntity.FetchedUntilOffset.Value;
}
if (result != null)
{
offset += (int)result.TotalReceived;
}
if (blogEntity != null && (!blogEntity.FetchedUntilOffset.HasValue || blogEntity.FetchedUntilOffset.Value < Constants.MaxPostsToFetch))
{
result = await postsGetter.GetPosts(log, blogToFetch.Blogname, offset, timeoutSeconds : timeoutLeft, updateNpf : blogToFetch.UpdateNpf);
if (result.Success)
{
success = true;
}
}
else
{
success = true; // enough fetched already, message can be deleted
}
if (success)
{
await blogToFetchQueueAdapter.DeleteMessage(message);
}
} while (success);
}
public async Task Send(T entity)
{
var message = new CloudQueueMessage(entity.Serialize());
await queue.AddMessageAsync(message);
}
public PendingDelivery(StreamSequenceToken token, CloudQueueMessage message)
{
this.Token = token;
this.Message = message;
}
private void ProcessQueueMessage(CloudQueueMessage msg)
{
Trace.TraceInformation("Processing queue message {0}", msg);
// Queue message contains AdId.
var adId = int.Parse(msg.AsString);
Ad ad = db.Ads.Find(adId);
if (ad == null)
{
throw new Exception(String.Format("AdId {0} not found, can't create thumbnail", adId.ToString()));
}
Uri blobUri = new Uri(ad.ImageURL);
string blobName = blobUri.Segments[blobUri.Segments.Length - 1];
CloudBlockBlob inputBlob = this.imagesBlobContainer.GetBlockBlobReference(blobName);
string thumbnailName = Path.GetFileNameWithoutExtension(inputBlob.Name) + "thumb.jpg";
CloudBlockBlob outputBlob = this.imagesBlobContainer.GetBlockBlobReference(thumbnailName);
using (Stream input = inputBlob.OpenRead())
using (Stream output = outputBlob.OpenWrite())
{
ConvertImageToThumbnailJPG(input, output);
outputBlob.Properties.ContentType = "image/jpeg";
}
Trace.TraceInformation("Generated thumbnail in blob {0}", thumbnailName);
ad.ThumbnailURL = outputBlob.Uri.ToString();
db.SaveChanges();
Trace.TraceInformation("Updated thumbnail URL in database: {0}", ad.ThumbnailURL);
// Remove message from queue.
this.imagesQueue.DeleteMessage(msg);
}
