private List <QueueMetadata> RunTask(QueueMetadata metadata, Task task)
{
List <QueueMetadata> result = null;
JObject metadataJson = JObject.FromObject(metadata);
JObject taskpropertiesJson = JUST.JsonTransformer.Transform((JObject)task.TaskProperties.DeepClone(), metadataJson);
JObject taskProperties = taskpropertiesJson;
string identifier = task.Identifier;
if (task.Type == TaskType.Splitter)
{
result = Splitter.RunTask(metadata, taskProperties, identifier);
}
else if (task.Type == TaskType.Transformer)
{
result = Transformer.RunTask(metadata, taskProperties, identifier, _orchestration);
}
else
{
result = RESTConnector.RunTask(metadata, taskProperties, identifier);
}
return(result);
}
/// <summary>
/// Returns information about the connected queue
/// </summary>
/// <returns></returns>
public QueueMetadata GetQueueInfo()
{
var count = RabbitMQHelper.UseConnectionWithRetries(c => c.MessageCount(_queueName), _channel);
if (count > int.MaxValue)
{
// This scenario seems unlikely enough that I don't feel like coding around it. Are we ever going to have queues
// with more than two billion messages at a time? We can cross that bridge when we come to it
throw new InvalidOperationException("Count of messages in the queue is greater than a 32bit signed integer.");
}
var result = new QueueMetadata((int)count, _queueName);
return(result);
}
public async Task <IActionResult> Upload(List <IFormFile> files)
{
var response = new UploadResponse {
Success = true
};
this.Response.StatusCode = (int)HttpStatusCode.OK;
string operationId = System.Diagnostics.Activity.Current?.RootId ?? "Could not pull Operation Id";
foreach (var formFile in files)
{
var uploadSuccess = false;
if (formFile.Length > 0)
{
string remoteFilePath = null;
using (var stream = formFile.OpenReadStream())
{
(uploadSuccess, remoteFilePath) = await this.cloudProvider.StorageProvider().UploadFile(formFile.FileName, stream);
var metadata = new QueueMetadata {
Fileuri = remoteFilePath, OperationId = operationId
};
uploadSuccess = uploadSuccess && await this.cloudProvider.QueueProvider().Enqueue(metadata);
}
}
this.LogFileUploadStatistics(formFile, operationId, uploadSuccess);
if (uploadSuccess == false)
{
response.Success = false;
break;
}
}
if (response.Success == false)
{
this.Response.StatusCode = (int)HttpStatusCode.BadRequest;
}
return(this.Json(response));
}
private void PublishToQueue(Models.Queue currentQueue, QueueMetadata currentMetadata, QueueMetadata result, string runTask, bool workflowCompleted,
bool active)
{
if (result == null)
{
_orchestration.PublishWorkflowStep(currentQueue.Type.Name, currentQueue.WorkflowID, null, null,
runTask, true, workflowCompleted, 0, active, runTask, null);
}
else
{
if (result.Success)
{
_orchestration.PublishWorkflowStep(currentQueue.Type.Name, currentQueue.WorkflowID, result.MessageBodyJson, result.CustomPropertiesJson,
runTask, true, workflowCompleted, 0, active, runTask, result.SplitID);
}
else
{
_orchestration.PublishWorkflowStep(currentQueue.Type.Name, currentQueue.WorkflowID, currentMetadata.MessageBodyJson, result.CustomPropertiesJson,
currentMetadata.StepIdentifier, false, workflowCompleted, currentMetadata.Retries + 1, active, runTask, currentMetadata.SplitID);
}
}
}
/// <summary>
/// Assertions in this method are based on MessageProcessor which picks
/// up an item from the queue and writes a file back to storage
/// </summary>
/// <param name="times">number of times to retry</param>
/// <returns>If queue item was processed</returns>
private async Task <bool> MessageIsProcessedInQueueAsync(int times)
{
HttpResponseMessage response;
int attempts;
(response, attempts) = await this.Retry(times, new TimeSpan(5 * TimeSpan.TicksPerSecond), this.DownloadQueueResponseAsync);
if (response == null)
{
return(false);
}
var actual = await this.ExtractResponseContent(response);
QueueMetadata data = JsonConvert.DeserializeObject <QueueMetadata>(actual);
if (data == null)
{
return(false);
}
Console.WriteLine("Queue message proccessed after {0} retries", attempts);
return(!string.IsNullOrEmpty(data.Fileuri));
}
public void BecauseOf()
{
var queueMetadata = new[]
{
new QueueMetadataNameValuePair {Name = "test1", Value = "value1"},
new QueueMetadataNameValuePair {Name = "test2", Value = "value2"},
};
_azureQueueService.PutQueueMetadata(_queueUri, queueMetadata);
_queueMetadata = _azureQueueService.GetQueueMetadata(_queueUri);
}
public void BecauseOf()
{
_queueMetadata = _azureQueueService.GetQueueMetadata(_queueUri);
}
public static List <QueueMetadata> RunTask(QueueMetadata metadata, JObject taskProperties, string identifier, IOrchestration orchestration)
{
List <QueueMetadata> result = null;
try
{
if (taskProperties == null)
{
throw new Exception("TaskProperties for Transformer cannot be NULL");
}
TransformerConfiguration taskConfig = taskProperties.ToObject <TransformerConfiguration>();
JObject transformerJson = orchestration.GetTransformer(taskConfig.TransformerID);
QueueMetadata transformedMetadata = new QueueMetadata()
{
CustomPropertiesJson = metadata.CustomPropertiesJson,
ID = Guid.NewGuid(),
MessageBodyJson = metadata.MessageBodyJson,
Retries = metadata.Retries,
SplitID = metadata.SplitID,
StepIdentifier = identifier,
Success = true,
Timestamp = DateTime.UtcNow
};
transformedMetadata.MessageBodyJson = JUST.JsonTransformer.Transform(transformerJson, metadata.MessageBodyJson);
result = new List <QueueMetadata>();
result.Add(transformedMetadata);
}
catch (Exception ex)
{
result = new List <QueueMetadata>();
QueueMetadata errorMetaData = new QueueMetadata()
{
CustomPropertiesJson = metadata.CustomPropertiesJson,
ID = Guid.NewGuid(),
MessageBodyJson = metadata.MessageBodyJson,
Retries = metadata.Retries,
SplitID = metadata.SplitID,
StepIdentifier = identifier,
Success = false,
Timestamp = DateTime.UtcNow
};
Dictionary <string, string> customPropertiesSerialized = errorMetaData.CustomPropertiesJson.ToObject <Dictionary <string, string> >();
if (customPropertiesSerialized.ContainsKey("Error"))
{
customPropertiesSerialized.Remove("Error");
}
customPropertiesSerialized.Add("Error", ex.Message);
errorMetaData.CustomPropertiesJson = JObject.FromObject(customPropertiesSerialized);
result.Add(errorMetaData);
}
return(result);
}
public static List <QueueMetadata> RunTask(QueueMetadata metadata, JObject taskProperties, string identifier)
{
List <QueueMetadata> result = null;
try
{
if (taskProperties == null)
{
throw new Exception("TaskProperties for Splitter cannot be NULL");
}
SplitterConfiguration taskConfig = taskProperties.ToObject <SplitterConfiguration>();
IEnumerable <JObject> messages = JUST.JsonTransformer.SplitJson(metadata.MessageBodyJson, taskConfig.ArrayPath);
result = new List <QueueMetadata>();
int i = 0;
foreach (JObject message in messages)
{
QueueMetadata splitMetadata = new QueueMetadata()
{
CustomPropertiesJson = metadata.CustomPropertiesJson,
ID = Guid.NewGuid(),
MessageBodyJson = message,
Retries = metadata.Retries,
SplitID = metadata.SplitID + "." + i.ToString(),
StepIdentifier = identifier,
Success = true,
Timestamp = DateTime.UtcNow
};
i++;
result.Add(splitMetadata);
}
}
catch (Exception ex)
{
result = new List <QueueMetadata>();
QueueMetadata errorMetaData = new QueueMetadata()
{
CustomPropertiesJson = metadata.CustomPropertiesJson,
ID = Guid.NewGuid(),
MessageBodyJson = metadata.MessageBodyJson,
Retries = metadata.Retries,
SplitID = metadata.SplitID,
StepIdentifier = identifier,
Success = false,
Timestamp = DateTime.UtcNow
};
Dictionary <string, string> customPropertiesSerialized = errorMetaData.CustomPropertiesJson.ToObject <Dictionary <string, string> >();
if (customPropertiesSerialized.ContainsKey("Error"))
{
customPropertiesSerialized.Remove("Error");
}
customPropertiesSerialized.Add("Error", ex.Message);
errorMetaData.CustomPropertiesJson = JObject.FromObject(customPropertiesSerialized);
result.Add(errorMetaData);
}
return(result);
}
public static List <QueueMetadata> RunTask(QueueMetadata metadata, JObject taskProperties, string identifier)
{
List <QueueMetadata> result = null;
try
{
if (taskProperties == null)
{
throw new Exception("TaskProperties for Transformer cannot be NULL");
}
RESTConfig taskConfig = taskProperties.ToObject <RESTConfig>();
QueueMetadata newMetadata = new QueueMetadata()
{
CustomPropertiesJson = metadata.CustomPropertiesJson,
ID = Guid.NewGuid(),
MessageBodyJson = metadata.MessageBodyJson,
Retries = metadata.Retries,
SplitID = metadata.SplitID,
StepIdentifier = identifier,
Success = true,
Timestamp = DateTime.UtcNow
};
HttpClient client = new HttpClient();
client.BaseAddress = new Uri(taskConfig.Url);
if (taskConfig.Headers != null)
{
foreach (KeyValuePair <string, string> header in taskConfig.Headers)
{
client.DefaultRequestHeaders.TryAddWithoutValidation(header.Key, header.Value);
}
}
client.DefaultRequestHeaders.Accept.Clear();
client.DefaultRequestHeaders.Accept.Add(new System.Net.Http.Headers.MediaTypeWithQualityHeaderValue("application/json"));
HttpContent content = new StringContent(JsonConvert.SerializeObject(taskConfig.Body), Encoding.UTF8, "application/json");
Task <HttpResponseMessage> response = null;
if (taskConfig.Method.ToLower() == "get")
{
response = client.GetAsync(taskConfig.QueryString);
}
else if (taskConfig.Method.ToLower() == "put")
{
response = client.PutAsync(taskConfig.QueryString, content);
}
else if (taskConfig.Method.ToLower() == "post")
{
response = client.PostAsync(taskConfig.QueryString, content);
}
else if (taskConfig.Method.ToLower() == "delete")
{
response = client.DeleteAsync(taskConfig.QueryString);
}
else
{
throw new Exception("Method : " + taskConfig.Method + " not implemented in RESTConnector.");
}
response.Wait();
if (response.Result.IsSuccessStatusCode)
{
Task <string> resultString = response.Result.Content.ReadAsStringAsync();
resultString.Wait();
JObject jObj = metadata.MessageBodyJson;
try
{
JObject token = JObject.Parse(resultString.Result);
JProperty property = jObj.Property(identifier);
if (property == null)
{
jObj.Add(new JProperty(identifier, token));
}
}
catch (JsonReaderException ex)
{
JProperty property = jObj.Property(identifier);
if (property == null)
{
jObj.Add(new JProperty(identifier, resultString.Result));
}
}
newMetadata.MessageBodyJson = jObj;
}
else
{
throw new Exception("Response from RESTService : " + response.Result.StatusCode.ToString() + " : " + response.Result.ReasonPhrase);
}
result = new List <QueueMetadata>();
result.Add(newMetadata);
}
catch (Exception ex)
{
result = new List <QueueMetadata>();
QueueMetadata errorMetaData = new QueueMetadata()
{
CustomPropertiesJson = metadata.CustomPropertiesJson,
ID = Guid.NewGuid(),
MessageBodyJson = metadata.MessageBodyJson,
Retries = metadata.Retries,
SplitID = metadata.SplitID,
StepIdentifier = identifier,
Success = false,
Timestamp = DateTime.UtcNow
};
Dictionary <string, string> customPropertiesSerialized = errorMetaData.CustomPropertiesJson.ToObject <Dictionary <string, string> >();
if (customPropertiesSerialized.ContainsKey("Error"))
{
customPropertiesSerialized.Remove("Error");
}
customPropertiesSerialized.Add("Error", ex.Message);
errorMetaData.CustomPropertiesJson = JObject.FromObject(customPropertiesSerialized);
result.Add(errorMetaData);
}
return(result);
}
/// <summary>
/// Processes item by writing a new file
/// </summary>
/// <param name="item">Queue item</param>
/// <returns>Async task</returns>
internal async Task Process(QueueMetadata item)
{
await this.cloud.StorageProvider().UploadFile(
Constants.NotifyFilename,
new MemoryStream(Encoding.ASCII.GetBytes(JsonConvert.SerializeObject(item))));
}
public List <Queue> Enqueue()
{
string tableName = Constants.QueueTableName;
Table table = Table.LoadTable(_client, tableName);
ScanFilter scanFilter = new ScanFilter();
scanFilter.AddCondition(Constants.QueueActive, ScanOperator.Equal, "True");
Search search = table.Scan(scanFilter);
List <Queue> list = new List <Queue>();
string queueID = null;
JObject businessLogicJson = null;
string workflowTypeName = null;
Guid workflowID = Guid.Empty;
JObject customPropertiesJson = null;
JObject messageBodyJson = null;
string stepIdentifier = null;
bool stepSucceeded = true;
int retries = 0;
DateTime timeStamp = DateTime.UtcNow;
string runStepIdentifier = null;
string splitID = null;
do
{
Task <List <Document> > documentListTask = search.GetNextSetAsync();
documentListTask.Wait();
List <Document> documentList = documentListTask.Result;
foreach (Document document in documentList)
{
queueID = document[Constants.QueueId].AsString();
businessLogicJson = JObject.Parse(document[Constants.BusinessLogic].AsDocument().ToJson());
workflowTypeName = document[Constants.WorkflowType].AsString();
workflowID = document[Constants.HistoryWorkflowId].AsGuid();
if (document.ContainsKey(Constants.QueueCustomProperties))
{
customPropertiesJson = JObject.Parse(document[Constants.QueueCustomProperties].AsDocument().ToJson());
}
if (document.ContainsKey(Constants.QueueMessageBody))
{
messageBodyJson = JObject.Parse(document[Constants.QueueMessageBody].AsDocument().ToJson());
}
if (document.ContainsKey(Constants.QueueStepIdentifier))
{
stepIdentifier = document[Constants.QueueStepIdentifier].AsString();
}
if (document.ContainsKey(Constants.QueueRunStepIdentifier))
{
runStepIdentifier = document[Constants.QueueRunStepIdentifier].AsString();
}
if (document.ContainsKey(Constants.QueueSlitID))
{
splitID = document[Constants.QueueSlitID].AsString();
}
if (document.ContainsKey(Constants.QueueRetries))
{
retries = document[Constants.QueueRetries].AsInt();
}
if (document.ContainsKey(Constants.QueueStepSucceeded))
{
stepSucceeded = document[Constants.QueueStepSucceeded].AsBoolean();
}
if (document.ContainsKey(Constants.Timestamp))
{
timeStamp = Convert.ToDateTime(document[Constants.Timestamp].AsString());
}
QueueMetadata metadata = new QueueMetadata()
{
CustomPropertiesJson = customPropertiesJson,
ID = Guid.Parse(queueID),
MessageBodyJson = messageBodyJson,
SplitID = splitID,
StepIdentifier = stepIdentifier,
Success = stepSucceeded,
Retries = retries,
Timestamp = timeStamp
};
Queue queue = GetQueueByWorkflowID(list, workflowID.ToString());
if (queue == null)
{
queue = new Queue()
{
Type = new WorkfowType()
{
BusinessLogicJson = businessLogicJson,
Name = workflowTypeName
},
WorkflowID = workflowID,
Metadata = new List <QueueMetadata>()
}
}
;
queue.Metadata.Add(metadata);
list.Add(queue);
DeleteFromQueue(queueID, true);
}
} while (!search.IsDone);
return(list);
}
public List <Queue> Enqueue()
{
List <Queue> instance = null;
MySqlConnection connection = null;
try
{
connection = new MySqlConnection(_connectionString);
MySqlCommand command = new MySqlCommand("Orchestration_Enqueue", connection);
command.CommandType = CommandType.StoredProcedure;
connection.Open();
MySqlDataReader reader = command.ExecuteReader();
Queue queue = null;
while (reader.Read())
{
if (instance == null)
{
instance = new List <Queue>();
}
string workflowID = reader.GetString("idWorkflow");
if (queue == null || queue.WorkflowID == null || queue.WorkflowID.ToString().ToLower() != workflowID.ToLower())
{
if (queue != null)
{
instance.Add(queue);
}
queue = new Queue();
queue.WorkflowID = Guid.Parse(workflowID);
queue.Type = new WorkfowType()
{
ID = reader.GetInt32("idWorkflowType"),
Name = reader.GetString("WorkflowTypeName"),
BusinessLogicJson = JObject.Parse(reader.GetString("BusinessLogic"))
};
queue.Metadata = new List <QueueMetadata>();
}
QueueMetadata queueMetadata = new QueueMetadata()
{
CustomPropertiesJson = !reader.IsDBNull(3) ? JObject.Parse(reader.GetString(3)) : null,
ID = reader.GetGuid("QueueID"),
MessageBodyJson = !reader.IsDBNull(7) ? JObject.Parse(reader.GetString(7)) : null,
Retries = reader.GetInt32("Retries"),
SplitID = reader.GetString("SplitID"),
StepIdentifier = !reader.IsDBNull(5) ? reader.GetString(5) : null,
Success = reader.GetBoolean("Success"),
Timestamp = reader.GetDateTime("Timestamp")
};
queue.Metadata.Add(queueMetadata);
}
if (instance != null)
{
instance.Add(queue);
}
}
catch (Exception ex)
{
throw ex;
}
finally
{
if (connection.State == ConnectionState.Open)
{
connection.Close();
}
}
return(instance);
}
