public async Task <IActionResult> SubmitILR(IFormFile file, bool IsShredAndProcess)
{
ViewData["Message"] = "Your application description page.";
CloudStorageAccount cloudStorageAccount = CloudStorageAccount.Parse($"DefaultEndpointsProtocol=https;AccountName={StorageAccountName};AccountKey={StorageAccountKey};EndpointSuffix=core.windows.net");
CloudBlobClient cloudBlobClient = cloudStorageAccount.CreateCloudBlobClient();
CloudBlobContainer cloudBlobContainer = cloudBlobClient.GetContainerReference("ilr-files");
var newFileName = $"{Path.GetFileNameWithoutExtension(file.FileName)}-{GetRandomCharacters(5)}.xml";
CloudBlockBlob cloudBlockBlob = cloudBlobContainer.GetBlockBlobReference(newFileName);
using (var outputStream = await cloudBlockBlob.OpenWriteAsync())
{
await file.CopyToAsync(outputStream);
}
//write it into a queue
var correlationId = Guid.NewGuid();
var model = new IlrContext()
{
CorrelationId = correlationId,
ContainerReference = "ilr-files",
Filename = newFileName,
//Filename = $"ILR-10006341-1718-20171107-113456-04.xml",
IsShredAndProcess = IsShredAndProcess
};
await _serviceBusQueueHelper.SendMessagesAsync(JsonConvert.SerializeObject(model), GetRandomCharacters(8));
return(RedirectToAction("Confirmation", new { correlationId }));
}
public async Task RunAsync()
{
try
{
CloudStorageAccount cloudStorageAccount = CloudStorageAccount.Parse(StorageConnectionString);
CloudBlobClient cloudBlobClient = cloudStorageAccount.CreateCloudBlobClient();
var container = cloudBlobClient.GetContainerReference("ilr-files");
var blobs = container.ListBlobs(useFlatBlobListing: true);
var blobNames = blobs.OfType <CloudBlockBlob>().Select(b => b.Name).ToList();
var queueClient = new QueueClient(ServiceBusConnectionString, _queueName);
var isShredAndProcess = true;
int counter = 1;
foreach (var blobName in blobNames.Where(x => x.Contains("ILR-10006341-1718-20180118-023456-1600")))
{
var correlationId = Guid.NewGuid();
var model = new IlrContext()
{
CorrelationId = correlationId,
ContainerReference = "ilr-files",
Filename = blobName,
//Filename = $"ILR-10006341-1718-20171107-113456-04.xml",
IsShredAndProcess = true
};
isShredAndProcess = !isShredAndProcess;
var message = new Message(Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(model)));
message.SessionId = Guid.NewGuid().ToString();
// Write the body of the message to the console.
Console.WriteLine($"Url: http://dcsflarge.westeurope.cloudapp.azure.com/Home/Status?correlationId={correlationId.ToString()}");
// Send the message to the queue.
await queueClient.ScheduleMessageAsync(message, DateTimeOffset.Parse("12/03/2018 15:42:00 PM"));
//await queueClient.SendAsync(message);
counter++;
if (counter == 41)
{
break;
}
}
await queueClient.CloseAsync();
}
catch (Exception exception)
{
Console.WriteLine($"{DateTime.Now} :: Exception: {exception.Message}");
throw;
}
}
// POST: api/Validation
public IEnumerable <string> Post([FromBody] IlrContext ilrContext)
{
try
{
var startDateTime = DateTime.Now;
Message message = new Message();
//try
//{
var stopwatch = new Stopwatch();
stopwatch.Start();
string xml;
CloudStorageAccount cloudStorageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString"));
var cloudStorageAccountElapsed = stopwatch.ElapsedMilliseconds;
CloudBlobClient cloudBlobClient = cloudStorageAccount.CreateCloudBlobClient();
var cloudBlobClientElapsed = stopwatch.ElapsedMilliseconds;
CloudBlobContainer cloudBlobContainer = cloudBlobClient.GetContainerReference(ilrContext.ContainerReference);
var cloudBlobContainerElapsed = stopwatch.ElapsedMilliseconds;
CloudBlockBlob cloudBlockBlob = cloudBlobContainer.GetBlockBlobReference(ilrContext.Filename);
var cloudBlockBlobElapsed = stopwatch.ElapsedMilliseconds;
xml = cloudBlockBlob.DownloadText();
var blob = stopwatch.ElapsedMilliseconds;
stopwatch.Restart();
using (var reader = XmlReader.Create(new StringReader(xml)))
{
var serializer = new XmlSerializer(typeof(Message));
message = serializer.Deserialize(reader) as Message;
}
var deserialize = stopwatch.ElapsedMilliseconds;
stopwatch.Restart();
var results = _validationService.Validate(message).ToList();
var validate = stopwatch.ElapsedMilliseconds;
return(new List <string>()
{
string.Format("Validation API Request Start Time : {0}", startDateTime.ToString("d/M/yyyy H:mm:ss.fff")),
string.Format("Errors : {0}", results.Count()),
string.Format("Blob Client : {0}", cloudBlobClientElapsed),
string.Format("Blob Container : {0}", cloudBlobContainerElapsed),
string.Format("Blob Block Blob : {0}", cloudBlockBlobElapsed),
string.Format("Blob Download Text : {0}", blob),
string.Format("Deserialize ms : {0}", deserialize),
string.Format("Validation ms : {0}", validate),
string.Format("Validation API Request End Time : {0}", DateTime.Now.ToString("d/M/yyyy H:mm:ss.fff")),
});
}
catch (Exception ex)
{
return(new List <string>()
{
ex.Message
});
}
}
/// <summary>
/// This is the main entry point for your service instance.
/// </summary>
/// <param name="cancellationToken">Canceled when Service Fabric needs to shut down this service instance.</param>
//protected override async Task RunAsync(CancellationToken cancellationToken)
//{
// // TODO: Replace the following sample code with your own logic
// // or remove this RunAsync override if it's not needed in your service.
// long iterations = 0;
// while (true)
// {
// cancellationToken.ThrowIfCancellationRequested();
// ServiceEventSource.Current.ServiceMessage(this.Context, "Working-{0}", ++iterations);
// await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
// }
//}
public async Task <bool> Validate(IlrContext ilrContext, IValidationService validationService)
{
using (var logger = ESFA.DC.Logging.LoggerManager.CreateDefaultLogger(ilrContext.CorrelationId.ToString()))
{
var startDateTime = DateTime.Now;
logger.LogInfo($"Validation started for:{ilrContext.Filename} at :{startDateTime}");
Message message = new Message();
//try
//{
var stopwatch = new Stopwatch();
stopwatch.Start();
string xml;
CloudStorageAccount cloudStorageAccount =
CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString"));
var cloudStorageAccountElapsed = stopwatch.ElapsedMilliseconds;
CloudBlobClient cloudBlobClient = cloudStorageAccount.CreateCloudBlobClient();
var cloudBlobClientElapsed = stopwatch.ElapsedMilliseconds;
CloudBlobContainer cloudBlobContainer =
cloudBlobClient.GetContainerReference(ilrContext.ContainerReference);
var cloudBlobContainerElapsed = stopwatch.ElapsedMilliseconds;
CloudBlockBlob cloudBlockBlob = cloudBlobContainer.GetBlockBlobReference(ilrContext.Filename);
var cloudBlockBlobElapsed = stopwatch.ElapsedMilliseconds;
xml = cloudBlockBlob.DownloadText();
var blob = stopwatch.ElapsedMilliseconds;
stopwatch.Restart();
using (var reader = XmlReader.Create(new StringReader(xml)))
{
var serializer = new XmlSerializer(typeof(Message));
message = serializer.Deserialize(reader) as Message;
}
var deserialize = stopwatch.ElapsedMilliseconds;
stopwatch.Restart();
IEnumerable <LearnerValidationError> results;
var totalLearners = message.Learner.Count();
if (ilrContext.IsShredAndProcess)
{
// create actors here.
results = DivideAndConquer(ilrContext.CorrelationId, message, logger);
}
else
{
results = await validationService.Validate(message);
}
var validate = stopwatch.ElapsedMilliseconds;
var endTime = DateTime.Now;
var processTimes = new List <string>()
{
string.Format("Start Time : {0}", startDateTime),
string.Format("Learners : {0}", totalLearners),
string.Format("Errors : {0}", results.Count()),
string.Format("Blob Client : {0}", cloudBlobClientElapsed),
string.Format("Blob Container : {0}", cloudBlobContainerElapsed),
string.Format("Blob Block Blob : {0}", cloudBlockBlobElapsed),
string.Format("Blob Download Text : {0}", blob),
string.Format("Deserialize ms : {0}", deserialize),
string.Format("Validation ms : {0}", validate),
string.Format("End Time : {0}", endTime),
string.Format("Total Time : {0}", (endTime - startDateTime).TotalMilliseconds),
};
stopwatch.Restart();
//store the results in reliable dictionary
await SaveResultsInDataService(ilrContext.CorrelationId, processTimes, xml);
logger.LogInfo("Stateless Validation Results:{@processTimes}", processTimes.ToArray(), "",
ilrContext.Filename);
var saveResultsTime = stopwatch.ElapsedMilliseconds;
logger.LogInfo($"saved Results to Db in: {saveResultsTime}");
stopwatch.Restart();
//send message topic to be picked up by next service
dynamic data = new { To = _fundingCalcSqlFilterValue };
var pubMessage =
new BrokeredMessage(new MemoryStream(Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(data))))
{
ContentType = "application/json",
Label = data.To,
CorrelationId = ilrContext.CorrelationId.ToString(),
MessageId = Guid.NewGuid().ToString(),
Properties =
{
{ "To", data.To },
{ "fileName", ilrContext.Filename }
},
TimeToLive = TimeSpan.FromMinutes(2)
};
await _topicHelper.SendMessage(pubMessage);
var pushedToTopicTime = stopwatch.ElapsedMilliseconds;
logger.LogInfo($"pushed message into Topic in: {pushedToTopicTime} , {data}");
return(true);
}
}
public async Task <bool> Validate(IlrContext ilrContext)
{
using (var childLifeTimeScope = _parentLifeTimeScope.BeginLifetimeScope("childLifeTimeScope"))
{
var validationService = childLifeTimeScope.Resolve <IValidationService>();
var startDateTime = DateTime.Now;
Message message = new Message();
//try
//{
var stopwatch = new Stopwatch();
stopwatch.Start();
string xml;
CloudStorageAccount cloudStorageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString"));
var cloudStorageAccountElapsed = stopwatch.ElapsedMilliseconds;
CloudBlobClient cloudBlobClient = cloudStorageAccount.CreateCloudBlobClient();
var cloudBlobClientElapsed = stopwatch.ElapsedMilliseconds;
CloudBlobContainer cloudBlobContainer = cloudBlobClient.GetContainerReference(ilrContext.ContainerReference);
var cloudBlobContainerElapsed = stopwatch.ElapsedMilliseconds;
CloudBlockBlob cloudBlockBlob = cloudBlobContainer.GetBlockBlobReference(ilrContext.Filename);
var cloudBlockBlobElapsed = stopwatch.ElapsedMilliseconds;
xml = cloudBlockBlob.DownloadText();
var blob = stopwatch.ElapsedMilliseconds;
stopwatch.Restart();
using (var reader = XmlReader.Create(new StringReader(xml)))
{
var serializer = new XmlSerializer(typeof(Message));
message = serializer.Deserialize(reader) as Message;
}
var deserialize = stopwatch.ElapsedMilliseconds;
stopwatch.Restart();
IEnumerable <LearnerValidationError> results;
if (ilrContext.IsShredAndProcess)
{
// create actors here.
results = DivideAndConquer(message);
}
else
{
results = validationService.Validate(message);
}
var validate = stopwatch.ElapsedMilliseconds;
var processTimes = new List <string>()
{
string.Format("Start Time : {0}", startDateTime),
string.Format("Errors : {0}", results.Count()),
string.Format("Blob Client : {0}", cloudBlobClientElapsed),
string.Format("Blob Container : {0}", cloudBlobContainerElapsed),
string.Format("Blob Block Blob : {0}", cloudBlockBlobElapsed),
string.Format("Blob Download Text : {0}", blob),
string.Format("Deserialize ms : {0}", deserialize),
string.Format("Validation ms : {0}", validate)
};
ServiceEventSource.Current.ServiceMessage(this.Context, "result-{0}", processTimes);
//store the results in reliable dictionary
var history = await StateManager.GetOrAddAsync <IReliableDictionary <Guid, List <string> > >("history");
using (var tx = StateManager.CreateTransaction())
{
await history.AddOrUpdateAsync(tx, ilrContext.CorrelationId, processTimes, (id, oldValue) => processTimes);
await tx.CommitAsync();
}
return(true);
}
}