public static async Task <HttpResponseMessage> Run([HttpTrigger(AuthorizationLevel.Anonymous, "get", "post", Route = null)] HttpRequestMessage req,
[Inject] IMergeManager mergeManager, [Inject] ILogger <IMergeManager> logger)
{
logger.LogInformation("Azure Function ExecuteMerge processed a request.");
try
{
dynamic data = await req.Content.ReadAsAsync <object>();
string datastring = data.ToString();
//this data had better be serializable to TranformResultsContainer
TransformResultsContainer trc = JsonConvert.DeserializeObject <TransformResultsContainer>(datastring);
MergeResultsContainer mrc = mergeManager.MergeAll(trc);
var response = req.CreateResponse(HttpStatusCode.OK, mrc);
return(response);
}
catch (System.Exception ex)
{
logger.LogError(ex, $"Error in Azure Function ExecuteMerge : {ex.Message}");
var response = req.CreateResponse(HttpStatusCode.InternalServerError, ex.Message);
return(response);
}
}
public MergeResultsContainer RetrieveMergeResults()
{
string fileContents = RetrieveLatestBlobContents(_azureBlobStorageOptions.MergeResultsStoragePath);
MergeResultsContainer mergeResultsContainer = JsonConvert.DeserializeObject <MergeResultsContainer>(fileContents);
return(mergeResultsContainer);
}
public MergeResultsContainer RetrieveMergeResults(string path)
{
string fileContents = RetrieveSerializedData(path);
MergeResultsContainer mergeResultsContainer = JsonConvert.DeserializeObject <MergeResultsContainer>(fileContents);
return(mergeResultsContainer);
}
public string StoreMergeResults(MergeResultsContainer mergeResultsContainer, string formattedDateString)
{
string serializedMergeResults = JsonConvert.SerializeObject(mergeResultsContainer);
string datedFilePath = StoreTextAsBlob(_azureBlobStorageOptions.MergeResultsStoragePath, serializedMergeResults, formattedDateString);
return(datedFilePath); //this is the identifier of the blob
}
public string StoreMergeResults(MergeResultsContainer mergeResultsContainer, string formattedDateString)
{
string serializedMergeResults = JsonConvert.SerializeObject(mergeResultsContainer);
string datedFilePath = StoreSerializedData(serializedMergeResults, formattedDateString, _fileMergeStorageOptions.MergeResultsStoragePath);
return(datedFilePath);
}
public MergeResultsContainer RetrieveMergeResults()
{
string fileContents = RetrieveSerializedData(_fileMergeStorageOptions.MergeResultsStoragePath, MergeResultFileFilterExpression);
MergeResultsContainer mergeResultsContainer = JsonConvert.DeserializeObject <MergeResultsContainer>(fileContents);
return(mergeResultsContainer);
}
public MergeResultsContainer RetrieveMergeResults(string path)
{
MergeResultsContainer mergeResultsContainer = new MergeResultsContainer();
CloudBlockBlob blockBlob = _container.GetBlockBlobReference(path);
string fileContents = blockBlob.DownloadTextAsync().Result;
mergeResultsContainer = JsonConvert.DeserializeObject <MergeResultsContainer>(fileContents);
return(mergeResultsContainer);
}
public MergeResultsContainer MergeAll(TransformResultsContainer transformResultsContainer)
{
try
{
MergeResultsContainer mergeResultsContainer = ExecuteMerge(transformResultsContainer);
_logger.LogInformation("MergeManager MergeAll method successfully completed.");
return(mergeResultsContainer);
}
catch (Exception ex)
{
_logger.LogError(ex, ex.Message);
_logger.LogInformation("Error in MergeManager MergeAll method - Merge was NOT completed.");
throw;
}
}
public MergeResultsContainer MergeAll(string transformFilePath)
{
try
{
TransformResultsContainer transformResults = _transformStorageProvider.RetrieveTransformResults(transformFilePath);
MergeResultsContainer mergeResultsContainer = ExecuteMerge(transformResults);
_logger.LogInformation("MergeManager MergeAll method successfully completed.");
return(mergeResultsContainer);
}
catch (Exception ex)
{
_logger.LogError(ex, ex.Message);
_logger.LogInformation("Error in MergeManager MergeAll method - Merge was NOT completed.");
throw;
}
}
public static async Task <HttpResponseMessage> Run([HttpTrigger(AuthorizationLevel.Anonymous, "get", "post", Route = null)] HttpRequestMessage req,
[Inject] IMergeStorageProvider mergeStorageProvider, [Inject] ILogger <IMergeStorageProvider> logger)
{
logger.LogInformation("Azure Function StoreMerge processed a request.");
try
{
// parse query parameter -can be null
var qs = req.RequestUri.ParseQueryString();
string formattedDateString = qs.Get("formattedDateString");
//string formattedDateString = req.GetQueryNameValuePairs()
// .FirstOrDefault(q => string.Compare(q.Key, "formattedDateString", true) == 0)
// .Value;
// Get request body
dynamic data = await req.Content.ReadAsAsync <object>();
string datastring = data.ToString();
//this data had better be serializable to MergeResultsContainer
MergeResultsContainer mrc = JsonConvert.DeserializeObject <MergeResultsContainer>(datastring);
string filePath = mergeStorageProvider.StoreMergeResults(mrc, formattedDateString);
var response = req.CreateResponse(HttpStatusCode.OK, filePath);
return(response);
}
catch (System.Exception ex)
{
logger.LogError(ex, $"Error in Azure Function StoreMerge : {ex.Message}");
var response = req.CreateResponse(HttpStatusCode.InternalServerError, ex.Message);
return(response);
}
}
public void Run()
{
try
{
Stopwatch stopwatchFull = Stopwatch.StartNew();
_logger.LogInformation($"{nameof(Application)}.{nameof(Run)} execution invoked");
_importValidator = _serviceProvider.GetService <IImportValidator>();
_typeListValidator = _serviceProvider.GetService <ITypeListValidator>();
_importManager = _serviceProvider.GetService <IImportManager>();
_importStorageProvider = _serviceProvider.GetService <IImportStorageProvider>();
_transformManager = _serviceProvider.GetService <ITransformManager>();
_transformStorageProvider = _serviceProvider.GetService <ITransformStorageProvider>();
_mergeManager = _serviceProvider.GetService <IMergeManager>();
_mergeStorageProvider = _serviceProvider.GetService <IMergeStorageProvider>();
//uncomment below to actually run import and transform stages - file based
string formattedDateString = DateTime.UtcNow.ToString(DateFormatSpecifier);
//Import
Stopwatch stopwatchImport = Stopwatch.StartNew();
string failureInfo;
//Before we take the overhead of downloading all the import data, check that the data has the right overall structure
bool isDataSourceValid = _importValidator.TryValidateDataSource(out failureInfo);
if (!isDataSourceValid)
{
_logger.LogWarning("Enlir Import Data not in Expected Format: \n" + failureInfo);
throw new ValidationException("Enlir Import Data not in Expected Format: \n" + failureInfo);
}
ImportResultsContainer importResultsContainer = _importManager.ImportAll();
string importStoragePath = _importStorageProvider.StoreImportResults(importResultsContainer, formattedDateString);
stopwatchImport.Stop();
//cheat data setup for testing - comment out when doing full run for real
//string importStoragePath = @"D:\Temp\FFRKApi\ImportResults-2018-12-21_09-48-46.json";
//string transformStoragePath = @"D:\Docs\Personal\FFRKLinqQuery\TransformResults-Latest.json";
//string formattedDateString = "2018-12-21_09-48-46";
//string importContents = File.ReadAllText(importStoragePath);
//ImportResultsContainer importResultsContainer = JsonConvert.DeserializeObject<ImportResultsContainer>(importContents);
////Now that we have the import data, we need to check whether our TypeLists (used to convert staring data into ids)
////is still accurate. If the source data has changed their list of values for each type, we need to stop and correct the TypeLists
IEnumerable <TypeListDifferences> typeListDifferences = _typeListValidator.TryValidateTypeLists(importResultsContainer);
if (typeListDifferences.Any(t => t.IsIdListDifferentFromSource))
{
_logger.LogWarning("Enlir TypeList Data differs from coded TypeLists.");
//write validation failure data to log for easy perusal
string typeListDifferencesLogPath = $"{AppContext.BaseDirectory}\\TypeListDifferencesLog.json";
string typeListDifferencesLogContents = JsonConvert.SerializeObject(typeListDifferences);
File.WriteAllText(typeListDifferencesLogPath, typeListDifferencesLogContents);
_logger.LogWarning("Enlir TypeList differences written to file: " + typeListDifferencesLogPath);
throw new ValidationException("Enlir Type List Data differs from coded TypeLists");
}
//Transform
Stopwatch stopwatchTransform = Stopwatch.StartNew();
TransformResultsContainer transformResultsContainer = _transformManager.TransformAll(importStoragePath);
string transformStoragePath = _transformStorageProvider.StoreTransformResults(transformResultsContainer, formattedDateString);
stopwatchTransform.Stop();
//Merge
Stopwatch stopwatchMerge = Stopwatch.StartNew();
MergeResultsContainer mergeResultsContainer = _mergeManager.MergeAll(transformStoragePath);
string mergeStoragePath = _mergeStorageProvider.StoreMergeResults(mergeResultsContainer, formattedDateString);
stopwatchMerge.Stop();
//test merge storage
MergeResultsContainer testMergeResultsContainer = _mergeStorageProvider.RetrieveMergeResults(mergeStoragePath);
stopwatchFull.Stop();
_logger.LogInformation("Import Completed in {ImportTime} seconds", stopwatchImport.Elapsed.Seconds);
_logger.LogInformation("Transform Completed in {TransformTime} seconds", stopwatchTransform.Elapsed.Seconds);
_logger.LogInformation("Merge Completed in {MergeTime} seconds", stopwatchMerge.Elapsed.Seconds);
_logger.LogInformation("Full Run Completed in {FullRunTime} seconds", stopwatchFull.Elapsed.Seconds);
int aggregateTime = stopwatchImport.Elapsed.Seconds + stopwatchMerge.Elapsed.Seconds + stopwatchFull.Elapsed.Seconds;
_logger.LogInformation("Full Run Aggregate Time in {AggregateTime} seconds", aggregateTime);
}
catch (Exception ex)
{
_logger.LogError(ex, ex.Message);
_logger.LogInformation("Error in Top Level Application execution. Validate, Import, Transform, and Merge operations were NOT successfully completed. Previously existing data is unchanged");
throw;
}
}
private MergeResultsContainer ExecuteMerge(TransformResultsContainer transformResults)
{
MergeResultsContainer mergeResultsContainer = new MergeResultsContainer();
//Id Wireup
WireUpLegendMateriaIds(transformResults);
_logger.LogInformation("Finished wiring up LegendMateriaIds");
WireUpCharacterIds(transformResults);
_logger.LogInformation("Finished wiring up CharacterIds");
WireUpSoulBreakIds(transformResults);
_logger.LogInformation("Finished wiring up SoulBreakIds");
WireUpRelicIds(transformResults);
_logger.LogInformation("Finished wiring up RelicIds");
WireUpMagiciteIds(transformResults);
_logger.LogInformation("Finished wiring up MagiciteIds");
WireUpEventIds(transformResults);
_logger.LogInformation("Finished wiring up EventIds");
WireUpOtherSourceInfo(transformResults);
_logger.LogInformation("Finished wiring up OtherSourceInfo");
//Object Wireup
WireUpMagiciteSkills(transformResults);
_logger.LogInformation("Finished wiring up MagiciteSkills");
WireUpRecordSpheres(transformResults);
_logger.LogInformation("Finished wiring up RecordSpheres");
WireUpLegendSpheres(transformResults);
_logger.LogInformation("Finished wiring up LegendSpheres");
WireUpRecordMaterias(transformResults);
_logger.LogInformation("Finished wiring up RecordMaterias");
WireUpLegendMaterias(transformResults);
_logger.LogInformation("Finished wiring up LegendMaterias");
WireUpCommands(transformResults);
_logger.LogInformation("Finished wiring up Commands");
WireUpBraveActions(transformResults);
_logger.LogInformation("Finished wiring up BraveActions");
WireUpStatuses(transformResults);
_logger.LogInformation("Finished wiring up Statuses");
WireUpOthers(transformResults);
_logger.LogInformation("Finished wiring up Others");
WireUpSoulBreaks(transformResults);
_logger.LogInformation("Finished wiring up SoulBreaks");
WireUpRelics(transformResults);
_logger.LogInformation("Finished wiring up Relics");
//Value Wireup/Calculation
CalculateLegendSphereTier(transformResults);
_logger.LogInformation("Finished calculating up LegendSphereTier");
_logger.LogInformation("Finished MergeAll Operation");
//copy data from transformResults container to merge results container
mergeResultsContainer.Abilities = transformResults.Abilities;
mergeResultsContainer.Characters = transformResults.Characters;
mergeResultsContainer.Commands = transformResults.Commands;
mergeResultsContainer.BraveActions = transformResults.BraveActions;
//mergeResultsContainer.Dungeons = transformResults.Dungeons;
mergeResultsContainer.Events = transformResults.Events;
mergeResultsContainer.Experiences = transformResults.Experiences;
mergeResultsContainer.LegendMaterias = transformResults.LegendMaterias;
mergeResultsContainer.LegendSpheres = transformResults.LegendSpheres;
mergeResultsContainer.MagiciteSkills = transformResults.MagiciteSkills;
mergeResultsContainer.Magicites = transformResults.Magicites;
mergeResultsContainer.Missions = transformResults.Missions;
mergeResultsContainer.Others = transformResults.Others;
mergeResultsContainer.RecordMaterias = transformResults.RecordMaterias;
mergeResultsContainer.RecordSpheres = transformResults.RecordSpheres;
mergeResultsContainer.Relics = transformResults.Relics;
mergeResultsContainer.SoulBreaks = transformResults.SoulBreaks;
mergeResultsContainer.Statuses = transformResults.Statuses;
//now add in list members
mergeResultsContainer.AbilityTypeList = new AbilityTypeList().TypeList;
mergeResultsContainer.AutoTargetTypeList = new AutoTargetTypeList().TypeList;
mergeResultsContainer.DamageFormulaTypeList = new DamageFormulaTypeList().TypeList;
mergeResultsContainer.ElementList = new ElementList().TypeList;
mergeResultsContainer.EquipmentTypeList = new EquipmentTypeList().TypeList;
mergeResultsContainer.EventTypeList = new EventTypeList().TypeList;
mergeResultsContainer.MissionTypeList = new MissionTypeList().TypeList;
mergeResultsContainer.OrbTypeList = new OrbTypeList().TypeList;
mergeResultsContainer.RealmList = new RealmList().TypeList;
mergeResultsContainer.RelicTypeList = new RelicTypeList().TypeList;
mergeResultsContainer.SchoolList = new SchoolList().TypeList;
mergeResultsContainer.StatSetTypeList = new StatSetTypeList().TypeList;
mergeResultsContainer.StatTypeList = new StatTypeList().TypeList;
mergeResultsContainer.SoulBreakTierList = new SoulBreakTierList().TypeList;
mergeResultsContainer.TargetTypeList = new TargetTypeList().TypeList;
//now add in model lookup lists
mergeResultsContainer.EventIdList = mergeResultsContainer.Events.Select(i => new KeyValuePair <int, string>(i.Id, i.EventName)).ToList();
mergeResultsContainer.MissionList = mergeResultsContainer.Missions.Select(i => new KeyValuePair <int, string>(i.Id, i.Description)).ToList();
mergeResultsContainer.ExperienceIdList = mergeResultsContainer.Experiences.Select(i => new KeyValuePair <int, string>(i.Id, i.Description)).ToList();
//mergeResultsContainer.DungeonIdList = mergeResultsContainer.Dungeons.Select(i => new KeyValuePair<int, string>(i.Id, i.DungeonName)).ToList();
mergeResultsContainer.MagiciteSkillIdList = mergeResultsContainer.MagiciteSkills.Select(i => new KeyValuePair <int, string>(i.Id, i.Description)).ToList();
mergeResultsContainer.MagiciteIdList = mergeResultsContainer.Magicites.Select(i => new KeyValuePair <int, string>(i.Id, i.MagiciteName)).ToList();
mergeResultsContainer.StatusIdList = mergeResultsContainer.Statuses.Select(i => new KeyValuePair <int, string>(i.Id, i.CommonName)).ToList();
mergeResultsContainer.OtherIdList = mergeResultsContainer.Others.Select(i => new KeyValuePair <int, string>(i.Id, i.Name)).ToList();
mergeResultsContainer.CommandIdList = mergeResultsContainer.Commands.Select(i => new KeyValuePair <int, string>(i.Id, i.Description)).ToList();
mergeResultsContainer.BraveActionIdList = mergeResultsContainer.BraveActions.Select(i => new KeyValuePair <int, string>(i.Id, i.Description)).ToList();
mergeResultsContainer.SoulBreakIdList = mergeResultsContainer.SoulBreaks.Select(i => new KeyValuePair <int, string>(i.Id, i.SoulBreakName)).ToList();
mergeResultsContainer.RelicIdList = mergeResultsContainer.Relics.Select(i => new KeyValuePair <int, string>(i.Id, i.Description)).ToList();
mergeResultsContainer.AbilityIdList = mergeResultsContainer.Abilities.Select(i => new KeyValuePair <int, string>(i.Id, i.AbilityName)).ToList();
mergeResultsContainer.LegendMateriaIdList = mergeResultsContainer.LegendMaterias.Select(i => new KeyValuePair <int, string>(i.Id, i.Description)).ToList();
mergeResultsContainer.RecordMateriaIdList = mergeResultsContainer.RecordMaterias.Select(i => new KeyValuePair <int, string>(i.Id, i.Description)).ToList();
mergeResultsContainer.RecordSphereIdList = mergeResultsContainer.RecordSpheres.Select(i => new KeyValuePair <int, string>(i.Id, i.Description)).ToList();
mergeResultsContainer.LegendSphereIdList = mergeResultsContainer.LegendSpheres.Select(i => new KeyValuePair <int, string>(i.Id, i.Description)).ToList();
mergeResultsContainer.CharacterIdList = mergeResultsContainer.Characters.Select(i => new KeyValuePair <int, string>(i.Id, i.CharacterName)).ToList();
return(mergeResultsContainer);
}
public void LoadMergeResultsContainer()
{
_mergeResultsContainer = _mergeStorageProvider.RetrieveMergeResults();
}
