private TransformResultsContainer ExecuteTransform(ImportResultsContainer importResults)
{
TransformResultsContainer transformResultsContainer = new TransformResultsContainer();
transformResultsContainer.Events = _eventRowTransformer.Transform(importResults.EventRows);
_logger.LogInformation("finished transforming EventRows to Events");
transformResultsContainer.Missions = _missionRowTransformer.Transform(importResults.MissionRows);
_logger.LogInformation("finished transforming MissionRows to Missions");
transformResultsContainer.Experiences = _experienceRowTransformer.Transform(importResults.ExperienceRows);
_logger.LogInformation("finished transforming ExperienceRows to a single Experience object in a list");
//transformResultsContainer.Dungeons = _dungeonRowTransformer.Transform(importResults.DungeonRows);
//_logger.LogInformation("finished transforming DungeonRows to Dungeons");
transformResultsContainer.MagiciteSkills = _magiciteSkillRowTransformer.Transform(importResults.MagiciteSkillRows.Where(ms => !String.IsNullOrWhiteSpace(ms.Name)));
_logger.LogInformation("finished transforming MagiciteSkillRows to MagiciteSkills");
transformResultsContainer.Magicites = _magiciteRowTransformer.Transform(importResults.MagiciteRows);
_logger.LogInformation("finished transforming MagiciteRows to Magicites");
transformResultsContainer.Statuses = _statusRowTransformer.Transform(importResults.StatusRows);
_logger.LogInformation("finished transforming StatusRows to Statuses");
transformResultsContainer.Others = _otherRowTransformer.Transform(importResults.OtherRows);
_logger.LogInformation("finished transforming OtherRows to Others");
transformResultsContainer.Commands = _commandRowTransformer.Transform(importResults.CommandRows);
_logger.LogInformation("finished transforming CommandRows to Commands");
transformResultsContainer.BraveActions = _braveActionRowTransformer.Transform(importResults.BraveActionRows);
_logger.LogInformation("finished transforming BraveActionRows to BraveActions");
transformResultsContainer.SoulBreaks = _soulBreakRowTransformer.Transform(importResults.SoulBreakRows);
_logger.LogInformation("finished transforming SoulBreakRows to SoulBreaks");
transformResultsContainer.Relics = _relicRowTransformer.Transform(importResults.RelicRows);
_logger.LogInformation("finished transforming RelicRows to Relics");
transformResultsContainer.Abilities = _abilityRowTransformer.Transform(importResults.AbilityRows);
_logger.LogInformation("finished transforming AbilityRows to Abilities");
transformResultsContainer.LegendMaterias = _legendMateriaRowTransformer.Transform(importResults.LegendMateriaRows);
_logger.LogInformation("finished transforming LegendMateriaRows to LegendMaterias");
transformResultsContainer.RecordMaterias = _recordMateriaRowTransformer.Transform(importResults.RecordMateriaRows);
_logger.LogInformation("finished transforming RecordMateriaRows to RecordMaterias");
transformResultsContainer.RecordSpheres = _recordSphereRowTransformer.Transform(importResults.RecordSphereRows);
_logger.LogInformation("finished transforming RecordMateriaRows to RecordMaterias");
transformResultsContainer.LegendSpheres = _legendSphereRowTransformer.Transform(importResults.LegendSphereRows);
_logger.LogInformation("finished transforming LegendSphereRows to LegendSpheres");
transformResultsContainer.Characters = _characterRowTransformer.Transform(importResults.CharacterRows);
_logger.LogInformation("finished transforming CharacterRows to Characters");
return(transformResultsContainer);
}
private TypeListDifferences ValidateTargetTypes(ImportResultsContainer irc)
{
var abilityTargetTypes = irc.AbilityRows.Select(ar => ar.Target.Split(",".ToCharArray())).
SelectMany(el => el).Select(s => s.Trim()).Distinct().OrderBy(e => e).ToList();
var commandTargetTypes = irc.CommandRows.Select(ar => ar.Target.Split(",".ToCharArray())).
SelectMany(el => el).Select(s => s.Trim()).Distinct().OrderBy(e => e).ToList();
var otherTargetTypes = irc.OtherRows.Where(ar => !String.IsNullOrWhiteSpace(ar.Type)).Select(ar => ar.Target.Split(",".ToCharArray())).
SelectMany(el => el).Select(s => s.Trim()).Distinct().OrderBy(e => e).ToList();
var soulBreakTargetTypes = irc.SoulBreakRows.Select(ar => ar.Target.Split(",".ToCharArray())).
SelectMany(el => el).Select(s => s.Trim()).Distinct().OrderBy(e => e).ToList();
var limitBreakTargetTypes = irc.LimitBreakRows.Select(ar => ar.Target.Split(",".ToCharArray())).
SelectMany(el => el).Select(s => s.Trim()).Distinct().OrderBy(e => e).ToList();
//TargetTypes - source
var unifiedTargetTypesList = abilityTargetTypes.Union(commandTargetTypes).Union(otherTargetTypes).Union(soulBreakTargetTypes).
Union(limitBreakTargetTypes).OrderBy(e => e);
//TargetTypes - id list
var targetTypesIdList = new TargetTypeList();
var targetTypesIdListTargetTypeNames = targetTypesIdList.TypeList.Where(il => il.Key != 0).Select(kvp => kvp.Value).ToList();
TypeListDifferences targetTypesDifferences = new TypeListDifferences()
{
IdListName = nameof(TargetTypeList),
ValuesMissingFromIdList = unifiedTargetTypesList.Except(targetTypesIdListTargetTypeNames).ToList(),
ValuesSuperfluousInIdList = targetTypesIdListTargetTypeNames.Except(unifiedTargetTypesList).ToList(),
SuggestedIdListContents = GenerateSuggestedIdListContents(unifiedTargetTypesList)
};
return(targetTypesDifferences);
}
public static async Task <HttpResponseMessage> Run(
[HttpTrigger(AuthorizationLevel.Anonymous, "get", "post", Route = null)] HttpRequestMessage req, TraceWriter log,
[Inject] ITransformManager transformManager, [Inject] ILogger <ITransformManager> logger)
{
logger.LogInformation("Azure Function ExecuteTransform processed a request.");
try
{
dynamic data = await req.Content.ReadAsAsync <object>();
string datastring = data.ToString();
//this data had better be serializable to ImportResultsContainer
ImportResultsContainer irc = JsonConvert.DeserializeObject <ImportResultsContainer>(datastring);
TransformResultsContainer trc = transformManager.TransformAll(irc);
var response = req.CreateResponse(HttpStatusCode.OK, trc);
return(response);
}
catch (System.Exception ex)
{
logger.LogError(ex, $"Error in Azure Function ExecuteTransform : {ex.Message}");
var response = req.CreateResponse(HttpStatusCode.InternalServerError, ex.Message);
return(response);
}
}
public ImportResultsContainer RetrieveImportResults(string path)
{
string fileContents = RetrieveSerializedData(path);
ImportResultsContainer importResultsContainer = JsonConvert.DeserializeObject <ImportResultsContainer>(fileContents);
return(importResultsContainer);
}
public ImportResultsContainer RetrieveImportResults()
{
string fileContents = RetrieveSerializedData(_fileImportStorageOptions.ImportResultsStoragePath, ImportResultFileFilterExpression);
ImportResultsContainer importResultsContainer = JsonConvert.DeserializeObject <ImportResultsContainer>(fileContents);
return(importResultsContainer);
}
public ImportResultsContainer RetrieveImportResults()
{
string fileContents = RetrieveLatestBlobContents(_azureBlobStorageOptions.ImportResultsStoragePath);
ImportResultsContainer importResultsContainer = JsonConvert.DeserializeObject <ImportResultsContainer>(fileContents);
return(importResultsContainer);
}
public string StoreImportResults(ImportResultsContainer importResultsContainer, string formattedDateString)
{
string serializedImportResults = JsonConvert.SerializeObject(importResultsContainer);
string datedFilePath = StoreSerializedData(serializedImportResults, formattedDateString, _fileImportStorageOptions.ImportResultsStoragePath);
return(datedFilePath);
}
public string StoreImportResults(ImportResultsContainer importResultsContainer, string formattedDateString)
{
string serializedImportResults = JsonConvert.SerializeObject(importResultsContainer);
string datedFilePath = StoreTextAsBlob(_azureBlobStorageOptions.ImportResultsStoragePath, serializedImportResults, formattedDateString);
return(datedFilePath); //this is the identifier of the blob
}
private TypeListDifferences ValidateRealms(ImportResultsContainer irc)
{
var characterRealms = irc.CharacterRows.Select(ar => ar.Realm.Split(",".ToCharArray())).
SelectMany(el => el).Select(s => s.Trim()).Distinct().OrderBy(e => e).ToList();
//var dungeonRealms = irc.DungeonRows.Select(ar => ar.Realm.Split(",".ToCharArray())).
// SelectMany(el => el).Select(s => s.Trim()).Distinct().OrderBy(e => e).ToList();
var eventRealms = irc.EventRows.Select(ar => ar.Realm.Split(",".ToCharArray())).
SelectMany(el => el).Select(s => s.Trim()).Distinct().OrderBy(e => e).ToList();
var legendMateriaRealms = irc.LegendMateriaRows.Select(ar => ar.Realm.Split(",".ToCharArray())).
SelectMany(el => el).Select(s => s.Trim()).Distinct().OrderBy(e => e).ToList();
var legendSphereRealms = irc.LegendSphereRows.Select(ar => ar.Realm.Split(",".ToCharArray())).
SelectMany(el => el).Select(s => s.Trim()).Distinct().OrderBy(e => e).ToList();
var magiciteRealms = irc.MagiciteRows.Select(ar => ar.Realm.Split(",".ToCharArray())).
SelectMany(el => el).Select(s => s.Trim()).Distinct().OrderBy(e => e).ToList();
var recordMateriaRealms = irc.RecordMateriaRows.Select(ar => ar.Realm.Split(",".ToCharArray())).
SelectMany(el => el).Select(s => s.Trim()).Distinct().OrderBy(e => e).ToList();
var recordSphereRealms = irc.RecordSphereRows.Select(ar => ar.Realm.Split(",".ToCharArray())).
SelectMany(el => el).Select(s => s.Trim()).Distinct().OrderBy(e => e).ToList();
var relicRealms = irc.RelicRows.Select(ar => ar.Realm.Split(",".ToCharArray())).
SelectMany(el => el).Select(s => s.Trim()).Distinct().OrderBy(e => e).ToList();
var soulBreakRealms = irc.SoulBreakRows.Select(ar => ar.Realm.Split(",".ToCharArray())).
SelectMany(el => el).Select(s => s.Trim()).Distinct().OrderBy(e => e).ToList();
var limitBreakRealms = irc.LimitBreakRows.Select(ar => ar.Realm.Split(",".ToCharArray())).
SelectMany(el => el).Select(s => s.Trim()).Distinct().OrderBy(e => e).ToList();
//Realms - source
var unifiedRealmsList = characterRealms.Union(eventRealms).
Union(legendMateriaRealms).Union(legendSphereRealms).Union(magiciteRealms).
Union(recordMateriaRealms).Union(recordSphereRealms).Union(relicRealms).Union(soulBreakRealms).
Union(limitBreakRealms).OrderBy(e => e);
//Realms - id list
var realmList = new RealmList();
var realmListRealmNames = realmList.TypeList.Where(il => il.Key != 0).Select(kvp => kvp.Value).ToList();
TypeListDifferences realmDifferences = new TypeListDifferences()
{
IdListName = nameof(RealmList),
ValuesMissingFromIdList = unifiedRealmsList.Except(realmListRealmNames).ToList(),
ValuesSuperfluousInIdList = realmListRealmNames.Except(unifiedRealmsList).ToList(),
SuggestedIdListContents = GenerateSuggestedIdListContents(unifiedRealmsList)
};
return(realmDifferences);
}
public ImportResultsContainer RetrieveImportResults(string path)
{
ImportResultsContainer importResultsContainer = new ImportResultsContainer();
CloudBlockBlob blockBlob = _container.GetBlockBlobReference(path);
string fileContents = blockBlob.DownloadTextAsync().Result;
importResultsContainer = JsonConvert.DeserializeObject <ImportResultsContainer>(fileContents);
return(importResultsContainer);
}
public TransformResultsContainer TransformAll(ImportResultsContainer importResultsContainer)
{
try
{
TransformResultsContainer transformResultsContainer = ExecuteTransform(importResultsContainer);
return(transformResultsContainer);
}
catch (Exception ex)
{
_logger.LogError(ex, ex.Message);
_logger.LogInformation("Error in TransformManager TransformAll method - transform was NOT completed.");
throw;
}
}
public static async Task <HttpResponseMessage> Run([HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = null)] HttpRequestMessage req,
[Inject] ITypeListValidator typeListValidator, [Inject] ILogger <ITypeListValidator> logger)
{
//IEnumerable<TypeListDifferences> typeListDifferencesForError = null;
logger.LogInformation("Azure Function ValidateTypeLists processed a request.");
try
{
HttpResponseMessage response = null;
dynamic data = await req.Content.ReadAsAsync <object>();
string datastring = data.ToString();
//this data had better be serializable to ImportResultsContainer
ImportResultsContainer irc = JsonConvert.DeserializeObject <ImportResultsContainer>(datastring);
IList <TypeListDifferences> typeListDifferences = typeListValidator.TryValidateTypeLists(irc).ToList();
if (typeListDifferences.Any(t => t.IsIdListDifferentFromSource))
{
logger.LogWarning("Enlir TypeList Data differs from coded TypeLists.");
//typeListDifferencesForError = typeListDifferences;
//throw new Exception("Enlir Type List Data differs from coded TypeLists");
response = req.CreateResponse(HttpStatusCode.OK, typeListDifferences);
}
else
{
response = req.CreateResponse(HttpStatusCode.OK, true);
}
return(response);
}
catch (System.Exception ex)
{
logger.LogError(ex, $"Error in Azure Function ValidateTypeLists : {ex.Message}");
var response = req.CreateResponse(HttpStatusCode.InternalServerError, ex.Message);
return(response);
}
}
public IEnumerable <TypeListDifferences> TryValidateTypeLists(ImportResultsContainer importResultsContainer)
{
IList <TypeListDifferences> validationResults = new List <TypeListDifferences>();
validationResults.Add(ValidateAbilityTypes(importResultsContainer));
validationResults.Add(ValidateAutoTargetTypes(importResultsContainer));
validationResults.Add(ValidateDamageFormulaTypes(importResultsContainer));
validationResults.Add(ValidateElements(importResultsContainer));
validationResults.Add(ValidateEventTypes(importResultsContainer));
validationResults.Add(ValidateMissionTypes(importResultsContainer));
validationResults.Add(ValidateOrbTypes(importResultsContainer));
validationResults.Add(ValidateRealms(importResultsContainer));
validationResults.Add(ValidateRelicTypes(importResultsContainer));
validationResults.Add(ValidateSchools(importResultsContainer));
validationResults.Add(ValidateSoulBreakTiers(importResultsContainer));
validationResults.Add(ValidateTargetTypes(importResultsContainer));
return(validationResults);
}
public TransformResultsContainer TransformAll(string importFilePath)
{
try
{
ImportResultsContainer importResults = _importStorageProvider.RetrieveImportResults(importFilePath);
TransformResultsContainer transformResultsContainer = ExecuteTransform(importResults);
return(transformResultsContainer);
}
catch (Exception ex)
{
_logger.LogError(ex, ex.Message);
_logger.LogInformation("Error in TransformManager TransformAll method - transform was NOT completed.");
throw;
}
}
private TypeListDifferences ValidateElements(ImportResultsContainer irc)
{
var abilityElements = irc.AbilityRows.Select(ar => ar.Element.Split(",".ToCharArray())).
SelectMany(el => el).Select(s => s.Trim()).Distinct().OrderBy(e => e).ToList();
var commandElements = irc.CommandRows.Select(ar => ar.Element.Split(",".ToCharArray())).
SelectMany(el => el).Select(s => s.Trim()).Distinct().OrderBy(e => e).ToList();
var magiciteElements = irc.MagiciteRows.Select(ar => ar.Element.Split(",".ToCharArray())).
SelectMany(el => el).Select(s => s.Trim()).Distinct().OrderBy(e => e).ToList();
var magiciteSkillElements = irc.MagiciteSkillRows.Where(ar => !String.IsNullOrWhiteSpace(ar.Type)).Select(ar => ar.Element.Split(",".ToCharArray())).
SelectMany(el => el).Select(s => s.Trim()).Distinct().OrderBy(e => e).ToList();
var otherElements = irc.OtherRows.Where(ar => !String.IsNullOrWhiteSpace(ar.Type)).Select(ar => ar.Element.Split(",".ToCharArray())).
SelectMany(el => el).Select(s => s.Trim()).Distinct().OrderBy(e => e).ToList();
var soulBreakElements = irc.SoulBreakRows.Select(ar => ar.Element.Split(",".ToCharArray())).
SelectMany(el => el).Select(s => s.Trim()).Distinct().OrderBy(e => e).ToList();
var limitBreakElements = irc.LimitBreakRows.Select(ar => ar.Element.Split(",".ToCharArray())).
SelectMany(el => el).Select(s => s.Trim()).Distinct().OrderBy(e => e).ToList();
//elements - source
var unifiedElementsList = abilityElements.Union(commandElements).Union(magiciteElements).
Union(magiciteSkillElements).Union(otherElements).Union(soulBreakElements).Union(limitBreakElements).OrderBy(e => e);
//elements - id list
var elementIdList = new ElementList();
var elementIdListElementNames = elementIdList.TypeList.Where(il => il.Key != 0).Select(kvp => kvp.Value).ToList();
TypeListDifferences elementDifferences = new TypeListDifferences()
{
IdListName = nameof(ElementList),
ValuesMissingFromIdList = unifiedElementsList.Except(elementIdListElementNames).ToList(),
ValuesSuperfluousInIdList = elementIdListElementNames.Except(unifiedElementsList).ToList(),
SuggestedIdListContents = GenerateSuggestedIdListContents(unifiedElementsList)
};
return(elementDifferences);
}
private TypeListDifferences ValidateLimitBreakTiers(ImportResultsContainer irc)
{
var limitBreakSoulBreakTiers = irc.LimitBreakRows.Select(m => m.Tier.Trim()).Distinct().OrderBy(e => e).ToList();
//SoulBreakTiers - source
var unifiedLimitBreakTierList = limitBreakSoulBreakTiers.OrderBy(e => e);
//SoulBreakTiers - id list
var limitBreakTierList = new LimitBreakTierList();
var limitBreakTierListNames = limitBreakTierList.TypeList.Where(il => il.Key != 0).Select(kvp => kvp.Value).ToList();
TypeListDifferences limitBreakTierDifferences = new TypeListDifferences()
{
IdListName = nameof(LimitBreakTierList),
ValuesMissingFromIdList = unifiedLimitBreakTierList.Except(limitBreakTierListNames).ToList(),
ValuesSuperfluousInIdList = limitBreakTierListNames.Except(unifiedLimitBreakTierList).ToList(),
SuggestedIdListContents = GenerateSuggestedIdListContents(unifiedLimitBreakTierList)
};
return(limitBreakTierDifferences);
}
private TypeListDifferences ValidateRelicTypes(ImportResultsContainer irc)
{
var relicRelicTypes = irc.RelicRows.Select(m => m.Type.Trim()).Distinct().OrderBy(e => e).ToList();
//RelicTypes - source
var unifiedRelicTypeList = relicRelicTypes.OrderBy(e => e);
//RelicTypes - id list
var relicTypeList = new RelicTypeList();
var relicTypeListNames = relicTypeList.TypeList.Where(il => il.Key != 0).Select(kvp => kvp.Value).ToList();
TypeListDifferences relicTypeDifferences = new TypeListDifferences()
{
IdListName = nameof(RelicTypeList),
ValuesMissingFromIdList = unifiedRelicTypeList.Except(relicTypeListNames).ToList(),
ValuesSuperfluousInIdList = relicTypeListNames.Except(unifiedRelicTypeList).ToList(),
SuggestedIdListContents = GenerateSuggestedIdListContents(unifiedRelicTypeList)
};
return(relicTypeDifferences);
}
private TypeListDifferences ValidateEventTypes(ImportResultsContainer irc)
{
var eventEventTypes = irc.EventRows.Select(ar => ar.Type.Split(",".ToCharArray())).
SelectMany(el => el).Select(s => s.Trim()).Distinct().OrderBy(e => e).ToList();
//events - source
var unifiedEventTypeList = eventEventTypes.OrderBy(e => e);
//events - id list
var eventTypeList = new EventTypeList();
var eventTypeListEventNames = eventTypeList.TypeList.Where(il => il.Key != 0).Select(kvp => kvp.Value).ToList();
TypeListDifferences eventTypeDifferences = new TypeListDifferences()
{
IdListName = nameof(EventTypeList),
ValuesMissingFromIdList = unifiedEventTypeList.Except(eventTypeListEventNames).ToList(),
ValuesSuperfluousInIdList = eventTypeListEventNames.Except(unifiedEventTypeList).ToList(),
SuggestedIdListContents = GenerateSuggestedIdListContents(unifiedEventTypeList)
};
return(eventTypeDifferences);
}
private TypeListDifferences ValidateDamageFormulaTypes(ImportResultsContainer irc)
{
var abilityDamageFormulaTypes = irc.AbilityRows.Select(ar => ar.Formula.Split(",".ToCharArray())).
SelectMany(el => el).Select(s => s.Trim()).Distinct().OrderBy(e => e).ToList();
var commandDamageFormulaTypes = irc.CommandRows.Select(ar => ar.Formula.Split(",".ToCharArray())).
SelectMany(el => el).Select(s => s.Trim()).Distinct().OrderBy(e => e).ToList();
var magiciteDamageFormulaTypes = irc.MagiciteRows.Select(ar => ar.Formula.Split(",".ToCharArray())).
SelectMany(el => el).Select(s => s.Trim()).Distinct().OrderBy(e => e).ToList();
var magiciteSkillDamageFormulaTypes = irc.MagiciteSkillRows.Where(ar => !String.IsNullOrWhiteSpace(ar.Type)).Select(ar => ar.Formula.Split(",".ToCharArray())).
SelectMany(el => el).Select(s => s.Trim()).Distinct().OrderBy(e => e).ToList();
var otherDamageFormulaTypes = irc.OtherRows.Where(ar => !String.IsNullOrWhiteSpace(ar.Type)).Select(ar => ar.Formula.Split(",".ToCharArray())).
SelectMany(el => el).Select(s => s.Trim()).Distinct().OrderBy(e => e).ToList();
var soulBreakDamageFormulaTypes = irc.SoulBreakRows.Select(ar => ar.Formula.Split(",".ToCharArray())).
SelectMany(el => el).Select(s => s.Trim()).Distinct().OrderBy(e => e).ToList();
//DamageFormulaTypes - source
var unifiedDamageFormulaTypesList = abilityDamageFormulaTypes.Union(commandDamageFormulaTypes).Union(magiciteDamageFormulaTypes).
Union(magiciteSkillDamageFormulaTypes).Union(otherDamageFormulaTypes).Union(soulBreakDamageFormulaTypes).OrderBy(e => e);
//DamageFormulaTypes - id list
var damageFormulaTypesIdList = new DamageFormulaTypeList();
var damageFormulaTypesIdListDamageFormulaTypeNames = damageFormulaTypesIdList.TypeList.Where(il => il.Key != 0).Select(kvp => kvp.Value).ToList();
TypeListDifferences damageFormulaTypesDifferences = new TypeListDifferences()
{
IdListName = nameof(DamageFormulaTypeList),
ValuesMissingFromIdList = unifiedDamageFormulaTypesList.Except(damageFormulaTypesIdListDamageFormulaTypeNames).ToList(),
ValuesSuperfluousInIdList = damageFormulaTypesIdListDamageFormulaTypeNames.Except(unifiedDamageFormulaTypesList).ToList(),
SuggestedIdListContents = GenerateSuggestedIdListContents(unifiedDamageFormulaTypesList)
};
return(damageFormulaTypesDifferences);
}
public static async Task <HttpResponseMessage> Run([HttpTrigger(AuthorizationLevel.Anonymous, "get", "post", Route = null)] HttpRequestMessage req,
[Inject] IImportStorageProvider importStorageProvider, [Inject] ILogger <IImportStorageProvider> logger)
{
logger.LogInformation("Azure Function StoreImport processed a request.");
try
{
// parse query parameter -can be null
var qs = req.RequestUri.ParseQueryString();
string formattedDateString = qs.Get("formattedDateString");
//string formattedDateString = req.GetQueryNameValuePairs()
// .FirstOrDefault(q => string.Compare(q.Key, "formattedDateString", true) == 0)
// .Value;
// Get request body
dynamic data = await req.Content.ReadAsAsync <object>();
string datastring = data.ToString();
//this data had better be serializable to ImportResultsContainer
ImportResultsContainer irc = JsonConvert.DeserializeObject <ImportResultsContainer>(datastring);
string filePath = importStorageProvider.StoreImportResults(irc, formattedDateString);
var response = req.CreateResponse(HttpStatusCode.OK, filePath);
return(response);
}
catch (System.Exception ex)
{
logger.LogError(ex, $"Error in Azure Function StoreImport : {ex.Message}");
var response = req.CreateResponse(HttpStatusCode.InternalServerError, ex.Message);
return(response);
}
}
public static async Task <HttpResponseMessage> Run(
[HttpTrigger(AuthorizationLevel.Anonymous, "get", "post", Route = null)] HttpRequestMessage req,
[Inject] IImportManager importManager, [Inject] ILogger <IImportManager> logger)
{
logger.LogInformation("Azure Function ExecuteImport processed a request.");
try
{
ImportResultsContainer irc = importManager.ImportAll();
var response = req.CreateResponse(HttpStatusCode.OK, irc);
return(response);
}
catch (System.Exception ex)
{
logger.LogError(ex, $"Error in Azure Function ExecuteImport : {ex.Message}");
var response = req.CreateResponse(HttpStatusCode.InternalServerError, ex.Message);
return(response);
}
}
private TypeListDifferences ValidateOrbTypes(ImportResultsContainer irc)
{
var abilityOrb1Types = irc.AbilityRows.Select(a => a.Orb1RequiredType.Trim()).Distinct().OrderBy(e => e).ToList();
var abilityOrb2Types = irc.AbilityRows.Select(a => a.Orb2RequiredType.Trim()).Distinct().OrderBy(e => e).ToList();
var abilityOrb3Types = irc.AbilityRows.Select(a => a.Orb3RequiredType.Trim()).Distinct().OrderBy(e => e).ToList();
var abilityOrb4Types = irc.AbilityRows.Where(ar => !String.IsNullOrWhiteSpace(ar.Orb4RequiredType)).Select(a => a.Orb4RequiredType.Trim()).Distinct().OrderBy(e => e).ToList();
//OrbTypes - source
var unifiedOrbTypeList = abilityOrb1Types.Union(abilityOrb2Types).Union(abilityOrb3Types).Union(abilityOrb4Types).OrderBy(e => e);
//OrbTypes - id list
var orbTypeList = new OrbTypeList();
var orbTypeListOrbNames = orbTypeList.TypeList.Where(il => il.Key != 0).Select(kvp => kvp.Value).ToList();
TypeListDifferences orbTypeDifferences = new TypeListDifferences()
{
IdListName = nameof(OrbTypeList),
ValuesMissingFromIdList = unifiedOrbTypeList.Except(orbTypeListOrbNames).ToList(),
ValuesSuperfluousInIdList = orbTypeListOrbNames.Except(unifiedOrbTypeList).ToList(),
SuggestedIdListContents = GenerateSuggestedIdListContents(unifiedOrbTypeList)
};
return(orbTypeDifferences);
}
public ImportResultsContainer ImportAll()
{
try
{
ImportResultsContainer resultsContainer = new ImportResultsContainer();
resultsContainer.CharacterRows = _characterImporter.Import();
_logger.LogInformation("finished loading {RowCount} CharacterRows", resultsContainer.CharacterRows?.Count());
resultsContainer.RecordSphereRows = _recordSphereImporter.Import();
_logger.LogInformation("finished loading {RowCount} RecordSphereRows", resultsContainer.RecordSphereRows?.Count());
resultsContainer.LegendSphereRows = _legendSphereImporter.Import();
_logger.LogInformation("finished loading {RowCount} LegendSphereRows", resultsContainer.LegendSphereRows?.Count());
resultsContainer.RecordMateriaRows = _recordMateriaImporter.Import();
_logger.LogInformation("finished loading {RowCount} RecordMateriaRows", resultsContainer.RecordMateriaRows?.Count());
resultsContainer.LegendMateriaRows = _legendMateriaImporter.Import();
_logger.LogInformation("finished loading {RowCount} LegendMateriaRows", resultsContainer.LegendMateriaRows?.Count());
resultsContainer.AbilityRows = _abilityImporter.Import();
_logger.LogInformation("finished loading {RowCount} AbilityRows", resultsContainer.AbilityRows?.Count());
resultsContainer.SoulBreakRows = _soulBreakImporter.Import();
_logger.LogInformation("finished loading {RowCount} SoulBreakRows", resultsContainer.SoulBreakRows?.Count());
resultsContainer.CommandRows = _commandImporter.Import();
_logger.LogInformation("finished loading {RowCount} CommandRows", resultsContainer.CommandRows?.Count());
resultsContainer.BraveActionRows = _braveActionImporter.Import();
_logger.LogInformation("finished loading {RowCount} BraveActionRows", resultsContainer.BraveActionRows?.Count());
resultsContainer.OtherRows = _otherImporter.Import();
_logger.LogInformation("finished loading {RowCount} OtherRows", resultsContainer.OtherRows?.Count());
resultsContainer.StatusRows = _statusImporter.Import();
_logger.LogInformation("finished loading {RowCount} StatusRows", resultsContainer.StatusRows?.Count());
resultsContainer.RelicRows = _relicImporter.Import();
_logger.LogInformation("finished loading {RowCount} RelicRows", resultsContainer.RelicRows?.Count());
resultsContainer.MagiciteRows = _magiciteImporter.Import();
_logger.LogInformation("finished loading {RowCount} MagiciteRows", resultsContainer.MagiciteRows?.Count());
resultsContainer.MagiciteSkillRows = _magiciteSkillImporter.Import();
_logger.LogInformation("finished loading {RowCount} MagiciteSkillRows", resultsContainer.MagiciteSkillRows?.Count());
//resultsContainer.DungeonRows = _dungeonImporter.Import();
//_logger.LogInformation("finished loading {RowCount} DungeonRows", resultsContainer.DungeonRows?.Count());
resultsContainer.EventRows = _eventImporter.Import();
_logger.LogInformation("finished loading {RowCount} EventRows", resultsContainer.EventRows?.Count());
resultsContainer.MissionRows = _missionImporter.Import();
_logger.LogInformation("finished loading {RowCount} MissionRows", resultsContainer.MissionRows?.Count());
resultsContainer.ExperienceRows = _experienceImporter.Import();
_logger.LogInformation("finished loading {RowCount} ExperienceRows", resultsContainer.ExperienceRows?.Count());
return(resultsContainer);
}
catch (Exception ex)
{
_logger.LogError(ex, ex.Message);
_logger.LogInformation("Error in ImportManager ImportAll method - import was NOT completed.");
throw;
}
}
public void Run()
{
try
{
Stopwatch stopwatchFull = Stopwatch.StartNew();
_logger.LogInformation($"{nameof(Application)}.{nameof(Run)} execution invoked");
_importValidator = _serviceProvider.GetService <IImportValidator>();
_typeListValidator = _serviceProvider.GetService <ITypeListValidator>();
_importManager = _serviceProvider.GetService <IImportManager>();
_importStorageProvider = _serviceProvider.GetService <IImportStorageProvider>();
_transformManager = _serviceProvider.GetService <ITransformManager>();
_transformStorageProvider = _serviceProvider.GetService <ITransformStorageProvider>();
_mergeManager = _serviceProvider.GetService <IMergeManager>();
_mergeStorageProvider = _serviceProvider.GetService <IMergeStorageProvider>();
//uncomment below to actually run import and transform stages - file based
string formattedDateString = DateTime.UtcNow.ToString(DateFormatSpecifier);
//Import
Stopwatch stopwatchImport = Stopwatch.StartNew();
string failureInfo;
//Before we take the overhead of downloading all the import data, check that the data has the right overall structure
bool isDataSourceValid = _importValidator.TryValidateDataSource(out failureInfo);
if (!isDataSourceValid)
{
_logger.LogWarning("Enlir Import Data not in Expected Format: \n" + failureInfo);
throw new ValidationException("Enlir Import Data not in Expected Format: \n" + failureInfo);
}
ImportResultsContainer importResultsContainer = _importManager.ImportAll();
string importStoragePath = _importStorageProvider.StoreImportResults(importResultsContainer, formattedDateString);
stopwatchImport.Stop();
//cheat data setup for testing - comment out when doing full run for real
//string importStoragePath = @"D:\Temp\FFRKApi\ImportResults-2018-12-21_09-48-46.json";
//string transformStoragePath = @"D:\Docs\Personal\FFRKLinqQuery\TransformResults-Latest.json";
//string formattedDateString = "2018-12-21_09-48-46";
//string importContents = File.ReadAllText(importStoragePath);
//ImportResultsContainer importResultsContainer = JsonConvert.DeserializeObject<ImportResultsContainer>(importContents);
////Now that we have the import data, we need to check whether our TypeLists (used to convert staring data into ids)
////is still accurate. If the source data has changed their list of values for each type, we need to stop and correct the TypeLists
IEnumerable <TypeListDifferences> typeListDifferences = _typeListValidator.TryValidateTypeLists(importResultsContainer);
if (typeListDifferences.Any(t => t.IsIdListDifferentFromSource))
{
_logger.LogWarning("Enlir TypeList Data differs from coded TypeLists.");
//write validation failure data to log for easy perusal
string typeListDifferencesLogPath = $"{AppContext.BaseDirectory}\\TypeListDifferencesLog.json";
string typeListDifferencesLogContents = JsonConvert.SerializeObject(typeListDifferences);
File.WriteAllText(typeListDifferencesLogPath, typeListDifferencesLogContents);
_logger.LogWarning("Enlir TypeList differences written to file: " + typeListDifferencesLogPath);
throw new ValidationException("Enlir Type List Data differs from coded TypeLists");
}
//Transform
Stopwatch stopwatchTransform = Stopwatch.StartNew();
TransformResultsContainer transformResultsContainer = _transformManager.TransformAll(importStoragePath);
string transformStoragePath = _transformStorageProvider.StoreTransformResults(transformResultsContainer, formattedDateString);
stopwatchTransform.Stop();
//Merge
Stopwatch stopwatchMerge = Stopwatch.StartNew();
MergeResultsContainer mergeResultsContainer = _mergeManager.MergeAll(transformStoragePath);
string mergeStoragePath = _mergeStorageProvider.StoreMergeResults(mergeResultsContainer, formattedDateString);
stopwatchMerge.Stop();
//test merge storage
MergeResultsContainer testMergeResultsContainer = _mergeStorageProvider.RetrieveMergeResults(mergeStoragePath);
stopwatchFull.Stop();
_logger.LogInformation("Import Completed in {ImportTime} seconds", stopwatchImport.Elapsed.Seconds);
_logger.LogInformation("Transform Completed in {TransformTime} seconds", stopwatchTransform.Elapsed.Seconds);
_logger.LogInformation("Merge Completed in {MergeTime} seconds", stopwatchMerge.Elapsed.Seconds);
_logger.LogInformation("Full Run Completed in {FullRunTime} seconds", stopwatchFull.Elapsed.Seconds);
int aggregateTime = stopwatchImport.Elapsed.Seconds + stopwatchMerge.Elapsed.Seconds + stopwatchFull.Elapsed.Seconds;
_logger.LogInformation("Full Run Aggregate Time in {AggregateTime} seconds", aggregateTime);
}
catch (Exception ex)
{
_logger.LogError(ex, ex.Message);
_logger.LogInformation("Error in Top Level Application execution. Validate, Import, Transform, and Merge operations were NOT successfully completed. Previously existing data is unchanged");
throw;
}
}
