public TransformManager(IRowTransformer <EventRow, Event> eventRowTransformer, IRowTransformer <MissionRow, Mission> missionRowTransformer,
IRowTransformer <ExperienceRow, Experience> experienceRowTransformer,
IRowTransformer <MagiciteSkillRow, MagiciteSkill> magiciteSkillRowTransformer, IRowTransformer <MagiciteRow, Magicite> magiciteRowTransformer,
IRowTransformer <StatusRow, Status> statusRowTransformer, IRowTransformer <OtherRow, Other> otherRowTransformer,
IRowTransformer <CommandRow, Command> commandRowTransformer, IRowTransformer <BraveActionRow, BraveAction> braveActionRowTransformer,
IRowTransformer <SoulBreakRow, SoulBreak> soulBreakRowTransformer,
IRowTransformer <RelicRow, Relic> relicRowTransformer, IRowTransformer <AbilityRow, Ability> abilityRowTransformer,
IRowTransformer <LegendMateriaRow, LegendMateria> legendMateriaRowTransformer, IRowTransformer <RecordMateriaRow, RecordMateria> recordMateriaRowTransformer,
IRowTransformer <RecordSphereRow, RecordSphere> recordSphereRowTransformer, IRowTransformer <LegendSphereRow, LegendSphere> legendSphereRowTransformer,
IRowTransformer <CharacterRow, Character> characterRowTransformer,
IImportStorageProvider importStorageProvider, ILogger <TransformManager> logger)
{
_eventRowTransformer = eventRowTransformer;
_missionRowTransformer = missionRowTransformer;
_experienceRowTransformer = experienceRowTransformer;
//_dungeonRowTransformer = dungeonRowTransformer;
_magiciteSkillRowTransformer = magiciteSkillRowTransformer;
_magiciteRowTransformer = magiciteRowTransformer;
_statusRowTransformer = statusRowTransformer;
_otherRowTransformer = otherRowTransformer;
_commandRowTransformer = commandRowTransformer;
_braveActionRowTransformer = braveActionRowTransformer;
_soulBreakRowTransformer = soulBreakRowTransformer;
_relicRowTransformer = relicRowTransformer;
_abilityRowTransformer = abilityRowTransformer;
_legendMateriaRowTransformer = legendMateriaRowTransformer;
_recordMateriaRowTransformer = recordMateriaRowTransformer;
_recordSphereRowTransformer = recordSphereRowTransformer;
_legendSphereRowTransformer = legendSphereRowTransformer;
_characterRowTransformer = characterRowTransformer;
_importStorageProvider = importStorageProvider;
_logger = logger;
}
public static async Task <HttpResponseMessage> Run([HttpTrigger(AuthorizationLevel.Anonymous, "get", "post", Route = null)] HttpRequestMessage req,
[Inject] IImportStorageProvider importStorageProvider, [Inject] ILogger <IImportStorageProvider> logger)
{
logger.LogInformation("Azure Function StoreImport processed a request.");
try
{
// parse query parameter -can be null
var qs = req.RequestUri.ParseQueryString();
string formattedDateString = qs.Get("formattedDateString");
//string formattedDateString = req.GetQueryNameValuePairs()
// .FirstOrDefault(q => string.Compare(q.Key, "formattedDateString", true) == 0)
// .Value;
// Get request body
dynamic data = await req.Content.ReadAsAsync <object>();
string datastring = data.ToString();
//this data had better be serializable to ImportResultsContainer
ImportResultsContainer irc = JsonConvert.DeserializeObject <ImportResultsContainer>(datastring);
string filePath = importStorageProvider.StoreImportResults(irc, formattedDateString);
var response = req.CreateResponse(HttpStatusCode.OK, filePath);
return(response);
}
catch (System.Exception ex)
{
logger.LogError(ex, $"Error in Azure Function StoreImport : {ex.Message}");
var response = req.CreateResponse(HttpStatusCode.InternalServerError, ex.Message);
return(response);
}
}
public void Run()
{
try
{
Stopwatch stopwatchFull = Stopwatch.StartNew();
_logger.LogInformation($"{nameof(Application)}.{nameof(Run)} execution invoked");
_importValidator = _serviceProvider.GetService <IImportValidator>();
_typeListValidator = _serviceProvider.GetService <ITypeListValidator>();
_importManager = _serviceProvider.GetService <IImportManager>();
_importStorageProvider = _serviceProvider.GetService <IImportStorageProvider>();
_transformManager = _serviceProvider.GetService <ITransformManager>();
_transformStorageProvider = _serviceProvider.GetService <ITransformStorageProvider>();
_mergeManager = _serviceProvider.GetService <IMergeManager>();
_mergeStorageProvider = _serviceProvider.GetService <IMergeStorageProvider>();
//uncomment below to actually run import and transform stages - file based
string formattedDateString = DateTime.UtcNow.ToString(DateFormatSpecifier);
//Import
Stopwatch stopwatchImport = Stopwatch.StartNew();
string failureInfo;
//Before we take the overhead of downloading all the import data, check that the data has the right overall structure
bool isDataSourceValid = _importValidator.TryValidateDataSource(out failureInfo);
if (!isDataSourceValid)
{
_logger.LogWarning("Enlir Import Data not in Expected Format: \n" + failureInfo);
throw new ValidationException("Enlir Import Data not in Expected Format: \n" + failureInfo);
}
ImportResultsContainer importResultsContainer = _importManager.ImportAll();
string importStoragePath = _importStorageProvider.StoreImportResults(importResultsContainer, formattedDateString);
stopwatchImport.Stop();
//cheat data setup for testing - comment out when doing full run for real
//string importStoragePath = @"D:\Temp\FFRKApi\ImportResults-2018-12-21_09-48-46.json";
//string transformStoragePath = @"D:\Docs\Personal\FFRKLinqQuery\TransformResults-Latest.json";
//string formattedDateString = "2018-12-21_09-48-46";
//string importContents = File.ReadAllText(importStoragePath);
//ImportResultsContainer importResultsContainer = JsonConvert.DeserializeObject<ImportResultsContainer>(importContents);
////Now that we have the import data, we need to check whether our TypeLists (used to convert staring data into ids)
////is still accurate. If the source data has changed their list of values for each type, we need to stop and correct the TypeLists
IEnumerable <TypeListDifferences> typeListDifferences = _typeListValidator.TryValidateTypeLists(importResultsContainer);
if (typeListDifferences.Any(t => t.IsIdListDifferentFromSource))
{
_logger.LogWarning("Enlir TypeList Data differs from coded TypeLists.");
//write validation failure data to log for easy perusal
string typeListDifferencesLogPath = $"{AppContext.BaseDirectory}\\TypeListDifferencesLog.json";
string typeListDifferencesLogContents = JsonConvert.SerializeObject(typeListDifferences);
File.WriteAllText(typeListDifferencesLogPath, typeListDifferencesLogContents);
_logger.LogWarning("Enlir TypeList differences written to file: " + typeListDifferencesLogPath);
throw new ValidationException("Enlir Type List Data differs from coded TypeLists");
}
//Transform
Stopwatch stopwatchTransform = Stopwatch.StartNew();
TransformResultsContainer transformResultsContainer = _transformManager.TransformAll(importStoragePath);
string transformStoragePath = _transformStorageProvider.StoreTransformResults(transformResultsContainer, formattedDateString);
stopwatchTransform.Stop();
//Merge
Stopwatch stopwatchMerge = Stopwatch.StartNew();
MergeResultsContainer mergeResultsContainer = _mergeManager.MergeAll(transformStoragePath);
string mergeStoragePath = _mergeStorageProvider.StoreMergeResults(mergeResultsContainer, formattedDateString);
stopwatchMerge.Stop();
//test merge storage
MergeResultsContainer testMergeResultsContainer = _mergeStorageProvider.RetrieveMergeResults(mergeStoragePath);
stopwatchFull.Stop();
_logger.LogInformation("Import Completed in {ImportTime} seconds", stopwatchImport.Elapsed.Seconds);
_logger.LogInformation("Transform Completed in {TransformTime} seconds", stopwatchTransform.Elapsed.Seconds);
_logger.LogInformation("Merge Completed in {MergeTime} seconds", stopwatchMerge.Elapsed.Seconds);
_logger.LogInformation("Full Run Completed in {FullRunTime} seconds", stopwatchFull.Elapsed.Seconds);
int aggregateTime = stopwatchImport.Elapsed.Seconds + stopwatchMerge.Elapsed.Seconds + stopwatchFull.Elapsed.Seconds;
_logger.LogInformation("Full Run Aggregate Time in {AggregateTime} seconds", aggregateTime);
}
catch (Exception ex)
{
_logger.LogError(ex, ex.Message);
_logger.LogInformation("Error in Top Level Application execution. Validate, Import, Transform, and Merge operations were NOT successfully completed. Previously existing data is unchanged");
throw;
}
}
