/// <summary>
/// This method must be started in the background
/// </summary>
public void StartPulling()
{
while (true)
{
var pullSources = _dbAccessor.GetPullSources().ToList();
foreach (var pullSource in pullSources)
{
try
{
WasteWaterTreatmentPlant wwtp = null;
if (pullSource.dataType.ToLower() == FileFormat.CSV.ToString().ToLower() || pullSource.dataType == FileFormat.XLS.ToString().ToLower() || pullSource.dataType == FileFormat.XLSX.ToString().ToLower())
{
var waterPlant = _utilityDbAccessor.GetWwtpNameForId((int)pullSource.waterPlantId);
var treatmentStepType = _utilityDbAccessor.GetWwtpNameForId((int)pullSource.treatmentStepTypeId);
foreach (var file in Directory.GetFiles(pullSource.sourcePath))
{
if (file.StartsWith("_"))
{
continue; // ignore files starting with _
}
using (FileStream stream = File.Open(Path.Combine(pullSource.sourcePath, file), FileMode.Open))
{
wwtp = _businessLogic.HarmonizeData(stream, (FileFormat)Enum.Parse(typeof(FileFormat), pullSource.dataType), waterPlant, treatmentStepType);
}
}
}
else if (pullSource.dataType.ToLower() == FileFormat.JSON.ToString().ToLower())
{
var waterPlant = pullSource.waterPlantId == null ? "" : _utilityDbAccessor.GetWwtpNameForId((int)pullSource.waterPlantId);
var treatmentStepType = pullSource.treatmentStepTypeId == null ? "" : _utilityDbAccessor.GetWwtpNameForId((int)pullSource.treatmentStepTypeId);
foreach (var file in Directory.GetFiles(pullSource.sourcePath))
{
if (file.StartsWith("_"))
{
continue; // ignore files starting with _
}
var json = File.ReadAllText(Path.Combine(pullSource.sourcePath, file));
wwtp = _businessLogic.HarmonizeData(json, waterPlant, treatmentStepType);
}
}
//file processed without errors - rename the file
var fileName = Path.GetFileName(pullSource.sourcePath);
var directory = Path.GetDirectoryName(pullSource.sourcePath);
File.Move(Path.Combine(directory, fileName), Path.Combine(directory, $"_{fileName}"));
Debug.WriteLine("-- WaterPlant just got harmonized!: \n" + wwtp);
}
catch (Exception e)
{
Debug.WriteLine(e);
}
}
Task.Delay(PULLFREQUENCY_SECONDS * 1000);
}
}
public WasteWaterTreatmentPlant HarmonizeTableObject(TableFormattedObject tableObject, string waterPlant, string treatmentStep)
{
var wwtp = _tableHarmonizer.HarmonizeTable(tableObject);
//fill remaining objects
if (int.TryParse(treatmentStep, out var treatmentTypeId))
{
treatmentStep = _dbUtilAccessor.GetTreatmentTypeForId(treatmentTypeId);
}
if (int.TryParse(waterPlant, out var wwtpId))
{
waterPlant = _dbUtilAccessor.GetWwtpNameForId(wwtpId);
}
wwtp.TreatmentSteps[0].Name = treatmentStep;
wwtp.Name = waterPlant;
return(wwtp);
}
public WasteWaterTreatmentPlant HarmonizeTree(TreeFormattedObject treeObject, int?waterPlantId = null, int?treatmentStepTypeId = null)
{
WasteWaterTreatmentPlant wwtp;
// if the object was conform to the predefined schema all along, just convert, map and return it
if (treeObject.IsPredefinedSchema)
{
var wwtpObject = treeObject.Object as WasteWaterTreatmentPlant;
_aliasNameToRealNameDict = waterPlantId != null?
_utilityDbAccessor.GetAliasToRealNameOnWwtpDict((int)waterPlantId) :
_utilityDbAccessor.GetAliasToRealNameOnWwtpDict(_utilityDbAccessor.GetIdForWwtpName(wwtpObject?.Name));
foreach (var treatmentStep in wwtpObject.TreatmentSteps)
{
treatmentStep.QualityIndicators = MapWaterPlantAliases(treatmentStep.QualityIndicators);
}
return(wwtpObject);
}
// ... otherwise first replace the single valeus by waterplant aliases and then try to convert it
// if that does not work, the object is not valid
// we can only map if waterPlantId is passed as parameter or can be read from the treeobject
if (waterPlantId != null && _aliasNameToRealNameDict != null)
{
_aliasNameToRealNameDict = _utilityDbAccessor.GetAliasToRealNameOnWwtpDict((int)waterPlantId);
}
else
{
try
{
var wwtpAsObject = ((JObject)treeObject.Object).ToObject <WasteWaterTreatmentPlant>();
var name = wwtpAsObject.Name;
if (!string.IsNullOrEmpty(name))
{
waterPlantId = waterPlantId ?? _utilityDbAccessor.GetIdForWwtpName(name);
_aliasNameToRealNameDict = _utilityDbAccessor.GetAliasToRealNameOnWwtpDict((int)waterPlantId);
}
}
catch (Exception e)
{
throw new Exception("Name & Id of Wwtp missing");
}
}
// map the dynamic object
dynamic mappingResult = null;
if (_aliasNameToRealNameDict != null && _aliasNameToRealNameDict.Any())
{
var dynamicJsonDict = JsonConvert.DeserializeObject <Dictionary <string, dynamic> >(treeObject.Object.ToString());
mappingResult = MapWaterPlantAliases(dynamicJsonDict);
// convert mapping result back into the correct format
ExpandoObject expandoObject = ((Dictionary <string, dynamic>)mappingResult).ToExpando();
var serializedExpandoObject = JsonConvert.SerializeObject(expandoObject);
mappingResult = JsonConvert.DeserializeObject <dynamic>(serializedExpandoObject);
}
// even if the mapping wasnt successful, we still might have a valid waterplant object
// last try to deserialize it and checks if everything is correct afterwars
var dynamicDict = mappingResult == null ? treeObject.Object : mappingResult;
//<<DESERIALIZATION CHECKS>>
//try to deserialuze the wwtp as a whole first
if (IsWholeWwtp(dynamicDict, out wwtp))
{
foreach (var treatmentStep in wwtp.TreatmentSteps)
{
treatmentStep.QualityIndicators = MapWaterPlantAliases(treatmentStep.QualityIndicators);
}
return(wwtp);
}
//if this didnt work, check if its an option to only deserialize parts of the wwtp
//treatment step list
if (waterPlantId != null && IsTreatmentSteps(dynamicDict, out wwtp))
{
wwtp.Name = _utilityDbAccessor.GetWwtpNameForId((int)waterPlantId);
foreach (var treatmentStep in wwtp.TreatmentSteps)
{
treatmentStep.QualityIndicators = MapWaterPlantAliases(treatmentStep.QualityIndicators);
}
return(wwtp);
}
//qualityIndicators
if (waterPlantId != null && treatmentStepTypeId != null && IsQualityIndicators(dynamicDict, out wwtp))
{
wwtp.Name = _utilityDbAccessor.GetWwtpNameForId((int)waterPlantId);
wwtp.TreatmentSteps.FirstOrDefault().Name = _utilityDbAccessor.GetTreatmentTypeForId((int)treatmentStepTypeId);
foreach (var treatmentStep in wwtp.TreatmentSteps)
{
treatmentStep.QualityIndicators = MapWaterPlantAliases(treatmentStep.QualityIndicators);
}
return(wwtp);
}
//if nothing worked so far - last try is to check if its a qualityindicatorlist that is directly assigned to the waterplant
if (waterPlantId != null && IsQualityIndicators(dynamicDict, out wwtp, true))
{
wwtp.Name = _utilityDbAccessor.GetWwtpNameForId((int)waterPlantId);
foreach (var treatmentStep in wwtp.TreatmentSteps)
{
treatmentStep.QualityIndicators = MapWaterPlantAliases(treatmentStep.QualityIndicators);
}
return(wwtp);
}
throw new Exception("Could not harmonize the dynamicObject");
}