private void TakeAddOnExtensionSwitches(List <ExeXml.AddOn> addOns)
{
// make a list of extensions that are switched by the add-on(s), if several add-ons switch the same extension, the last one wins
Dictionary <string, bool> aoExtensionSwitches = new Dictionary <string, bool>(); // key: extension-id, value: on/off (on=true)
foreach (ExeXml.AddOn addOn in addOns) // run over all add-ons
{
foreach (ExeXml.Fun aoFun in addOn.polAOControl.funs.Values) // search for add-on-function AddOn_ExtensionSwitch
{
if (!aoFun.on || aoFun.Name.ToLower() != DefFun.AddOn_ExtensionSwitch.ToLower())
{
continue;
}
// first sort the groups together (there may be several groups of Extension_Id or Extension_Name + Extension_Switch)
Dictionary <string, List <ExeXml.Par> > parGroups = new Dictionary <string, List <ExeXml.Par> >();
foreach (ExeXml.Par aoPar in aoFun.pars.Values)
{
if (!parGroups.ContainsKey(aoPar.Group))
{
parGroups.Add(aoPar.Group, new List <ExeXml.Par>());
}
parGroups[aoPar.Group].Add(aoPar);
}
// check if the content of each group is ok
Description fDesc = new Description(addOn.polAOControl, aoFun); // just for error-messages
foreach (var parGroup in parGroups)
{
string extID = null, extName = null, extSwitch = null;
List <string> dataPatterns = new List <string>(), sysPatterns = new List <string>();
foreach (ExeXml.Par par in parGroup.Value)
{
if (par.val == DefPar.Value.NA)
{
continue;
}
if (par.Name.ToLower() == DefPar.AddOn_ExtensionSwitch.Extension_Id.ToLower())
{
extID = par.val.Trim().ToLower();
}
else if (par.Name.ToLower() == DefPar.AddOn_ExtensionSwitch.Extension_Name.ToLower())
{
extName = par.val.Trim().ToLower();
}
else if (par.Name.ToLower() == DefPar.AddOn_ExtensionSwitch.Extension_Switch.ToLower())
{
extSwitch = par.val.Trim().ToLower();
}
else if (par.Name.ToLower() == DefPar.AddOn_ExtensionSwitch.Dataset.ToLower())
{
dataPatterns.Add(par.val.Trim());
}
else if (par.Name.ToLower() == DefPar.AddOn_ExtensionSwitch.System.ToLower())
{
sysPatterns.Add(par.val.Trim());
}
else
{
MakeWarning($"unknown parameter {par.Name} is ignored", false);
}
}
if (extID == null && extName == null && extSwitch == null &&
dataPatterns.Count() == 0 && sysPatterns.Count() == 0)
{
continue; // probably all set to n/a
}
if (extID == null && extName == null)
{
MakeWarning($"neither {DefPar.AddOn_ExtensionSwitch.Extension_Name} nor {DefPar.AddOn_ExtensionSwitch.Extension_Id} defined");
continue;
}
if (extID != null)
{
if (!infoStore.country.extensions.ContainsKey(extID))
{
MakeWarning($"unknown extension id {extID}"); continue;
}
}
if (extName != null)
{
if (extID == null)
{
extID = GetIdByName(extName);
if (extID == null)
{
MakeWarning($"unknown extension name {extName}"); continue;
}
}
else
{
MakeWarning($"{DefPar.AddOn_ExtensionSwitch.Extension_Name} is ignored, as {DefPar.AddOn_ExtensionSwitch.Extension_Id} is defined as well", false);
}
// note that no warning is issued, if a global extension exsits, but there is no implementation for the country
// or even only no switch for the system-data-combination
// one could easily catch this, by some rearranging of code, but I think it's safer to avoid a maybe unwanted error-messsage
}
if (extSwitch != null)
{
if (extSwitch != DefPar.Value.ON && extSwitch != DefPar.Value.OFF)
{
MakeWarning($"invalid value for {DefPar.AddOn_ExtensionSwitch.Extension_Switch}, must be {DefPar.Value.ON} or {DefPar.Value.OFF}"); continue;
}
}
else
{
MakeWarning($"parameter {DefPar.AddOn_ExtensionSwitch.Extension_Switch} missing"); continue;
}
bool patternMatches = dataPatterns.Count == 0;
foreach (string pattern in dataPatterns)
{
if (infoStore.IsUsedDatabase(pattern))
{
patternMatches = true; break;
}
}
if (!patternMatches)
{
continue;
}
patternMatches = sysPatterns.Count == 0;
foreach (string pattern in sysPatterns)
{
if (EM_Helpers.DoesValueMatchPattern(pattern, infoStore.country.sys.name))
{
patternMatches = true; break;
}
}
if (!patternMatches)
{
continue;
}
aoExtensionSwitches.Add(extID, extSwitch == DefPar.Value.ON);
void MakeWarning(string message, bool addIgnore = true)
{
infoStore.communicator.ReportError(new Communicator.ErrorInfo()
{
isWarning = true,
message = $"{fDesc.Get()} (group {parGroup.Key}): {message}" + (addIgnore ? ", switch-redefinition is ignored" : string.Empty)
});
}
}
}
}
TakeExtensionSwitches(aoExtensionSwitches);
string GetIdByName(string name)
{
// first try if the name belongs to a local extension (where we have got the name, as it is stored in the country file) ...
var lext = from e in infoStore.country.extensions
where (e.Value.localLongName != null && e.Value.localLongName.ToLower() == name) ||
(e.Value.localShortName != null && e.Value.localShortName.ToLower() == name)
select e;
if (lext.Count() > 0)
{
return(lext.First().Key);
}
// ... only if not found, need to read gloabl extensions
var gext = from e in ExeXmlReader.ReadExtensions(infoStore.runConfig.pathHandler.GetExtensionsFilePath(), infoStore.communicator)
where (e.Value.Item1 != null && e.Value.Item1.ToLower() == name) ||
(e.Value.Item2 != null && e.Value.Item2.ToLower() == name)
select e;
return(gext.Count() > 0 ? gext.First().Key : null);
}
}
private static bool HandleExtensionSwitches(Dictionary <string, string> config, out Dictionary <string, string> modifiedExtensionSwitches)
{
modifiedExtensionSwitches = new Dictionary <string, string>();
Dictionary <string, string> toReplace = new Dictionary <string, string>();
foreach (var entry in config)
{
if (!entry.Key.StartsWith(TAGS.EXTENSION_SWITCH))
{
continue;
}
string[] splitVal = entry.Value.Split("="); if (splitVal.Count() < 2)
{
continue;
}
string extName = splitVal[0], extVal = splitVal[1];
modifiedExtensionSwitches.Add(extName, extVal);
if (!EM_Helpers.IsGuid(extName))
{
toReplace.Add(entry.Key, extName.ToLower());
}
}
if (toReplace.Count == 0)
{
return(true);
}
EMPath pathHandler = new EMPath(config[TAGS.CONFIG_PATH_EUROMODFILES]);
Communicator dummyCommunicator = new Communicator();
var globExt = ExeXmlReader.ReadExtensions(pathHandler.GetExtensionsFilePath(), dummyCommunicator);
// first search in global file ...
List <string> done = new List <string>();
foreach (var tr in toReplace)
{
string extName = tr.Value, configKey = tr.Key;
// the name of the extension must be replaced by the id:
foreach (var e in globExt)
{
string extId = e.Key, shortName = e.Value.Item1.ToLower(), longName = e.Value.Item2.ToLower();
if (extName == shortName || extName == longName)
{
Replace(configKey, extId); break;
}
}
}
// ... if any extension-name was not found in global file, search in country file
if (done.Count == toReplace.Count)
{
return(true);
}
ExeXml.Country country = ExeXmlReader.ReadCountry(pathHandler.GetCountryFilePath(config[TAGS.CONFIG_COUNTRY]),
config[TAGS.CONFIG_ID_SYSTEM], config[TAGS.CONFIG_ID_DATA], false, dummyCommunicator);
foreach (var tr in toReplace)
{
string extName = tr.Value, configKey = tr.Key;
if (done.Contains(configKey))
{
continue;
}
foreach (var e in from ce in country.extensions where ce.Value.localShortName != null select ce)
{
string extId = e.Key, shortName = e.Value.localShortName.ToLower(),
longName = e.Value.localLongName; if (longName != null)
{
longName = longName.ToLower();
}
if (extName == shortName || extName == longName)
{
Replace(configKey, extId); break;
}
}
}
if (done.Count == toReplace.Count)
{
return(true);
}
string notDone = string.Empty; foreach (var tr in toReplace)
{
if (!done.Contains(tr.Key))
{
notDone += tr.Value + " ";
}
}
Console.WriteLine($"Unknown extension(s): {notDone}");
return(false);
void Replace(string configKey, string extId)
{
config[configKey] = $"{extId}={config[configKey].Split("=")[1]}"; done.Add(configKey);
}
}
/// <summary>
/// runs the executable (after configuration is set by either "exe"- or "lib"-call)
/// *** NOTE ON ERROR HANDLING ***
/// each of the called functions is expected to handle errors as follows:
/// - critical error (further programme execution is not possible or does not make sense): throw informative(!) exception
/// - errors and warnings: use Communicator.ReportError (amongst others indicating whether it is an error or a warning)
/// thus the caller gets the errors and warnings and can handle them as it wants (issue in a box, put to an error-log-file, ...)
/// in addition the errors are counted by the Communicator, allowing the Run-function to stop, e.g. after reading, if there are errors
/// </summary>
internal bool Run()
{
EM_Executable_Test.infoStore = infoStore; // initialise the testing-class
infoStore.communicator.maxRunTimeErrors = infoStore.runConfig.maxRunTimeErrors;
// if output should be in memory, initiate the output dictionary
if (infoStore.runConfig.returnOutputInMemory)
{
infoStore.output = new Dictionary <string, System.Text.StringBuilder>();
}
// READ COUNTRY FILE
// note on case-sensitivity: parameter values are converted to lower-case, everything else is original-case (to keep open for more case-sensitivity in future)
infoStore.country = ExeXmlReader.ReadCountry(
infoStore.runConfig.pathHandler.GetCountryFilePath(infoStore.runConfig.countryShortName),
infoStore.runConfig.sysID, infoStore.runConfig.dataID, // ids (or names) of system and data
infoStore.runConfig.ignorePrivate, // ignore (do not read) private policies/functions/parameters
infoStore.communicator); // see note on error-handling above
if (!infoStore.communicator.ReportProgress(new Communicator.ProgressInfo()
{
id = Communicator.EXEPROG_PAR_READ,
message = $"{infoStore.country.sys.name.ToUpper()} parameters read", detailedInfo = ComposeRunInfo()
}))
{
return(false);
}
// READ VARIABLES FILE (this is a prerequisite for formula interpretation)
// note on case-sensitivity: the indexVarConfig-dictionary has a case-insensitive key-comparer
infoStore.operandAdmin.indexVarConfig = ExeXmlReader.ReadVars(
infoStore.runConfig.pathHandler.GetVarFilePath(),
infoStore.communicator);
if (!infoStore.communicator.ReportProgress(new Communicator.ProgressInfo()
{
id = Communicator.EXEPROG_VAR_READ,
message = $"{infoStore.operandAdmin.indexVarConfig.Count} definitions read from the Variables file"
}))
{
return(false);
}
// READ OTHER GLOBAL FILES
infoStore.exRate = ExeXmlReader.ReadExRate( // note: returns -1 if not available (which may be ok if not used)
infoStore.runConfig.pathHandler.GetExRatesFilePath(),
infoStore.country.cao.shortName,
infoStore.country.sys.name, // name of system allows selecting the suitable exchange rate
infoStore.runConfig.dateExchangeRate, // e.g. TAGS.EXRATE_JUNE30 or TAGS.DEFAULT ...
infoStore.communicator);
// note on case-sensitivity: the upFacs-dictionary has a case-insensitive key-comparer
ExeXml.UpIndDict hicp = ExeXmlReader.ReadHICP( // note: returns hicp.data/hicp.sys=null if not available (which may be ok if not used)
infoStore.runConfig.pathHandler.GetHICPFilePath(),
infoStore.country.cao.shortName,
infoStore.country.sys.year, // ids of system and data allow returning
infoStore.country.data.year, // the relevant values
infoStore.communicator);
if (infoStore.country.upFacs.ContainsKey(DefPar.Value.HICP))
{
infoStore.country.upFacs[DefPar.Value.HICP] = hicp;
}
else
{
infoStore.country.upFacs.Add(DefPar.Value.HICP, hicp);
}
if (!infoStore.communicator.ReportProgress(new Communicator.ProgressInfo()
{
id = Communicator.EXEPROG_GLOB_PAR_READ,
message = $"Global parameters read"
}))
{
return(false);
}
// MAKE UPRATING FACTORS AVAILABLE AS GLOBAL VARIABLES, 1st step: generation (to be filled with values once data is read)
foreach (var upFac in infoStore.country.upFacs)
{
infoStore.operandAdmin.RegisterVar(upFac.Key.ToLower(),
DefFun.DefConst, // put DefConst as "creator" mainly to allow overwritting by DefConst (done in CZ), but can be justified by it was like this in the old exe
null, false, true, false, true);
}
// MAKE INDIRECT TAXES AVAILABLE AS GLOBAL VARIABLES, 1st step (same procedure as for uprating factors)
foreach (var indTax in infoStore.country.indTaxes)
{
infoStore.operandAdmin.RegisterVar(indTax.Key.ToLower(), "IndirectTaxTable", null, false, true, false, true);
}
// IF APPROPRIATE, READ ADD-ONS
// note: read here to have extension switches (AddOn_ExtensionSwitch) at disposal, as they overwrite any other switches
// even those set via run-tool, justification: the add-on creator probably knows why she wants sth on or off
// add-ons are still integrated after "switching" (i.e. split from reading) as there may be still (old) "manual" switching (via ChangeParam)
List <ExeXml.AddOn> xmlAddOns = new List <ExeXml.AddOn>();
if (infoStore.runConfig.addOns.Count > 0)
{
foreach (var addOn in infoStore.runConfig.addOns)
{
xmlAddOns.Add(ExeXmlReader.ReadAddOn(
path: infoStore.runConfig.pathHandler.GetAddOnFilePath(addOn.Key),
addOnSysIdentifier: addOn.Value,
communicator: infoStore.communicator));
}
if (infoStore.communicator.errorCount > 0)
{
return(false);
}
if (!infoStore.communicator.ReportProgress(new Communicator.ProgressInfo()
{
id = Communicator.EXEPROG_ADDON_READ,
message = $"Add-on info read"
}))
{
return(false);
}
}
// if the "All Output in €" box is checked in the run-tool
if (infoStore.runConfig.forceOutputEuro)
{
infoStore.country.sys.isOutputEuro = true;
}
// APPLY GLOBAL SWITCHES IN COMBINATION WITH ADD-ON- AND CONFIG-SETTINGS
HandleExtensionSwitches(xmlAddOns);
// IF APPROPRIATE, INTEGRATE ADD-ON
if (xmlAddOns.Count > 0)
{
IntegrateAddOns(xmlAddOns);
if (infoStore.communicator.errorCount > 0)
{
return(false);
}
if (!infoStore.communicator.ReportProgress(new Communicator.ProgressInfo()
{
id = Communicator.EXEPROG_ADDON_INTEG,
message = $"Add-on info integrated"
}))
{
return(false);
}
}
// IF APPROPRIATE, HANDLE PARAMETER MODIFICATIONS VIA JSON-FILE
if (!string.IsNullOrEmpty(infoStore.runConfig.pathParModifications))
{
HandleParModificationsByFile();
}
// IF APPROPRIATE, HANDLE PARAMETER MODIFICATIONS VIA JSON-STRING
if (!string.IsNullOrEmpty(infoStore.runConfig.stringParModifications))
{
HandleParModificationsByString();
}
// IF APPROPRIATE, REPLACE =cc= AND =sys= BY COUNTRY-SHORTNAME RESPECTIVELY SYSTEM-NAME
ReplaceCCSys(infoStore.country.cao.pols);
// IF APPROPRIATE, APPLY ChangeParam AND ChangeSwitch
ApplyChangeParam(); ApplyChangeSwitch();
// SWITCH OFF SetDefaults AND Uprates IF THEY DO NOT APPLY TO THE CURRENT DATASET
SwitchOffUprateSetDefault();
// IF APPROPRIATE, handle break function
if (!HandleBreak())
{
return(false); // stop if the break is misdefined
}
// DROP EVERYTHING THAT IS OFF OR N/A (with the intention to enhance run-time-speed by not having to ask for it)
DropOff();
// GET VARIABLES CONTAINED IN INPUT DATA (we need to do this upfront as the info is required for addressing variables with wildcards)
HHAdmin.GetAllDataVariables(infoStore);
// IF APPROPRIATE, REGISTER EXPENDITURE VARIABLES
if (infoStore.country.data.readXVar)
{
RegisterXVar(infoStore.allDataVariables);
}
// IF APPROPRIATE, REGISTER STRING VARIABLES IN INPUT DATA, WHICH ARE TO BE TRANSFERRED TO OUTPUT
RegisterOutputStringVar();
// CHECK AND ANALYSE THE OBTAINED INFO AND PREPARE FOR RUN
CheckAndPrepare();
// check for availability of applicable Uprate functions
if (infoStore.applicableUprateFunctions.Count != 1)
{
string m = string.Empty;
foreach (Description d in infoStore.applicableUprateFunctions)
{
m += d.Get() + " & ";
}
if (m == string.Empty)
{
m = $"No applicable {DefFun.Uprate} function found";
}
else
{
m = $"More than one applicable {DefFun.Uprate} function found: " + m.TrimEnd(new char[] { '&', ' ' });
}
infoStore.communicator.ReportError(new Communicator.ErrorInfo()
{
isWarning = true, message = m
});
}
// ERROR-ASSESSMENT
if (!infoStore.communicator.ReportProgress(new Communicator.ProgressInfo()
{
id = Communicator.EXEPROG_PAR_CHECKED,
message = $"Parameters checked and prepared", detailedInfo = ComposeOutputInfo()
}))
{
return(false);
}
if (infoStore.communicator.errorCount > 0)
{
return(false);
}
// GENERATION OF HOUSEHOLD DATA (READING AND PREPARING VARIABLES)
infoStore.hhAdmin = new HHAdmin(infoStore);
infoStore.hhAdmin.GenerateData();
// MAKE UPRATING FACTORS AVAILABLE AS GLOBAL VARIABLES, 2nd step: fill with values
// NOTE: Uprating factors will have the default system year value. The DBYearVar is not supported in this case!
// ( * T O D O * maybe we can support it with the use of conditional constants?)
// * T O D O * this uprating factor logic shouldn't be here! especially as there is no warning if 1 is used
foreach (var upFac in infoStore.country.upFacs)
{
if (upFac.Value.Get(infoStore.country.data.year, out double dui) && upFac.Value.Get(infoStore.country.sys.year, out double sui))
{
infoStore.hhAdmin.GlobalSetVar(infoStore.operandAdmin.GetIndexInPersonVarList(upFac.Key.ToLower()), sui / dui);
}
else
{
infoStore.hhAdmin.GlobalSetVar(infoStore.operandAdmin.GetIndexInPersonVarList(upFac.Key.ToLower()), 1);
}
}
// MAKE INDIRECT TAXES AVAILABLE AS GLOBAL VARIABLES, 2nd step: fill with values
foreach (var indTax in infoStore.country.indTaxes)
{
infoStore.hhAdmin.GlobalSetVar(infoStore.operandAdmin.GetIndexInPersonVarList(indTax.Key.ToLower()), indTax.Value);
}
foreach (FunBase fun in infoStore.spine.Values) // before reading data the HH.personVarList does not exist and var-parameters
{
fun.ReplaceVarNameByIndex(); // cannot know their index inthere, now the index is available and must be spread
}
// IF NECESSARY, CURRENCY CONVERSION
// note: in terms of performance it seems to be irrelevant if this conversation is integrating into the reading data process
// thus this "separate approach" seems appropriate as it is clearer
if (infoStore.country.data.isEuro != infoStore.country.sys.areParamEuro)
{
double factor = infoStore.exRate;
if (infoStore.country.sys.areParamEuro)
{
factor = 1 / factor;
}
if (factor != 1)
{
infoStore.hhAdmin.GlobalScaleFileReadVars(factor);
}
}
// ERROR-ASSESSMENT
if (infoStore.communicator.errorCount > 0)
{
return(false);
}
if (!infoStore.communicator.ReportProgress(new Communicator.ProgressInfo()
{
id = Communicator.EXEPROG_HH_READ,
message = $"{infoStore.country.data.Name}: {infoStore.hhAdmin.hhs[0].personVarList[0].Count:N0} variables for {infoStore.hhAdmin.hhs.Count:N0} households ({infoStore.hhAdmin.hhs.Sum(x => x.GetPersonCount()):N0} individuals) read - largest household had {infoStore.hhAdmin.hhs.Max(x => x.GetPersonCount()):N0} members"
}))
{
return(false);
}
// PREPARE THE SPINE (what to run before/after/in spine and, if necessary, looping)
SpineAdmin spineManager = new SpineAdmin(infoStore);
bool runSequential = infoStore.runConfig.forceSequentialRun || infoStore.runConfig.forceAutoSequentialRun;
spineManager.PrepareSpine(ref runSequential);
// ERROR-ASSESSMENT
if (infoStore.communicator.errorCount > 0)
{
return(false);
}
if (!infoStore.communicator.ReportProgress(new Communicator.ProgressInfo()
{
id = Communicator.EXEPROG_SPINE_DONE,
message = $"Sequence of calculations prepared"
}))
{
return(false);
}
// RUN THE WHOLE THING
if (runSequential)
{
if (!RunSequential(spineManager))
{
return(false);
}
} // only if e.g. Totals is used or DefOutput is used in spine, ...
else
{
RunParallel(spineManager);
}
infoStore.communicator.ReportProgress(new Communicator.ProgressInfo()
{
id = Communicator.EXEPROG_SPINE_DONE,
message = $"Finished calculations"
});
// last ERROR-ASSESSMENT
infoStore.communicator.ReportProgress(new Communicator.ProgressInfo()
{
id = Communicator.EXEPROG_FINISHED, // hopefully with 0 warnings
message = $"FINISHED with {infoStore.communicator.warningCount + infoStore.communicator.errorCount} errors/warnings"
});
return(true);
}
