/// <summary>
/// reads a country's EM2 country- and dataconfig-XML-files and transfers them to EM3 style
/// creates the country folder, if it does not exist, and overwrites any existing country file
/// note: the intended usage is "single country transformation"
/// the EM3All class is responsible for complete EM-content transformation (using the EM3Country.Write function)
/// </summary>
/// <param name="emPath"> EuromodFiles folder (containing EM2-files in XMLParam and (will contain) EM3-files in EM3Translation\XMLParam) </param>
/// <param name="country"> short-name of country </param>
/// <param name="errors"> critical and non-critical erros during the transformation-process, empty structure for no errors </param>
public static bool Transform(string emPath, string country, out List <string> errors)
{
errors = new List <string>(); EMPath pathHandler = new EMPath(emPath);
string em2CountryFile = string.Empty, em2DataFile = string.Empty;
try
{
DirectoryInfo di = Directory.CreateDirectory(pathHandler.GetCountryFolderPath(country));
// read EM2-files
em2CountryFile = pathHandler.GetCountryFilePath(country: country, em2: true);
em2DataFile = pathHandler.GetEM2DataConfigFilePath(country);
bool up2D1 = TransformerCommon.IsFileUpToDate(em2CountryFile, pathHandler.GetCountryFolderPath(country), out string hash1);
bool up2D2 = TransformerCommon.IsFileUpToDate(em2DataFile, pathHandler.GetCountryFolderPath(country), out string hash2);
if (up2D1 && up2D2)
{
return(true); // do not combine in one if, to make sure that both hash-files (for country and dataconfig) are generated
}
EM2Country.Content ctryContent = EM2Country.Read(em2CountryFile, out List <string> cErrors);
EM2Data.Content dataContent = EM2Data.Read(em2DataFile, out List <string> dErrors);
// need the global file with policy-switches for proper transformation of local policy switches
List <List <MultiProp> > extensions = EM2Global.ReadSwitchPol(pathHandler.GetFolderConfig(em2: true), out List <string> gErrors);
errors.AddRange(cErrors); errors.AddRange(dErrors); errors.AddRange(gErrors);
if (ctryContent == null || dataContent == null || extensions == null)
{
return(false);
}
// write EM3-file (includes EM2->EM3 adaptations, via EM23Adapt class)
string em3CountryFile = pathHandler.GetCountryFilePath(country);
bool success = Write(ctryContent, dataContent, extensions, em3CountryFile, out List <string> wErrors);
errors.AddRange(wErrors);
if (success && errors.Count == 0)
{
TransformerCommon.WriteUpToDate(em2CountryFile, pathHandler.GetCountryFolderPath(country), hash1);
TransformerCommon.WriteUpToDate(em2DataFile, pathHandler.GetCountryFolderPath(country), hash2);
}
return(success);
}
catch (Exception exception)
{
errors.Add($"{country}: {exception.Message}");
return(false);
}
}
public static bool IsUpToDate(string emPath, string country)
{
EMPath pathHandler = new EMPath(emPath);
return(TransformerCommon.IsFileUpToDate(pathHandler.GetCountryFilePath(country, true), pathHandler.GetCountryFolderPath(country), out string hash1) &&
TransformerCommon.IsFileUpToDate(pathHandler.GetEM2DataConfigFilePath(country), pathHandler.GetCountryFolderPath(country), out string hash2));
}
// note: this assumes an existing EM3Translation-folder (as sub-folder of emPath) and takes only the translated countries in there into account
public static bool GetInfoCountries(string emPath, out List <CountryInfo> countryInfo, out string errorsAndWarnings,
bool getDatasets = true, bool getRunOptions = true)
{
errorsAndWarnings = string.Empty; countryInfo = new List <CountryInfo>();
EMPath pathHelper = new EMPath(emPath);
try
{
if (!Directory.Exists(pathHelper.GetFolderCountries()))
{
errorsAndWarnings = $"Folder {pathHelper.GetFolderCountries()} not found"; return(false);
}
object writeLock = new object(); List <CountryInfo> localCountryInfo = new List <CountryInfo>(); string localErrors = string.Empty;
Parallel.ForEach(new DirectoryInfo(pathHelper.GetFolderCountries()).GetDirectories(), folder =>
{
lock (writeLock)
{
if (GetInfoCountry(pathHelper.GetCountryFilePath(folder.Name), out CountryInfo ci, out string error, getDatasets, getRunOptions))
{
localCountryInfo.Add(ci);
}
else
{
localErrors += error + Environment.NewLine;
}
}
/// <summary> reads all components of an EM2 version </summary>
/// <param name="emPath"> path to EUROMOD version (e.g. C:\euromod\EuromodContent\EuromodFiles_H0.13\) </param>
/// <param name="errors"> list of errors, only critical errors lead to failure-return (null), i.e. there may be errors on "success"</param>
/// <param name="progressAction">
/// optional function for progress reporting, in the form 'void funName(string message)'
/// is called for each item (country, add-on, global file) and sets message to e.g. 'Finished reading ...', 'Failed reading ...'
/// </param>
/// <param name="cancelSrc"> optional cancellation item </param>
/// <returns> Content structure upon success, null upon failure </returns>
public static Content Read(string emPath, out List <string> errors,
Action <string> progressAction = null, CancellationTokenSource cancelSrc = null)
{
Content content = new Content();
List <string> _errors = new List <string>();
object writeLock = new object();
ParallelOptions parallelOptions = new ParallelOptions();
if (cancelSrc != null)
{
parallelOptions.CancellationToken = cancelSrc.Token;
}
try
{
EMPath pathHandler = new EMPath(emPath);
// READ COUNTRIES IN PARALLEL
Parallel.ForEach(new DirectoryInfo(pathHandler.GetFolderCountries(em2: true)).GetDirectories(), parallelOptions, folder =>
{
parallelOptions.CancellationToken.ThrowIfCancellationRequested();
string ctryFileName = pathHandler.GetCountryFilePath(country: folder.Name, em2: true);
string dataFileName = pathHandler.GetEM2DataConfigFilePath(country: folder.Name);
EM2Country.Content c = EM2Country.Read(ctryFileName, out List <string> cErrors);
EM2Data.Content d = EM2Data.Read(dataFileName, out List <string> dErrors);
bool success = c != null && d != null;
lock (writeLock)
{
if (success)
{
content.countries.Add(new CountryContent()
{
country = c, data = d
});
}
_errors.AddRange(cErrors); _errors.AddRange(dErrors);
}
ReportSuccess(folder.Name, success, cErrors.Count + dErrors.Count);
});
// the rest is read sequentially (parallel would be unnecessary overhead)
// READ ADD-ONS
foreach (DirectoryInfo folder in new DirectoryInfo(pathHandler.GetFolderAddOns(em2: true)).GetDirectories())
{
if ((new List <string>()
{
"HypoData", "SumStat", "XYZ"
}).Contains(folder.Name))
{
continue; // that's just old stuff, I think hardcoding isn't a faux pas
}
string aoFileName = pathHandler.GetAddOnFilePath(addOn: folder.Name, em2: true);
EM2Country.Content ao = EM2Country.Read(aoFileName, out List <string> aoErrors);
if (ao != null)
{
content.addOns.Add(ao);
}
_errors.AddRange(aoErrors); ReportSuccess(folder.Name, ao != null, aoErrors.Count);
}
;
// READ GLOBAL FILES
string configPath = pathHandler.GetFolderConfig(em2: true);
content.exRates = EM2Global.ReadExRates(configPath, out List <string> exErrors); _errors.AddRange(exErrors);
ReportSuccess("Exchange rates", exErrors.Count == 0, exErrors.Count);
content.hicp = EM2Global.ReadHICP(configPath, out List <string> hicpErrors); _errors.AddRange(hicpErrors);
ReportSuccess("HICP", hicpErrors.Count == 0, hicpErrors.Count);
content.switchPol = EM2Global.ReadSwitchPol(configPath, out List <string> spErrors); _errors.AddRange(spErrors);
ReportSuccess("Policy switches", spErrors.Count == 0, spErrors.Count);
// READ VARIABLES
content.varConfig = EM2Variables.Read(configPath, out List <string> vErrors); _errors.AddRange(vErrors);
ReportSuccess(EM2Variables.FILE_VARCONFIG, content.varConfig != null, vErrors.Count);
return(content);
}
catch (OperationCanceledException) { progressAction($"Reading {emPath} cancelled!"); return(content); }
catch (Exception exception)
{
lock (writeLock) { _errors.Add(exception.Message); } return(null);
}
finally { errors = _errors; }
void ReportSuccess(string what, bool success, int cntErrors)
{
if (success)
{
progressAction($"Finished reading {what} with " + $"{(cntErrors == 0 ? "success" : $"{cntErrors} errors") }");
}
private static bool HandleExtensionSwitches(Dictionary <string, string> config, out Dictionary <string, string> modifiedExtensionSwitches)
{
modifiedExtensionSwitches = new Dictionary <string, string>();
Dictionary <string, string> toReplace = new Dictionary <string, string>();
foreach (var entry in config)
{
if (!entry.Key.StartsWith(TAGS.EXTENSION_SWITCH))
{
continue;
}
string[] splitVal = entry.Value.Split("="); if (splitVal.Count() < 2)
{
continue;
}
string extName = splitVal[0], extVal = splitVal[1];
modifiedExtensionSwitches.Add(extName, extVal);
if (!EM_Helpers.IsGuid(extName))
{
toReplace.Add(entry.Key, extName.ToLower());
}
}
if (toReplace.Count == 0)
{
return(true);
}
EMPath pathHandler = new EMPath(config[TAGS.CONFIG_PATH_EUROMODFILES]);
Communicator dummyCommunicator = new Communicator();
var globExt = ExeXmlReader.ReadExtensions(pathHandler.GetExtensionsFilePath(), dummyCommunicator);
// first search in global file ...
List <string> done = new List <string>();
foreach (var tr in toReplace)
{
string extName = tr.Value, configKey = tr.Key;
// the name of the extension must be replaced by the id:
foreach (var e in globExt)
{
string extId = e.Key, shortName = e.Value.Item1.ToLower(), longName = e.Value.Item2.ToLower();
if (extName == shortName || extName == longName)
{
Replace(configKey, extId); break;
}
}
}
// ... if any extension-name was not found in global file, search in country file
if (done.Count == toReplace.Count)
{
return(true);
}
ExeXml.Country country = ExeXmlReader.ReadCountry(pathHandler.GetCountryFilePath(config[TAGS.CONFIG_COUNTRY]),
config[TAGS.CONFIG_ID_SYSTEM], config[TAGS.CONFIG_ID_DATA], false, dummyCommunicator);
foreach (var tr in toReplace)
{
string extName = tr.Value, configKey = tr.Key;
if (done.Contains(configKey))
{
continue;
}
foreach (var e in from ce in country.extensions where ce.Value.localShortName != null select ce)
{
string extId = e.Key, shortName = e.Value.localShortName.ToLower(),
longName = e.Value.localLongName; if (longName != null)
{
longName = longName.ToLower();
}
if (extName == shortName || extName == longName)
{
Replace(configKey, extId); break;
}
}
}
if (done.Count == toReplace.Count)
{
return(true);
}
string notDone = string.Empty; foreach (var tr in toReplace)
{
if (!done.Contains(tr.Key))
{
notDone += tr.Value + " ";
}
}
Console.WriteLine($"Unknown extension(s): {notDone}");
return(false);
void Replace(string configKey, string extId)
{
config[configKey] = $"{extId}={config[configKey].Split("=")[1]}"; done.Add(configKey);
}
}
internal static int GoPet(string[] args)
{
try
{
if (args.Length < 2)
{
return(WrongArgumentError());
}
string emPath = EM_Helpers.RemoveQuotes(args[0].Trim());
string country = args[1];
bool? transformGlobal = EM_Helpers.GetBool(args[2]);
// other than for the UI-call, the country-files are saved in the temp-folder, thus we cannot call the standard EM3Country.Transform, etc.
// as a solution (which is considered to be temporary and therefore should touch other code as less as possible)
// we do "by hand read-write" (see below) and create an EM3Translation-folder in the temp-folder from where we start the PET-runs
// here we have the information: EM2 country-file (in temp-folder) and global-files (in config-folder)
EMPath em2PathHandler = new EMPath(emPath);
// here we write the EM3 files in a structure the EM3-exe can process, i.e. we create EM3Translation-folder in temp-folder
EMPath transPathHandler = new EMPath(em2PathHandler.GetFolderTemp());
List <string> errors = new List <string>();
bool success = TransformCountry() && (transformGlobal == false || (TransformGlobals() && TransformVariables()));
foreach (string error in errors)
{
Console.Error.WriteLine(error);
}
return(success ? 0 : 1);
bool TransformCountry()
{
// read EM2-files
string ccPath = Path.Combine(em2PathHandler.GetFolderTemp(), EMPath.GetCountryFileName(country));
string dcPath = Path.Combine(em2PathHandler.GetFolderTemp(), EMPath.GetEM2DataConfigFileName(country));
EM2Country.Content ctryContent = EM2Country.Read(ccPath, out List <string> _errors); errors.AddRange(_errors);
EM2Data.Content dataContent = EM2Data.Read(dcPath, out _errors); errors.AddRange(_errors);
List <List <MultiProp> > extensions = EM2Global.ReadSwitchPol(em2PathHandler.GetFolderConfig(em2: true), out _errors); errors.AddRange(_errors);
if (ctryContent == null || dataContent == null || extensions == null)
{
return(false);
}
// write EM3-file
DirectoryInfo di = Directory.CreateDirectory(transPathHandler.GetCountryFolderPath(country));
string em3CountryFile = transPathHandler.GetCountryFilePath(country);
bool ok = EM3Country.Write(ctryContent, dataContent, extensions, em3CountryFile, out _errors); errors.AddRange(_errors);
return(ok);
}
bool TransformGlobals()
{
// read EM2-files
List <List <MultiProp> > exRates = EM2Global.ReadExRates(em2PathHandler.GetFolderConfig(em2: true), out List <string> _errors); errors.AddRange(_errors);
List <List <MultiProp> > hicp = EM2Global.ReadHICP(em2PathHandler.GetFolderConfig(em2: true), out _errors); errors.AddRange(_errors);
List <List <MultiProp> > switches = EM2Global.ReadSwitchPol(em2PathHandler.GetFolderConfig(em2: true), out _errors); errors.AddRange(_errors);
// transfer to EM3-structure
bool ok = true;
try
{
DirectoryInfo di = Directory.CreateDirectory(transPathHandler.GetFolderConfig());
if (!EM3Global.WriteExRates(exRates, transPathHandler.GetFolderEuromodFiles(), out _errors))
{
ok = false;
}
errors.AddRange(_errors);
if (!EM3Global.WriteHICP(hicp, transPathHandler.GetFolderEuromodFiles(), out _errors))
{
ok = false;
}
errors.AddRange(_errors);
if (!EM3Global.WriteExtensions(switches, transPathHandler.GetFolderEuromodFiles(), out _errors))
{
ok = false;
}
errors.AddRange(_errors);
}
catch { } // this is a primitive way for provide thread-security, as more than one PETs may be started
return(ok);
}
bool TransformVariables()
{
// read EM2-file
EM2Variables.Content content = EM2Variables.Read(em2PathHandler.GetFolderConfig(em2: true), out List <string> _errors); errors.AddRange(_errors);
if (content == null)
{
return(false);
}
// transfer to EM3-structure
try
{
DirectoryInfo di = Directory.CreateDirectory(transPathHandler.GetFolderConfig());
bool ok = EM3Variables.Write(content, transPathHandler.GetFolderEuromodFiles(), out _errors); errors.AddRange(_errors);
return(ok);
}
catch { return(true); } // see above
}
}
catch (Exception exception)
{
Console.Error.WriteLine("EUROMOD3 TRANSFORM: " + exception.Message);
return(1);
}
int WrongArgumentError()
{
string actArgs = string.Empty; foreach (string arg in args)
{
actArgs += arg + " ";
}
Console.Error.WriteLine("EUROMOD3 TRANSFORM: invalid use. Correct: EM_ExecutableCaller UI_TRANSFORM_PET emPath 0/1 cc [ao1[|ao2|...|aoN]]" +
Environment.NewLine + $"Acutal use: EM_ExecutableCaller TRANSFORM {actArgs}");
return(1);
}
}
/// <summary> note: this is a private function, exclusively called by EM3All.Transform, just for reasons of clearer arrangement </summary>
private static bool Write(EM2All.Content content, string emPath, out List <string> errors,
Action <string> report = null, CancellationTokenSource cancelSrc = null)
{
List <string> _errors = new List <string>();
object errWriteLock = new object();
ParallelOptions parallelOptions = new ParallelOptions();
if (cancelSrc != null)
{
parallelOptions.CancellationToken = cancelSrc.Token;
}
try
{
EMPath pathHandler = new EMPath(emPath);
// WRITE COUNTRIES IN PARALLEL
Parallel.ForEach(content.countries, parallelOptions, country =>
{
parallelOptions.CancellationToken.ThrowIfCancellationRequested();
string countryName = country.country.general.properties[EM2TAGS.SHORTNAME];
DirectoryInfo di = Directory.CreateDirectory(pathHandler.GetCountryFolderPath(countryName));
bool success = EM3Country.Write(country.country, country.data, content.switchPol,
pathHandler.GetCountryFilePath(countryName), out List <string> cErrors);
if (cErrors.Count > 0)
{
lock (errWriteLock) { _errors.AddRange(cErrors); }
}
ReportSuccess(countryName, success, cErrors.Count);
if (success && cErrors.Count == 0) // produce up2Date-files
{
TransformerCommon.WriteUpToDate(pathHandler.GetCountryFilePath(countryName, true), pathHandler.GetCountryFolderPath(countryName));
TransformerCommon.WriteUpToDate(pathHandler.GetEM2DataConfigFilePath(countryName), pathHandler.GetCountryFolderPath(countryName));
}
});
// the rest is written sequentially (parallel would be unnecessary overhead)
// WRITE ADD-ONS
foreach (var addOn in content.addOns)
{
string addOnName = addOn.general.properties[EM2TAGS.SHORTNAME];
DirectoryInfo di = Directory.CreateDirectory(pathHandler.GetAddOnFolderPath(addOnName));
bool success = EM3Country.Write(addOn, null, content.switchPol,
pathHandler.GetAddOnFilePath(addOnName), out List <string> aoErrors);
if (aoErrors.Count > 0)
{
_errors.AddRange(aoErrors);
}
ReportSuccess(addOnName, success, aoErrors.Count);
// produce up2Date-file
if (success && aoErrors.Count == 0)
{
TransformerCommon.WriteUpToDate(pathHandler.GetAddOnFilePath(addOnName, true), pathHandler.GetAddOnFolderPath(addOnName));
}
}
// WRITE GLOBAL FILES
if (!Directory.Exists(pathHandler.GetFolderConfig()))
{
Directory.CreateDirectory(pathHandler.GetFolderConfig());
}
bool erSuccess = EM3Global.WriteExRates(content.exRates, emPath, out List <string> erErrors);
ReportSuccess("Exchangerates", erSuccess, erErrors.Count); _errors.AddRange(erErrors);
bool hicSuccess = EM3Global.WriteHICP(content.hicp, emPath, out List <string> hicErrors);
ReportSuccess("HICP", hicSuccess, hicErrors.Count); _errors.AddRange(hicErrors);
bool gsSuccess = EM3Global.WriteExtensions(content.switchPol, emPath, out List <string> gsErrors);
ReportSuccess("Extensions", gsSuccess, gsErrors.Count); _errors.AddRange(gsErrors);
// WRITE VARIABLES
bool varSuccess = EM3Variables.Write(content.varConfig, emPath, out List <string> vErrors);
ReportSuccess("Variables", varSuccess, vErrors.Count); _errors.AddRange(vErrors);
if (varSuccess && vErrors.Count == 0)
{
TransformerCommon.WriteUpToDate(pathHandler.GetVarFilePath(true), pathHandler.GetFolderConfig()); // produce up2Date-file
}
return(true);
}
catch (OperationCanceledException) { report($"Writing {emPath} cancelled!"); return(true); }
catch (Exception exception)
{
lock (errWriteLock) { _errors.Add(exception.Message); }
return(false);
}
finally { errors = _errors; }
void ReportSuccess(string what, bool success, int cntErrors)
{
if (success)
{
report($"Finished writing {what} with " + $"{(cntErrors == 0 ? "success" : $"{cntErrors} errors") }");
}
