private static List <string> WriteExcelResultFiles([NotNull] ScenarioSliceParameters parameters,
[NotNull] Func <string, ScenarioSliceParameters, bool, string> makeAndRegisterFullFilename,
ProcessingMode processingMode,
[NotNull] ProfileGenerationRo pgRo)
{
List <string> excelFileNames = new List <string>();
var fn1 = makeAndRegisterFullFilename("AllGeneratedLoadProfilesAndEnergy." + processingMode + ".Tree.xlsx", parameters, true);
excelFileNames.Add(fn1);
pgRo.DumpToExcel(fn1, XlsResultOutputMode.Tree);
var fn2 = makeAndRegisterFullFilename("AllGeneratedLoadProfilesAndEnergy." + processingMode + ".Full.xlsx", parameters, true);
pgRo.DumpToExcel(fn2, XlsResultOutputMode.FullLine);
excelFileNames.Add(fn2);
var fn3 = makeAndRegisterFullFilename("AllGeneratedLoadProfilesAndEnergy." + processingMode + ".ByTrafoStationTree.xlsx",
parameters,
true);
excelFileNames.Add(fn3);
pgRo.DumpToExcel(fn3, XlsResultOutputMode.ByTrafoStationTree);
var fn4 = makeAndRegisterFullFilename("AllGeneratedLoadProfilesAndEnergy." + processingMode + ".ByTrafoStationHausanschlussTree.xlsx",
parameters,
true);
excelFileNames.Add(fn4);
pgRo.DumpToExcel(fn4, XlsResultOutputMode.ByTrafoStationHausanschlussTree);
return(excelFileNames);
}
public void RunTest()
{
CompositeResolver.RegisterAndSetAsDefault(NativeDateTimeResolver.Instance, StandardResolver.Instance);
PrepareUnitTest();
Config.Directories.ResultStorageDirectory = WorkingDirectory.Dir;
Config.Directories.CalcServerLpgDirectory = WorkingDirectory.Dir;
// ReSharper disable twice AssignNullToNotNullAttribute
HouseCreationAndCalculationJob hcj = new HouseCreationAndCalculationJob(Scenario.Present().ToString(), "2017", "trafokreis");
HouseData hd = new HouseData("houseguid", "HT01", 1000, 1000, "houseName");
HouseholdData hhd = new HouseholdData("householdguid",
2000,
ElectricCarUse.UseElectricCar,
"householdname",
ElectricCarProvider.ChargingStationSet,
ElectricCarProvider.TransportationDevicesOneCar,
ElectricCarProvider.TravelRouteSet,
new List <TransportationDistanceModifier>(),
HouseholdDataSpecifictionType.ByPersons);
hd.Households.Add(hhd);
hhd.UseElectricCar = ElectricCarUse.UseElectricCar;
hhd.TransportationDeviceSet = ElectricCarProvider.TransportationDevicesOneCar;
hhd.TravelRouteSet = ElectricCarProvider.TravelRouteSet;
hhd.ChargingStationSet = ElectricCarProvider.ChargingStationSet;
hhd.HouseholdDataPersonSpecification = new HouseholdDataPersonSpecification(new List <PersonData> {
new PersonData(30, Gender.Male)
});
hcj.House = hd;
List <HouseCreationAndCalculationJob> houseJobs = new List <HouseCreationAndCalculationJob>();
houseJobs.Add(hcj);
FileHelpers.CopyRec(Config.Directories.LPGReleaseDirectory, WorkingDirectory.Dir, Logger, true);
var endTime = new DateTime(Constants.PresentSlice.DstYear, 1, 10);
ProfileGenerationRo pgro = new ProfileGenerationRo();
HouseProcessor.WriteDistrictsForLPG(houseJobs,
WorkingDirectory.DirDi,
Logger,
Constants.PresentSlice,
endTime,
pgro);
string districtsDir = WorkingDirectory.Combine("Districts");
var districtsDi = new DirectoryInfo(districtsDir);
var files = districtsDi.GetFiles("*.json");
void RunOneFile(FileInfo myfi)
{
ProcessStartInfo psi = new ProcessStartInfo();
psi.FileName = WorkingDirectory.Combine("simulationengine.exe");
psi.UseShellExecute = true;
psi.WorkingDirectory = WorkingDirectory.Dir;
psi.Arguments = "ProcessHouseJob -j \"" + myfi.FullName + "\"";
Info("running " + psi.FileName + " " + psi.Arguments);
using (Process p = new Process()) {
p.StartInfo = psi;
p.Start();
p.WaitForExit();
}
}
foreach (var housejob in files)
{
RunOneFile(housejob);
}
DBDto dbDto = new DBDto(new List <House>(), new List <Hausanschluss>(), new List <Car>(), new List <Household>(), new List <RlmProfile>());
CachingLPGProfileLoader ca = new CachingLPGProfileLoader(Logger, dbDto);
List <int> isns = new List <int>();
isns.Add(10);
CarDistanceEntry cde = new CarDistanceEntry("houseguid",
"householdguid",
"carguid",
20,
20,
isns,
10,
"haguid",
"sourceguid",
"cdename",
CarType.Electric);
HouseComponentRo hcro = new HouseComponentRo("housecomponent", "componeenttype", 1000, 200, "processingstatus", "isns", "standort", 0);
ProviderParameterDto ppd = new ProviderParameterDto(cde, WorkingDirectory.Dir, hcro);
SqlConnectionPreparer scp = new SqlConnectionPreparer(Config);
MyDb db = scp.GetDatabaseConnection(Stage.Testing, Constants.PresentSlice);
SaveableEntry <Profile> sa = SaveableEntry <Profile> .GetSaveableEntry(db, SaveableEntryTableType.LPGProfile, Logger);
sa.MakeTableForListOfFieldsIfNotExists(true);
string dstDir = Path.Combine(WorkingDirectory.Dir, hcj.Trafokreis, hcj.House.Name);
FileHelpers.CopyRec(WorkingDirectory.Combine("Results"), dstDir, Logger, true);
//normal electricity test and cache test
Info("================== ");
Info("electricity");
Info("================== ");
var profElec1 = ca.LoadLPGProfile(ppd,
hcj.Trafokreis,
"Electricity",
sa,
hhd.HouseholdGuid,
out var profsource,
hcj.House.Name,
Config,
true);
Info("Source: " + profsource);
Assert.NotNull(profElec1);
Assert.NotNull(profsource);
var profElecCache = ca.LoadLPGProfile(ppd,
hcj.Trafokreis,
"Electricity",
sa,
hhd.HouseholdGuid,
out var profsourceCache,
hcj.House.Name,
Config,
true);
Info("Source 2: " + profsourceCache);
Assert.NotNull(profsourceCache);
Assert.NotNull(profsource);
profElec1.Should().BeEquivalentTo(profElecCache, options => options.Excluding(ctx => ctx.SelectedMemberPath.EndsWith("BinaryProfile")));
//Car Charging Electricity electricity test and cache test
Info("================== ");
Info("Car Charging Electricity electricity");
Info("================== ");
SaveableEntry <Profile> sa2 = SaveableEntry <Profile> .GetSaveableEntry(db, SaveableEntryTableType.EvProfile, Logger);
sa2.MakeCleanTableForListOfFields(true);
var prof2 = ca.LoadLPGProfile(ppd,
hcj.Trafokreis,
"Car Charging Electricity",
sa2,
hhd.HouseholdGuid,
out var profsource2,
hcj.House.Name,
Config,
true);
Info("Source Wp 1: " + profsource2);
Assert.NotNull(prof2);
Assert.NotNull(profsource2);
var prof3 = ca.LoadLPGProfile(ppd,
hcj.Trafokreis,
"Car Charging Electricity",
sa2,
hhd.HouseholdGuid,
out var profsource3,
hcj.House.Name,
Config,
true);
Info("Source Wp 2: " + profsource3);
Assert.NotNull(prof3);
Assert.NotNull(profsource3);
prof2.Should().BeEquivalentTo(prof3, options => options.Excluding(ctx =>
ctx.SelectedMemberPath.EndsWith("BinaryProfile")));
}
public void ProcessAllHouses([NotNull] ScenarioSliceParameters parameters,
[NotNull] Func <string, ScenarioSliceParameters, bool, string> makeAndRegisterFullFilename,
ProcessingMode processingMode,
[NotNull][ItemNotNull] List <string> developmentStatus)
{
if (!Directory.Exists(_processingResultPathForProfiles))
{
Directory.CreateDirectory(_processingResultPathForProfiles);
Thread.Sleep(500);
}
else
{
var dstDi = new DirectoryInfo(_processingResultPathForProfiles);
var files = dstDi.GetFiles();
Info("Cleaning " + files.Length + " files from result directory " + _processingResultPathForProfiles);
foreach (var file in files)
{
file.Delete();
}
}
var dbHouses = _services.SqlConnectionPreparer.GetDatabaseConnection(Stage.Houses, parameters);
Info("using house db in " + dbHouses.ConnectionString);
var houses = dbHouses.Fetch <House>();
HouseComponentRepository hcr = new HouseComponentRepository(dbHouses);
var households = dbHouses.Fetch <Household>();
var hausanschlusses = dbHouses.Fetch <Hausanschluss>();
var cars = dbHouses.Fetch <Car>();
var dbRaw = _services.SqlConnectionPreparer.GetDatabaseConnection(Stage.Raw, Constants.PresentSlice);
var measuredRlmProfiles = dbRaw.Fetch <RlmProfile>();
DBDto dbdto = new DBDto(houses, hausanschlusses, cars, households, measuredRlmProfiles);
ProsumerComponentResultArchiver pra = new ProsumerComponentResultArchiver(Stage.ProfileGeneration, parameters, processingMode, _services);
var trafokreiseToProcess = PrepareListOfTrafokreiseToProcess(hausanschlusses);
List <HouseCreationAndCalculationJob> districts = new List <HouseCreationAndCalculationJob>();
var vdewValues = dbRaw.Fetch <VDEWProfileValue>();
var feiertage = dbRaw.Fetch <FeiertagImport>();
var slpProvider = new SLPProvider(parameters.DstYear, vdewValues, feiertage);
var loadProfileProviders = MakeDiContrainer(parameters, developmentStatus, hausanschlusses, houses, districts, slpProvider, dbdto);
if (processingMode == ProcessingMode.Collecting)
{
ClearAllExistingExportProfiles();
}
var lpgDirectoryInfo = GetLPGCalcDirectoryInfo(parameters);
ProfileGenerationRo pgRo = new ProfileGenerationRo();
DateTime startTime = DateTime.Now;
DateTime lastLog = DateTime.Now;
Info("Processing " + houses.Count + " houses.");
Info("Processing mode is " + processingMode);
List <string> brokenLpgDirectories = new List <string>();
List <string> brokenLpgJsons = new List <string>();
Dictionary <string, List <HouseComponentEntry> > houseComponentsByObjectID = BuildDictionaryByObjektID(houses,
hcr,
hausanschlusses,
pgRo,
out var numberOfcomponents);
int processedComponents = 0;
Stopwatch swCollecting = new Stopwatch();
Stopwatch swWriting = new Stopwatch();
Dictionary <string, int> numberOfEmptyProsumers = new Dictionary <string, int>();
HashSet <string> validHouseGuids = houses.Select(x => x.Guid).ToHashSet();
foreach (KeyValuePair <string, List <HouseComponentEntry> > pair in houseComponentsByObjectID)
{
var houseProsumers = new List <Prosumer>();
//erst alle profile einsammeln / vorbereiten
swCollecting.Start();
foreach (var component in pair.Value)
{
processedComponents++;
if (processedComponents % 1000 == 0)
{
Info(processingMode + " processed Components " + processedComponents + " / " + numberOfcomponents);
}
ProviderParameterDto ppdto = new ProviderParameterDto(component.Component, lpgDirectoryInfo.FullName, pgRo[component.Component]);
var provider = loadProfileProviders.GetCorrectProvider(component.Component);
pgRo[component.Component].UsedProvider = provider.Name;
if (processingMode == ProcessingMode.Preparing)
{
provider.PrepareLoadProfileIfNeeded(ppdto);
}
if (processingMode == ProcessingMode.Collecting)
{
if (trafokreiseToProcess.Contains(component.Hausanschluss.Trafokreis))
{
Prosumer p = provider.ProvideProfile(ppdto);
//Todo: add up profile per trafokreis, profile per provider etc right here
if (p != null)
{
// some providers that are not ready will return null
if (p.Profile?.EnergyOrPower != EnergyOrPower.Energy)
{
throw new FlaException("Got a power profile from " + provider.Name);
}
CheckProfileIntegrity(p, provider, component.Component, validHouseGuids);
// ReSharper disable once AssignNullToNotNullAttribute
pgRo[component.Component].AddProsumerInformation(p);
houseProsumers.Add(p);
pra.Archive(p);
}
else
{
if (!numberOfEmptyProsumers.ContainsKey(provider.Name))
{
numberOfEmptyProsumers.Add(provider.Name, 0);
}
numberOfEmptyProsumers[provider.Name]++;
}
}
}
}
swCollecting.Stop();
//dann alles rausschreiben
swWriting.Start();
if (processingMode == ProcessingMode.Collecting)
{
var generationProsumers = houseProsumers.Where(x => x.GenerationOrLoad == GenerationOrLoad.Generation).ToList();
var loadProsumers = houseProsumers.Where(x => x.GenerationOrLoad == GenerationOrLoad.Load).ToList();
var component = pair.Value[0];
if (loadProsumers.Count > 0)
{
Prosumer summedProsumer = GetSummedProsumer(loadProsumers, component.Hausanschluss.Trafokreis);
var haros = pgRo.HausanschlussByObjectId(pair.Key);
double maxPower = summedProsumer.Profile?.MaxPower() ?? 0;
foreach (var haro in haros)
{
haro.MaximumPower = maxPower;
}
WriteSumLineToCsv(summedProsumer, component.Hausanschluss.Trafokreis, GenerationOrLoad.Load);
}
if (generationProsumers.Count > 0)
{
var sumProfile = GetSummedProsumer(generationProsumers, component.Hausanschluss.Trafokreis);
WriteSumLineToCsv(sumProfile, component.Hausanschluss.Trafokreis, GenerationOrLoad.Generation);
// ReSharper disable once PossibleNullReferenceException
}
}
swWriting.Stop();
ReportProgress(startTime, processedComponents, numberOfcomponents, parameters, ref lastLog, swCollecting, swWriting);
}
Info("Finished processing all components, finishing up now. Duration: " + (DateTime.Now - startTime).TotalMinutes.ToString("F2") +
" minutes");
Info("collecting took " + swCollecting.Elapsed + " and writing took " + swWriting.Elapsed);
foreach (var pair in numberOfEmptyProsumers)
{
Info("NumberOfEmptyProsumers for " + pair.Key + " was " + pair.Value);
}
if (processingMode == ProcessingMode.Preparing)
{
DateTime endtime = new DateTime(parameters.DstYear, 12, 31);
string houseJobDirectory = Path.Combine(_services.RunningConfig.Directories.HouseJobsDirectory,
parameters.DstScenario.ToString(),
parameters.DstYear.ToString());
DirectoryInfo houseJobDi = new DirectoryInfo(houseJobDirectory);
WriteDistrictsForLPG(districts, houseJobDi, _services.Logger, parameters, endtime, pgRo);
}
else
{
DateTime startSavingToDB = DateTime.Now;
pra.FinishSavingEverything();
Info("Finished writing prosumers to db. Duration: " + (DateTime.Now - startSavingToDB).TotalMinutes.ToString("F2") + " minutes");
}
pra.Dispose();
var excelFiles = WriteExcelResultFiles(parameters, makeAndRegisterFullFilename, processingMode, pgRo);
if (_services.RunningConfig.CollectFilesForArchive)
{
SaveToArchiveDirectory(parameters, excelFiles, _services.StartingTime, _services.RunningConfig);
}
WriteBrokenLpgCalcCleanupBatch(parameters, makeAndRegisterFullFilename, processingMode, brokenLpgDirectories, brokenLpgJsons);
if (processingMode == ProcessingMode.Collecting)
{
foreach (var provider in loadProfileProviders.Providers)
{
provider.DoFinishCheck();
}
}
}
private Dictionary <string, List <HouseComponentEntry> > BuildDictionaryByObjektID([NotNull][ItemNotNull] List <House> houses,
[NotNull] HouseComponentRepository hcr,
[NotNull][ItemNotNull] List <Hausanschluss> hausanschlusses,
[NotNull] ProfileGenerationRo pgRo,
out int numberOfcomponents)
{
var houseComponentsByObjectID = new Dictionary <string, List <HouseComponentEntry> >();
numberOfcomponents = 0;
int housecount = 0;
foreach (House house in houses)
{
pgRo.AddHouse(house);
var houseComponents = house.CollectHouseComponents(hcr);
pgRo[house].NumberOfComponents = houseComponents.Count;
var hausanschlussGuids = houseComponents.Where(x => x.HausAnschlussGuid != null).Select(x => x.HausAnschlussGuid).Distinct().ToList();
foreach (string hausanschlussGuid in hausanschlussGuids)
{
var hausanschluss = hausanschlusses.First(x => x.Guid == hausanschlussGuid);
pgRo.AddHausanschluss(house, hausanschluss, "Mit Komponenten");
var anschlusshouseComponents = houseComponents.Where(x => x.HausAnschlussGuid == hausanschlussGuid).ToList();
if (!houseComponentsByObjectID.ContainsKey(hausanschluss.ObjectID))
{
houseComponentsByObjectID.Add(hausanschluss.ObjectID, new List <HouseComponentEntry>());
}
foreach (IHouseComponent component in anschlusshouseComponents)
{
houseComponentsByObjectID[hausanschluss.ObjectID].Add(new HouseComponentEntry(component, hausanschluss, house));
numberOfcomponents++;
pgRo.AddHouseComponent(house, hausanschluss, component);
}
}
foreach (var hausanschluss in house.Hausanschluss)
{
if (hausanschlussGuids.Contains(hausanschluss.Guid))
{
continue;
}
pgRo.AddHausanschluss(house, hausanschluss, "Ohne Komponenten");
}
housecount++;
if (_services.RunningConfig.LimitLoadProfileGenerationToHouses > 0)
{
if (housecount > _services.RunningConfig.LimitLoadProfileGenerationToHouses)
{
break;
}
}
}
return(houseComponentsByObjectID);
}
public static void WriteDistrictsForLPG([NotNull][ItemNotNull] List <HouseCreationAndCalculationJob> houses,
[NotNull] DirectoryInfo di,
[NotNull] ILogger logger,
[NotNull] ScenarioSliceParameters slice,
DateTime endDateForCalc,
[NotNull] ProfileGenerationRo pgRo)
{
var calcSpec = MakeCalcSpec(slice, endDateForCalc);
//first write all the districts
logger.Info("starting to write the lpg housejobs", Stage.ProfileGeneration, nameof(HouseProcessor));
//next write the filtered districts
var directory = Path.Combine(di.FullName, "Districts");
var filteredHouses = new List <HouseCreationAndCalculationJob>();
int skippedCount = 0;
foreach (HouseCreationAndCalculationJob houseJob in houses)
{
//filter houses that don't need to calculated
if (houseJob.House.Households.Count == 0)
{
continue;
}
bool allHouseholdsFound = true;
bool allCarsFound = true;
List <HouseholdData> missingHouseholds = new List <HouseholdData>();
List <HouseholdData> missingCars = new List <HouseholdData>();
foreach (HouseholdData householdJob in houseJob.House.Households)
{
if (!householdJob.IsHouseholdProfileCalculated)
{
allHouseholdsFound = false;
missingHouseholds.Add(householdJob);
}
if (!householdJob.IsCarProfileCalculated && householdJob.UseElectricCar == ElectricCarUse.UseElectricCar)
{
allCarsFound = false;
missingCars.Add(householdJob);
}
}
var houseRo = pgRo.FindByHousename(houseJob.House.Name);
if (allHouseholdsFound && allCarsFound)
{
skippedCount++;
houseRo.LpgCalculationStatus = "All Calculations Finished";
continue;
}
var householdsWithCars = houseJob.House.Households.Where(x => x.UseElectricCar == ElectricCarUse.UseElectricCar).ToList();
houseRo.LpgCalculationStatus = "Incomplete, " + missingHouseholds.Count + "/" + houseJob.House.Households.Count +
" households missing, " + +missingCars.Count + " / " + householdsWithCars.Count + " cars missing";
filteredHouses.Add(houseJob);
}
logger.Info("Skipping " + skippedCount + " houses due to complete results in cache", Stage.ProfileGeneration, nameof(HouseProcessor));
WriteHousejobsToDirectory(directory, filteredHouses, calcSpec, logger);
}