public void HouseGeneratorTestWithHouseholdSpec()
{
//setup
Logger.Get().StartCollectingAllMessages();
using (DatabaseSetup db = new DatabaseSetup(Utili.GetCurrentMethodAndClass())) {
Simulator sim = new Simulator(db.ConnectionString);
using (WorkingDir wd = new WorkingDir(Utili.GetCurrentMethodAndClass())) {
//housedata
HouseData houseData = new HouseData(Guid.NewGuid().ToStrGuid(),
"HT01", 10000, 1000, "HouseGeneratorJobHouse");
var householdData = new HouseholdData(Guid.NewGuid().ToString(),
"blub", null, null, null, null,
HouseholdDataSpecificationType.ByHouseholdName);
houseData.Households.Add(householdData);
householdData.HouseholdNameSpec =
new HouseholdNameSpecification(sim.ModularHouseholds[0].GetJsonReference());
HouseCreationAndCalculationJob houseJob =
new HouseCreationAndCalculationJob("present", "2019", "trafokreis",
HouseDefinitionType.HouseData);
houseJob.House = houseData;
MakeAndCalculateHouseJob(houseJob, sim, wd, db);
}
}
}
public static HouseCreationAndCalculationJob PrepareExistingHouseForTesting([JetBrains.Annotations.NotNull] House house)
{
var hj = new HouseCreationAndCalculationJob();
hj.CalcSpec = JsonCalcSpecification.MakeDefaultsForTesting();
hj.HouseDefinitionType = HouseDefinitionType.HouseName;
hj.HouseRef = new HouseReference(house.GetJsonReference());
return(hj);
}
private void ReportFinishedCalcJob([NotNull] MessageFromServerToClient job,
[NotNull] RequestSocket client,
[CanBeNull] HouseCreationAndCalculationJob hcj)
{
var msg = new MessageFromClientToServer(ClientRequestEnum.ReportFinish, _threadId.Name, "finished calculation", job.TaskGuid);
if (hcj != null)
{
msg.Scenario = hcj.Scenario;
msg.Year = hcj.Year;
msg.Trafokreis = hcj.Trafokreis;
msg.HouseName = hcj.House?.Name;
}
string resultDirectory = Path.Combine(_mySettings.ClientSettings.LPGCalcDirectory, "Results");
DirectoryInfo di = new DirectoryInfo(resultDirectory);
if (di.Exists)
{
var files = di.GetFiles("*.*", SearchOption.AllDirectories);
var filteredFiles = new List <FileInfo>();
foreach (var file in files)
{
if (file.Name.ToLower(CultureInfo.InvariantCulture).EndsWith(".dat"))
{
_logger.Error("Refusing dat file: " + file.FullName, _threadId);
continue;
}
if (file.Length > 100_000_000)
{
_logger.Error("Refusing too big file: " + file.FullName, _threadId);
continue;
}
filteredFiles.Add(file);
}
msg.ResultFiles = MsgFile.ReadMsgFiles(true, filteredFiles, di, _logger, _threadId);
var reportAnswer = MakeRequest(client, msg);
_logger.Info("Answer from the finish report:" + reportAnswer.ServerResponse, _threadId);
}
else
{
_logger.Info("No output directory created, reporting failure to the server", _threadId);
var msg2 = new MessageFromClientToServer(ClientRequestEnum.ReportFailure,
_threadId.Name,
"result directory is missing",
job.TaskGuid);
var reportAnswer = MakeRequest(client, msg2);
_logger.Info("Answer from the finish report:" + reportAnswer.ServerResponse, _threadId);
}
}
public void HouseGeneratorTestForPrecreated()
{
//setup
Logger.Get().StartCollectingAllMessages();
using (DatabaseSetup db = new DatabaseSetup(Utili.GetCurrentMethodAndClass())) {
Simulator sim = new Simulator(db.ConnectionString);
using (WorkingDir wd = new WorkingDir(Utili.GetCurrentMethodAndClass())) {
const string fn = @"C:\work\LPGUnitTest\HouseJob.Felseggstrasse 29.json";
string txt = File.ReadAllText(fn);
HouseCreationAndCalculationJob houseJob =
JsonConvert.DeserializeObject <HouseCreationAndCalculationJob>(txt);
MakeAndCalculateHouseJob(houseJob, sim, wd, db);
}
}
}
private static void SetCalcSpec([NotNull] HouseCreationAndCalculationJob hj, [NotNull] Simulator sim)
{
hj.CalcSpec = JsonCalcSpecification.MakeDefaultsForProduction();
hj.CalcSpec.OutputDirectory = "Results";
hj.CalcSpec.CalcOptions = new List <CalcOption>();
hj.CalcSpec.CalcOptions.Add(CalcOption.SumProfileExternalIndividualHouseholdsAsJson);
hj.CalcSpec.DefaultForOutputFiles = OutputFileDefault.NoFiles;
hj.CalcSpec.ExternalTimeResolution = "00:15:00";
hj.CalcSpec.StartDate = new DateTime(2019, 1, 1);
hj.CalcSpec.EndDate = new DateTime(2019, 12, 31);
hj.CalcSpec.TemperatureProfile = sim.TemperatureProfiles[0].GetJsonReference();
var geoloc = sim.GeographicLocations.FindFirstByName("Chemnitz", FindMode.Partial) ?? throw new LPGException("Chemnitz not found");
hj.CalcSpec.GeographicLocation = geoloc.GetJsonReference();
}
public static void WriteJsonToFile([NotNull] string fullPath, [NotNull] HouseCreationAndCalculationJob hj)
{
using (StreamWriter sw = new StreamWriter(fullPath)) {
if (hj.CalcSpec == null)
{
throw new LPGException("Trying to write a house job without a calc spec");
}
if (hj.House == null)
{
throw new LPGException("Trying to write a house job without a house definition");
}
string json = TurnJsonCalcSpecIntoCommentedString(hj);
sw.Write(json);
sw.Close();
}
}
public void Run([NotNull] string directory, [NotNull] string districtDefinitionJsonPath)
{
DirectoryInfo di = new DirectoryInfo(directory);
var subdirs = di.GetDirectories();
List <StatisticsEntry> entries = new List <StatisticsEntry>();
foreach (var subdir in subdirs)
{
var files = subdir.GetFiles("*.json");
foreach (var file in files)
{
string json = File.ReadAllText(file.FullName);
HouseCreationAndCalculationJob jcs = JsonConvert.DeserializeObject <HouseCreationAndCalculationJob>(json);
var e = new StatisticsEntry(file.FullName, jcs, subdir.Name);
entries.Add(e);
}
var database = subdir.GetFiles("*.db3").ToList();
if (database.Count != 1)
{
throw new LPGException("No database found");
}
//string connectionString = "Data Source=" + database[0].FullName;
//Simulator sim = new Simulator(connectionString);
//todo: fix
//foreach (StatisticsEntry entry in entries) {
// var house = sim.Houses.FindByJsonReference(entry.hou);
// if (house != null) {
// entry.HouseholdCount = house.Households.Count;
// entry.TotalPersonCount = house.Households.Sum(x => x.CalcObject?.AllPersons.Count ?? 0);
// }
//}
}
//ReadSourceData(entries, districtDefinitionJsonPath);
StreamWriter sw = new StreamWriter(di.CombineName("Statistics.csv"));
sw.WriteLine("Calc Definition;Name;HouseGuid;Directory;Real Persons;Real Households;Planned Persons;Planned Households;Source File");
foreach (StatisticsEntry statisticsEntry in entries)
{
sw.WriteLine(statisticsEntry.GetCsvLine());
}
sw.Close();
}
public void RunHouseholdTemplate()
{
using (DatabaseSetup db = new DatabaseSetup(Utili.GetCurrentMethodAndClass()))
{
using (WorkingDir wd = new WorkingDir(Utili.GetCurrentMethodAndClass()))
{
Simulator sim = new Simulator(db.ConnectionString);
string dir = wd.Combine("DirectHouseJobs");
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
Random rnd = new Random();
List <string> houseTypes = sim.HouseTypes.It.Select(x => x.Name.Substring(0, x.Name.IndexOf(" ", StringComparison.Ordinal))).ToList();
foreach (var mhh in sim.HouseholdTemplates.It)
{
for (int i = 0; i < 100; i++)
{
HouseCreationAndCalculationJob hj = new HouseCreationAndCalculationJob("TemplatedRandomHouseType", "2019", "TK", HouseDefinitionType.HouseData);
string ht = houseTypes[rnd.Next(houseTypes.Count)];
Logger.Info(ht);
hj.House = new HouseData(Guid.NewGuid().ToStrGuid(), ht, 10000, 10000, "House for " + mhh.Name + " " + i);
hj.House.Households.Add(new HouseholdData(Guid.NewGuid().ToString(), mhh.Name, null, null, null,
null, HouseholdDataSpecificationType.ByTemplateName));
hj.House.Households[0].HouseholdTemplateSpec = new HouseholdTemplateSpecification(mhh.Name);
SetCalcSpec(hj, sim);
if (hj.CalcSpec?.CalcOptions == null)
{
throw new LPGException("calcoption not set");
}
hj.CalcSpec.CalcOptions.Add(CalcOption.EnergyCarpetPlot);
hj.CalcSpec.CalcOptions.Add(CalcOption.HouseSumProfilesFromDetailedDats);
string fn = Path.Combine(dir, AutomationUtili.CleanFileName(mhh.Name) + "." + i + ".json");
File.WriteAllText(fn, JsonConvert.SerializeObject(hj, Formatting.Indented));
}
}
//CopyAll(new DirectoryInfo(dir), new DirectoryInfo(@"X:\HouseJobs\Blockstrom\TemplatedHouses"));
}
}
}
public void HouseJobForHeatpump()
{
//setup
Logger.Get().StartCollectingAllMessages();
using (DatabaseSetup db = new DatabaseSetup(Utili.GetCurrentMethodAndClass())) {
Simulator sim = new Simulator(db.ConnectionString);
using (WorkingDir wd = new WorkingDir(Utili.GetCurrentMethodAndClass())) {
File.Copy(db.FileName, wd.Combine("profilegenerator.db3"));
Directory.SetCurrentDirectory(wd.WorkingDirectory);
//housedata
HouseData houseData = new HouseData(Guid.NewGuid().ToStrGuid(),
"HT01", 10000, 1000, "HouseGeneratorJobHouse");
HouseCreationAndCalculationJob houseJob = new HouseCreationAndCalculationJob(
"present", "2019", "trafokreis", HouseDefinitionType.HouseData);
houseJob.House = houseData;
var householdData = new HouseholdData(Guid.NewGuid().ToString(),
"blub", null, null, null, null, HouseholdDataSpecificationType.ByPersons);
houseData.Households.Add(householdData);
var persons = new List <PersonData> {
new PersonData(30, Gender.Male)
};
householdData.HouseholdDataPersonSpec = new HouseholdDataPersonSpecification(persons);
houseJob.CalcSpec = new JsonCalcSpecification {
DefaultForOutputFiles = OutputFileDefault.NoFiles,
StartDate = new DateTime(2017, 1, 1),
EndDate = new DateTime(2017, 1, 3),
GeographicLocation = sim.GeographicLocations.FindFirstByName("Berlin", FindMode.Partial)
?.GetJsonReference() ??
throw new LPGException("No Berlin in the DB"),
TemperatureProfile = sim.TemperatureProfiles[0].GetJsonReference(),
OutputDirectory = wd.Combine("Results"),
CalcOptions = new List <CalcOption> {
CalcOption.HouseSumProfilesFromDetailedDats, CalcOption.DeviceProfilesIndividualHouseholds,
CalcOption.EnergyStorageFile, CalcOption.EnergyCarpetPlot,
CalcOption.HouseholdContents
}
};
StartHouseJob(houseJob, wd, "xxx");
}
}
}
public void HouseGeneratorTestWithPersonSpecAndTransport()
{
//setup
Logger.Get().StartCollectingAllMessages();
using (WorkingDir workingDir = new WorkingDir(Utili.GetCurrentMethodAndClass())) {
using (DatabaseSetup db = new DatabaseSetup(Utili.GetCurrentMethodAndClass())) {
Simulator sim = new Simulator(db.ConnectionString);
WorkingDir wd = workingDir;
//housedata
HouseData houseData = new HouseData(Guid.NewGuid().ToStrGuid(),
"HT01", 10000, 1000, "HouseGeneratorJobHouse");
var chargingStationSet = sim.ChargingStationSets
.SafeFindByName("Charging At Home with 03.7 kW, output results to Car Electricity")
.GetJsonReference();
Logger.Info("Using charging station " + chargingStationSet);
var transportationDeviceSet = sim.TransportationDeviceSets[0].GetJsonReference();
var travelRouteSet = sim.TravelRouteSets[0].GetJsonReference();
var work = new TransportationDistanceModifier("Work", "Car", 0);
var entertainment = new TransportationDistanceModifier("Entertainment", "Car", 12000);
List <TransportationDistanceModifier> tdm = new List <TransportationDistanceModifier>
{
work, entertainment
};
var householdData = new HouseholdData(Guid.NewGuid().ToString(),
"blub", chargingStationSet, transportationDeviceSet,
travelRouteSet, tdm, HouseholdDataSpecificationType.ByPersons);
houseData.Households.Add(householdData);
var persons = new List <PersonData> {
new PersonData(30, Gender.Male)
};
householdData.HouseholdDataPersonSpec = new HouseholdDataPersonSpecification(persons);
HouseCreationAndCalculationJob houseJob =
new HouseCreationAndCalculationJob("present", "2019", "trafokreis",
HouseDefinitionType.HouseData);
houseJob.House = houseData;
MakeAndCalculateHouseJob(houseJob, sim, wd, db);
}
}
}
private static void MakeAndCalculateHouseJob([JetBrains.Annotations.NotNull] HouseCreationAndCalculationJob houseJob,
[JetBrains.Annotations.NotNull] Simulator sim, [JetBrains.Annotations.NotNull] WorkingDir wd,
[JetBrains.Annotations.NotNull] DatabaseSetup db)
{
//calcSpec
houseJob.CalcSpec = new JsonCalcSpecification {
DefaultForOutputFiles = OutputFileDefault.Reasonable,
StartDate = new DateTime(2017, 1, 1),
EndDate = new DateTime(2017, 1, 3),
GeographicLocation = sim.GeographicLocations.FindFirstByName("Berlin", FindMode.Partial)
?.GetJsonReference() ??
throw new LPGException("No Berlin in the DB"),
TemperatureProfile = sim.TemperatureProfiles[0].GetJsonReference(),
OutputDirectory = wd.Combine("Results")
};
var dstDir = wd.Combine("profilegenerator.db3");
File.Copy(db.FileName, dstDir, true);
houseJob.PathToDatabase = dstDir;
StartHouseJob(houseJob, wd, "xxx");
}
public static HouseCreationAndCalculationJob PrepareNewHouseForOutputFileTesting(Simulator sim)
{
var hj = new HouseCreationAndCalculationJob();
hj.CalcSpec = JsonCalcSpecification.MakeDefaultsForTesting();
hj.CalcSpec.StartDate = new DateTime(2020, 1, 1);
hj.CalcSpec.EndDate = new DateTime(2020, 1, 3);
hj.CalcSpec.DeleteDAT = false;
hj.CalcSpec.DefaultForOutputFiles = OutputFileDefault.NoFiles;
hj.CalcSpec.DeleteSqlite = false;
hj.CalcSpec.ExternalTimeResolution = "00:15:00";
hj.House = new HouseData(StrGuid.FromString("houseguid"), "HT01", 1000, 100, "housename");
hj.House.Households = new List <HouseholdData>();
var hhd = new HouseholdData("householdid",
"householdname", null, null, null, null,
HouseholdDataSpecificationType.ByHouseholdName);
var hh = sim.ModularHouseholds.FindFirstByName("CHR01", FindMode.StartsWith);
hhd.HouseholdNameSpec = new HouseholdNameSpecification(hh.GetJsonReference());
hj.House.Households.Add(hhd);
return(hj);
}
public void ProcessSingleHouseJob([NotNull] HouseCreationAndCalculationJob hcj, Action <CalculationProfiler, string> makeallthecharts, [NotNull] Simulator sim)
{
if (hcj.House == null)
{
throw new LPGException("No house was defined");
}
if (hcj.CalcSpec == null)
{
throw new LPGException("No calc spec was defined");
}
var geoloc = FindGeographicLocation(sim, hcj.CalcSpec);
var temperatureProfile = FindTemperatureProfile(sim, hcj.CalcSpec);
Random rnd = new Random();
JsonReference objectToCalc;
if (hcj.HouseDefinitionType == HouseDefinitionType.HouseData)
{
objectToCalc = CreateSingleHouse(hcj, sim, geoloc, temperatureProfile, rnd);
if (objectToCalc == null)
{
throw new LPGException("Failed to create a house");
}
}
else
{
objectToCalc = hcj.HouseRef?.House;
if (objectToCalc == null)
{
throw new LPGException("no house reference was set");
}
}
JsonCalculator jc = new JsonCalculator();
//hcj.CalcSpec.CalcObject = null;
jc.StartHousehold(sim, hcj.CalcSpec, objectToCalc, makeallthecharts);
}
private static void StartHouseJob([JetBrains.Annotations.NotNull] HouseCreationAndCalculationJob houseJob, [JetBrains.Annotations.NotNull] WorkingDir wd,
string fnSuffix)
{
string houseJobFile = wd.Combine("houseJob." + fnSuffix + ".json");
if (File.Exists(houseJobFile))
{
File.Delete(houseJobFile);
}
using (StreamWriter sw = new StreamWriter(houseJobFile)) {
sw.WriteLine(JsonConvert.SerializeObject(houseJob, Formatting.Indented));
sw.Close();
}
Logger.Info("======================================================");
Logger.Info("======================================================");
Logger.Info("starting house generation");
Logger.Info("======================================================");
HouseGenerator houseGenerator = new HouseGenerator();
houseGenerator.ProcessSingleHouseJob(houseJobFile, null);
}
public void ProcessSingleHouseJob([NotNull] string houseJobFile, Action <CalculationProfiler, string> makeallthecharts)
{
string resultDir = "Results";
try {
char[] charsToTrim = { '\n', ' ' };
string houseJobStr = File.ReadAllText(houseJobFile).Trim(charsToTrim);
HouseCreationAndCalculationJob hcj = JsonConvert.DeserializeObject <HouseCreationAndCalculationJob>(houseJobStr);
resultDir = hcj.CalcSpec?.OutputDirectory ?? "Results";
if (!Directory.Exists(resultDir))
{
Directory.CreateDirectory(resultDir);
Thread.Sleep(100);
}
string finishedFlagFile = Path.Combine(resultDir, Constants.FinishedFileFlag);
if (File.Exists(finishedFlagFile))
{
Logger.Info("File already exists: " + finishedFlagFile);
string filecontent = File.ReadAllText(finishedFlagFile).Trim(charsToTrim);
if (filecontent == houseJobStr)
{
Logger.Info("This calculation seems to be finished. Quitting.");
return;
}
Logger.Info("There is a previous calculation in the result directory but it used different parameters. Cleaning and recalculating.");
var prevarr = houseJobStr.Split('\n');
var newarr = filecontent.Split('\n');
for (int i = 0; i < newarr.Length && i < prevarr.Length; i++)
{
if (prevarr[i] != newarr[i])
{
Logger.Info("Line: " + i);
Logger.Info("Prev: " + prevarr[i]);
Logger.Info("New : " + newarr[i]);
}
}
}
string srcDbFile = hcj.PathToDatabase ?? throw new LPGException("No db source path");
if (!File.Exists(srcDbFile))
{
throw new LPGException("Could not found source database file: " + srcDbFile);
}
//string houseName = AutomationUtili.CleanFileName(hcj.House?.Name ?? "House");
string dstDbFile = Path.Combine(resultDir, "profilegenerator.copy.db3");
File.Copy(srcDbFile, dstDbFile, true);
//File.SetAttributes(dstDbFile, File.GetAttributes(dstDbFile) & ~FileAttributes.ReadOnly);
string dstConnectionString = "Data Source=" + dstDbFile;
// ReSharper disable once ConditionIsAlwaysTrueOrFalse
Simulator sim = new Simulator(dstConnectionString);
ProcessSingleHouseJob(hcj, makeallthecharts, sim);
if (Logger.Get().Errors.Count == 0)
{
using (var sw = new StreamWriter(finishedFlagFile))
{
sw.Write(houseJobStr);
}
}
else
{
Logger.Info("Didn't mark the calculation as finished, since there were the following errors:");
foreach (var logMessage in Logger.Get().Errors)
{
Logger.Info("Error: " + logMessage.Message);
}
}
}
catch (Exception ex) {
var rdi = new DirectoryInfo(resultDir);
if (!rdi.Exists)
{
rdi.Create();
}
using (StreamWriter sw = new StreamWriter(Path.Combine(resultDir, "CalculationExceptions.txt"))) {
sw.WriteLine(ex.Message);
sw.WriteLine(ex.StackTrace);
sw.WriteLine(ex.ToString());
sw.WriteLine(GetAllFootprints(ex));
sw.Close();
}
throw;
}
}
public static void CreateExampleHouseJob([NotNull] string connectionString)
{
const string relativePathHousejobs = "Example\\HouseJobs";
DirectoryInfo diHouseJobs = new DirectoryInfo(relativePathHousejobs);
if (!diHouseJobs.Exists)
{
diHouseJobs.Create();
}
const string relativePathGuids = "Example\\GuidLists";
DirectoryInfo diGuids = new DirectoryInfo(relativePathGuids);
if (!diGuids.Exists)
{
diGuids.Create();
}
Simulator sim = new Simulator(connectionString);
HouseData houseData1 = new HouseData(Guid.NewGuid().ToStrGuid(), "HT01", 20000, 10000, "MyFirstHouse");
HouseholdData hhd1 = new HouseholdData(Guid.NewGuid().ToString(), "My First Household, template randomly chosen based on persons", null, null,
null, new List <TransportationDistanceModifier>(), HouseholdDataSpecificationType.ByPersons);
HouseholdDataPersonSpecification personSpec = new HouseholdDataPersonSpecification(new List <PersonData>()
{
new PersonData(25, Gender.Male)
});
hhd1.HouseholdDataPersonSpec = personSpec;
houseData1.Households.Add(hhd1);
HouseholdData hhd2 = new HouseholdData(Guid.NewGuid().ToString(),
"My Second Household (with transportation, template defined by name )",
sim.ChargingStationSets[0].GetJsonReference(),
sim.TransportationDeviceSets[0].GetJsonReference(),
sim.TravelRouteSets[0].GetJsonReference(),
null, HouseholdDataSpecificationType.ByTemplateName);
hhd2.HouseholdTemplateSpec = new HouseholdTemplateSpecification("CHR01");
houseData1.Households.Add(hhd2);
HouseData houseData2 = new HouseData(Guid.NewGuid().ToStrGuid(), "HT02", 20000, 10000, "MySecondHouse");
HouseholdData hhd3 = new HouseholdData(Guid.NewGuid().ToString(),
"My Third Household, using predefined household",
null, null, null, null, HouseholdDataSpecificationType.ByHouseholdName);
hhd3.HouseholdNameSpec = new HouseholdNameSpecification(sim.ModularHouseholds[0].GetJsonReference());
houseData2.Households.Add(hhd3);
HouseholdData hhd4 = new HouseholdData(Guid.NewGuid().ToString(), "My Fourth Household", null, null, null, null, HouseholdDataSpecificationType.ByPersons);
hhd4.HouseholdDataPersonSpec = new HouseholdDataPersonSpecification(new List <PersonData>()
{
new PersonData(75, Gender.Male),
new PersonData(74, Gender.Female)
});
houseData2.Households.Add(hhd4);
var calculationSettings = new JsonCalcSpecification {
StartDate = new DateTime(2019, 1, 1),
EndDate = new DateTime(2019, 1, 3),
DeleteDAT = true,
DefaultForOutputFiles = OutputFileDefault.OnlySums
};
if (calculationSettings.CalcOptions == null)
{
throw new LPGException("error");
}
calculationSettings.CalcOptions.Add(CalcOption.HouseSumProfilesFromDetailedDats);
//calculationSettings.CalcOptions.Add(CalcOption.OverallSum.ToString());
calculationSettings.CalcOptions.Add(CalcOption.SumProfileExternalEntireHouse);
calculationSettings.CalcOptions.Add(CalcOption.SumProfileExternalIndividualHouseholds);
calculationSettings.LoadtypesForPostprocessing?.Add("Electricity");
calculationSettings.CalculationName = "My Comment";
calculationSettings.ExternalTimeResolution = "00:15:00";
calculationSettings.InternalTimeResolution = "00:01:00";
calculationSettings.LoadTypePriority = LoadTypePriority.RecommendedForHouses;
calculationSettings.TemperatureProfile = sim.TemperatureProfiles[0].GetJsonReference();
calculationSettings.GeographicLocation = sim.GeographicLocations[0].GetJsonReference();
Logger.Info("--------");
Logger.Info("Writing example file and additional data file that you might need.");
HouseCreationAndCalculationJob hj = new HouseCreationAndCalculationJob("scenario", "year", "districtname", HouseDefinitionType.HouseData);
hj.House = houseData1;
hj.CalcSpec = calculationSettings;
hj.CalcSpec.OutputDirectory = "Example1-Results";
HouseJobSerializer.WriteJsonToFile(Path.Combine(diHouseJobs.FullName, "ExampleHouseJob-1.json"), hj);
hj.House = houseData2;
hj.CalcSpec.OutputDirectory = "Example2-Results";
HouseJobSerializer.WriteJsonToFile(Path.Combine(diHouseJobs.FullName, "ExampleHouseJob-2.json"), hj);
Logger.Info("Finished writing example house jobs...");
WriteGuidList("GuidsForAllHouseholds.csv", sim.ModularHouseholds.It.Select(x => (DBBase)x).ToList(), diGuids);
WriteGuidList("GuidsForAllHouses.csv", sim.Houses.It.Select(x => (DBBase)x).ToList(), diGuids);
WriteGuidList("GuidsForAllChargingStationSets.csv", sim.ChargingStationSets.It.Select(x => (DBBase)x).ToList(), diGuids);
WriteGuidList("GuidsForAllDeviceSelections.csv", sim.DeviceSelections.It.Select(x => (DBBase)x).ToList(), diGuids);
WriteGuidList("GuidsForAllGeographicLocations.csv", sim.GeographicLocations.It.Select(x => (DBBase)x).ToList(), diGuids);
WriteGuidList("GuidsForAllTemperatureProfiles.csv", sim.TemperatureProfiles.It.Select(x => (DBBase)x).ToList(), diGuids);
WriteGuidList("GuidsForAllTransportationDeviceSets.csv", sim.TransportationDeviceSets.It.Select(x => (DBBase)x).ToList(), diGuids);
WriteGuidList("GuidsForAllTravelRouteSets.csv", sim.TravelRouteSets.It.Select(x => (DBBase)x).ToList(), diGuids);
}
public void HouseJobForHouseTypes()
{
//setup
Logger.Get().StartCollectingAllMessages();
using (DatabaseSetup db = new DatabaseSetup(Utili.GetCurrentMethodAndClass())) {
Simulator sim = new Simulator(db.ConnectionString);
using (WorkingDir wd = new WorkingDir(Utili.GetCurrentMethodAndClass())) {
var count = 0;
foreach (var houseType in sim.HouseTypes.It)
{
count++;
if (count < 22)
{
continue;
}
Logger.Info("================================================");
Logger.Info("================================================");
Logger.Info("================================================");
Logger.Info("Starting " + houseType.Name);
Logger.Info("================================================");
Logger.Info("================================================");
Logger.Info("================================================");
Logger.Get().StartCollectingAllMessages();
string htcode = houseType.Name.Substring(0, 4);
//housedata
const int targetheatdemand = 10000;
HouseData houseData = new HouseData(Guid.NewGuid().ToStrGuid(), htcode, targetheatdemand, 1000,
"HouseGeneratorJobHouse");
HouseCreationAndCalculationJob houseJob = new HouseCreationAndCalculationJob("present", "2019",
"trafokreis", HouseDefinitionType.HouseData);
houseJob.House = houseData;
houseJob.CalcSpec = new JsonCalcSpecification {
DefaultForOutputFiles = OutputFileDefault.Reasonable,
StartDate = new DateTime(2017, 1, 1),
EndDate = new DateTime(2017, 12, 31),
GeographicLocation = sim.GeographicLocations.FindFirstByName("Berlin", FindMode.Partial)
?.GetJsonReference() ??
throw new LPGException("No Berlin in the DB"),
TemperatureProfile = sim.TemperatureProfiles[0].GetJsonReference(),
OutputDirectory = wd.Combine("Results." + htcode),
SkipExisting = false,
CalcOptions = new List <CalcOption> {
CalcOption.HouseSumProfilesFromDetailedDats, CalcOption.DeviceProfilesIndividualHouseholds,
CalcOption.EnergyStorageFile, CalcOption.EnergyCarpetPlot,
CalcOption.HouseholdContents, CalcOption.TotalsPerLoadtype
},
DeleteDAT = false
};
StartHouseJob(houseJob, wd, htcode);
SqlResultLoggingService srls = new SqlResultLoggingService(houseJob.CalcSpec.OutputDirectory);
HouseholdKeyLogger hhkslogger = new HouseholdKeyLogger(srls);
var hhks = hhkslogger.Load();
TotalsPerLoadtypeEntryLogger tel = new TotalsPerLoadtypeEntryLogger(srls);
foreach (var entry in hhks)
{
if (entry.KeyType == HouseholdKeyType.General)
{
continue;
}
Logger.Info(entry.HHKey.ToString());
var res = tel.Read(entry.HHKey);
foreach (var totalsEntry in res)
{
Logger.Info(totalsEntry.Loadtype + ": " + totalsEntry.Value);
if (totalsEntry.Loadtype.Name == "Space Heating")
{
if (Math.Abs(totalsEntry.Value - targetheatdemand) > 10)
{
throw new LPGException("Target heat demand didn't match for " + houseType.Name);
}
}
}
}
}
}
}
}
public void RunTest()
{
CompositeResolver.RegisterAndSetAsDefault(NativeDateTimeResolver.Instance, StandardResolver.Instance);
PrepareUnitTest();
Config.Directories.ResultStorageDirectory = WorkingDirectory.Dir;
Config.Directories.CalcServerLpgDirectory = WorkingDirectory.Dir;
// ReSharper disable twice AssignNullToNotNullAttribute
HouseCreationAndCalculationJob hcj = new HouseCreationAndCalculationJob(Scenario.Present().ToString(), "2017", "trafokreis");
HouseData hd = new HouseData("houseguid", "HT01", 1000, 1000, "houseName");
HouseholdData hhd = new HouseholdData("householdguid",
2000,
ElectricCarUse.UseElectricCar,
"householdname",
ElectricCarProvider.ChargingStationSet,
ElectricCarProvider.TransportationDevicesOneCar,
ElectricCarProvider.TravelRouteSet,
new List <TransportationDistanceModifier>(),
HouseholdDataSpecifictionType.ByPersons);
hd.Households.Add(hhd);
hhd.UseElectricCar = ElectricCarUse.UseElectricCar;
hhd.TransportationDeviceSet = ElectricCarProvider.TransportationDevicesOneCar;
hhd.TravelRouteSet = ElectricCarProvider.TravelRouteSet;
hhd.ChargingStationSet = ElectricCarProvider.ChargingStationSet;
hhd.HouseholdDataPersonSpecification = new HouseholdDataPersonSpecification(new List <PersonData> {
new PersonData(30, Gender.Male)
});
hcj.House = hd;
List <HouseCreationAndCalculationJob> houseJobs = new List <HouseCreationAndCalculationJob>();
houseJobs.Add(hcj);
FileHelpers.CopyRec(Config.Directories.LPGReleaseDirectory, WorkingDirectory.Dir, Logger, true);
var endTime = new DateTime(Constants.PresentSlice.DstYear, 1, 10);
ProfileGenerationRo pgro = new ProfileGenerationRo();
HouseProcessor.WriteDistrictsForLPG(houseJobs,
WorkingDirectory.DirDi,
Logger,
Constants.PresentSlice,
endTime,
pgro);
string districtsDir = WorkingDirectory.Combine("Districts");
var districtsDi = new DirectoryInfo(districtsDir);
var files = districtsDi.GetFiles("*.json");
void RunOneFile(FileInfo myfi)
{
ProcessStartInfo psi = new ProcessStartInfo();
psi.FileName = WorkingDirectory.Combine("simulationengine.exe");
psi.UseShellExecute = true;
psi.WorkingDirectory = WorkingDirectory.Dir;
psi.Arguments = "ProcessHouseJob -j \"" + myfi.FullName + "\"";
Info("running " + psi.FileName + " " + psi.Arguments);
using (Process p = new Process()) {
p.StartInfo = psi;
p.Start();
p.WaitForExit();
}
}
foreach (var housejob in files)
{
RunOneFile(housejob);
}
DBDto dbDto = new DBDto(new List <House>(), new List <Hausanschluss>(), new List <Car>(), new List <Household>(), new List <RlmProfile>());
CachingLPGProfileLoader ca = new CachingLPGProfileLoader(Logger, dbDto);
List <int> isns = new List <int>();
isns.Add(10);
CarDistanceEntry cde = new CarDistanceEntry("houseguid",
"householdguid",
"carguid",
20,
20,
isns,
10,
"haguid",
"sourceguid",
"cdename",
CarType.Electric);
HouseComponentRo hcro = new HouseComponentRo("housecomponent", "componeenttype", 1000, 200, "processingstatus", "isns", "standort", 0);
ProviderParameterDto ppd = new ProviderParameterDto(cde, WorkingDirectory.Dir, hcro);
SqlConnectionPreparer scp = new SqlConnectionPreparer(Config);
MyDb db = scp.GetDatabaseConnection(Stage.Testing, Constants.PresentSlice);
SaveableEntry <Profile> sa = SaveableEntry <Profile> .GetSaveableEntry(db, SaveableEntryTableType.LPGProfile, Logger);
sa.MakeTableForListOfFieldsIfNotExists(true);
string dstDir = Path.Combine(WorkingDirectory.Dir, hcj.Trafokreis, hcj.House.Name);
FileHelpers.CopyRec(WorkingDirectory.Combine("Results"), dstDir, Logger, true);
//normal electricity test and cache test
Info("================== ");
Info("electricity");
Info("================== ");
var profElec1 = ca.LoadLPGProfile(ppd,
hcj.Trafokreis,
"Electricity",
sa,
hhd.HouseholdGuid,
out var profsource,
hcj.House.Name,
Config,
true);
Info("Source: " + profsource);
Assert.NotNull(profElec1);
Assert.NotNull(profsource);
var profElecCache = ca.LoadLPGProfile(ppd,
hcj.Trafokreis,
"Electricity",
sa,
hhd.HouseholdGuid,
out var profsourceCache,
hcj.House.Name,
Config,
true);
Info("Source 2: " + profsourceCache);
Assert.NotNull(profsourceCache);
Assert.NotNull(profsource);
profElec1.Should().BeEquivalentTo(profElecCache, options => options.Excluding(ctx => ctx.SelectedMemberPath.EndsWith("BinaryProfile")));
//Car Charging Electricity electricity test and cache test
Info("================== ");
Info("Car Charging Electricity electricity");
Info("================== ");
SaveableEntry <Profile> sa2 = SaveableEntry <Profile> .GetSaveableEntry(db, SaveableEntryTableType.EvProfile, Logger);
sa2.MakeCleanTableForListOfFields(true);
var prof2 = ca.LoadLPGProfile(ppd,
hcj.Trafokreis,
"Car Charging Electricity",
sa2,
hhd.HouseholdGuid,
out var profsource2,
hcj.House.Name,
Config,
true);
Info("Source Wp 1: " + profsource2);
Assert.NotNull(prof2);
Assert.NotNull(profsource2);
var prof3 = ca.LoadLPGProfile(ppd,
hcj.Trafokreis,
"Car Charging Electricity",
sa2,
hhd.HouseholdGuid,
out var profsource3,
hcj.House.Name,
Config,
true);
Info("Source Wp 2: " + profsource3);
Assert.NotNull(prof3);
Assert.NotNull(profsource3);
prof2.Should().BeEquivalentTo(prof3, options => options.Excluding(ctx =>
ctx.SelectedMemberPath.EndsWith("BinaryProfile")));
}
public StatisticsEntry([NotNull] string fileName, [NotNull] HouseCreationAndCalculationJob jcs, [NotNull] string sourceDirectory)
{
this.FileName = fileName;
Jcs = jcs;
SourceDirectory = sourceDirectory;
}
private static void MakeSingleHouse([JetBrains.Annotations.NotNull] JsonCalcSpecification cspec, [JetBrains.Annotations.NotNull] WorkingDir wd, [JetBrains.Annotations.NotNull] HouseCreationAndCalculationJob hcj,
[JetBrains.Annotations.NotNull] House house1, string housename)
{
cspec.OutputDirectory = housename;
if (cspec.OutputDirectory == null)
{
throw new LPGException("Output directory was null");
}
var dirtodelete = wd.Combine(cspec.OutputDirectory);
if (Directory.Exists(dirtodelete))
{
Directory.Delete(dirtodelete, true);
}
hcj.House = house1.MakeHouseData();
HouseJobSerializer.WriteJsonToFile(wd.Combine(housename + ".json"), hcj);
}
private void SingleThreadExecutor(int index)
{
try {
Logger.Info("Starting thread " + index);
var di = new DirectoryInfo(Directory.GetCurrentDirectory());
DriveInfo driveInfo = null;
if (di.FullName.Contains(":"))
{
var drive = di.FullName.Substring(0, 1);
driveInfo = new DriveInfo(drive);
}
while (!_calculationsToProcess.IsEmpty && _continueProcessing)
{
try {
if (Console.KeyAvailable)
{
if (Console.ReadKey(true).Key == ConsoleKey.Q)
{
_continueProcessing = false;
while (!_calculationsToProcess.IsEmpty)
{
_calculationsToProcess.TryDequeue(out var _);
}
Logger.Info("Thread: " + index + "Q was pressed. Quitting soon.");
}
if (Console.ReadKey(true).Key == ConsoleKey.T)
{
foreach (Thread thread in _threads)
{
Logger.Info(thread.Name + ": " + thread.IsAlive);
}
}
}
}
catch (Exception ex) {
Logger.Info(ex.Message);
//Logger.Exception(ex);
}
try {
var success = _calculationsToProcess.TryDequeue(out var path);
if (success && _continueProcessing)
{
CheckForSufficentFreeSpace(index, driveInfo);
if (!_continueProcessing)
{
return;
}
DateTime startTime = DateTime.Now;
var processesleft = _calculationsToProcess.Count;
var processesdone = _totalCalculations - processesleft;
var progress = " (" + processesdone + "/" + _totalCalculations + ")";
Logger.Info("Thread " + index + ": Processing " + path.JsonFile.Name + progress);
var startinfo = new ProcessStartInfo();
string arguments = "processhousejob -j \"" + path.JsonFile.FullName + "\"";
Logger.Info("Launching simulationengine.exe " + arguments);
using (var process = new Process()) {
startinfo.Arguments = arguments;
startinfo.UseShellExecute = true;
startinfo.WindowStyle = ProcessWindowStyle.Normal;
startinfo.FileName = "SimulationEngine.exe";
process.StartInfo = startinfo;
process.Start();
SetWindowText(process.MainWindowHandle, path.JsonFile.FullName);
process.WaitForExit();
}
Logger.Info("Thread " + index + ": Finished \"" + path.JsonFile.Name + "\"");
CompletedCalc completedCalc = new CompletedCalc();
_completedCalcs.Add(completedCalc);
completedCalc.FileName = path.JsonFile.Name;
Thread.Sleep(1000);
string jsonStr = File.ReadAllText(path.JsonFile.FullName);
HouseCreationAndCalculationJob jcs = JsonConvert.DeserializeObject <HouseCreationAndCalculationJob>(jsonStr);
completedCalc.Status = "No finished.flag found";
if (jcs.CalcSpec.OutputDirectory != null)
{
DirectoryInfo outputFolder = new DirectoryInfo(jcs.CalcSpec.OutputDirectory);
Logger.Info("Checking output folder " + outputFolder.FullName);
if (outputFolder.Exists)
{
string finishedFile = Path.Combine(outputFolder.FullName, "finished.flag");
if (File.Exists(finishedFile))
{
completedCalc.Status = "finished.flag found";
completedCalc.IsSuccess = true;
_foldersToArchive.Enqueue(outputFolder.FullName);
Logger.Info("Thread " + index + ": Enqueued for moving: " + outputFolder.FullName + progress);
}
else
{
completedCalc.Status = "No folder and no finished.flag found";
Logger.Info("Thread " + index + ": Not enqueued for moving due to missing finished flag: " + outputFolder.FullName + progress);
}
if (jcs.CalcSpec.DeleteDAT)
{
var dats = outputFolder.GetFiles("*.dat", SearchOption.AllDirectories);
foreach (var info in dats)
{
info.Delete();
}
}
}
else
{
completedCalc.Status = "No folder found";
}
}
else
{
Logger.Info("No output folder found for " + path.JsonFile.Name);
}
completedCalc.Duration = (DateTime.Now - startTime);
}
}
catch (Exception ex) {
Logger.Exception(ex);
}
}
}
catch (Exception ex) {
Logger.Info("Thread " + index + ": Exception:" + Environment.NewLine + ex.Message);
Logger.Exception(ex);
}
}
private void FillCalculationQueue([NotNull] ParallelJsonLauncherOptions options)
{
Logger.Info("Starting to look for all the json files to calculate...");
if (options.JsonDirectory == null)
{
throw new LPGCommandlineException("Json directory was not set.");
}
DirectoryInfo di = new DirectoryInfo(options.JsonDirectory);
if (!di.Exists)
{
throw new LPGCommandlineException("Directory " + di.FullName + " was not found.");
}
var inputFiles = di.GetFiles("*.json").ToList();
if (options.SearchSecondLevel)
{
foreach (DirectoryInfo subdir in di.GetDirectories())
{
inputFiles.AddRange(subdir.GetFiles("*.json"));
}
}
if (inputFiles.Count == 0)
{
throw new LPGCommandlineException("No input files were found.");
}
if (options.MaximumNumberOfCalculations > 0 && options.MaximumNumberOfCalculations < inputFiles.Count)
{
inputFiles = inputFiles.Take(options.MaximumNumberOfCalculations).ToList();
}
for (var calcidx = 0; calcidx < inputFiles.Count; calcidx++)
{
HouseCreationAndCalculationJob jcs = LoadFromFile(inputFiles[calcidx].FullName);
if (jcs == null)
{
continue;
}
//check if there even is an output directory
if (string.IsNullOrWhiteSpace(jcs.CalcSpec?.OutputDirectory))
{
Logger.Error("Skipping file: No output directory set in the file " + inputFiles[calcidx].Name);
if (ThrowOnInvalidFile)
{
throw new LPGException("No output file was set: " + jcs.CalcSpec?.OutputDirectory);
}
continue;
}
// ReSharper disable once AssignNullToNotNullAttribute
DirectoryInfo outputDir = new DirectoryInfo(jcs.CalcSpec.OutputDirectory);
//check the archive directory if the results already exist
if (!string.IsNullOrWhiteSpace(options.ArchiveDirectory))
{
//make archive path
string archivePath = GetArchiveDirectory(outputDir, options);
string finishedFlagPath = Path.Combine(archivePath, "finished.flag");
if (File.Exists(finishedFlagPath))
{
Logger.Info("Found finished folder in archive: " + archivePath);
continue;
}
}
//check current directory
if (outputDir.Exists)
{
var calculatedFiles = outputDir.GetFiles("finished.flag");
if (calculatedFiles.Length == 1)
{
Logger.Warning(outputDir.Name + " seems finished. Enqueueing for archiving");
_foldersToArchive.Enqueue(outputDir.FullName);
}
else
{
Logger.Warning(outputDir.Name + " had files left over. Deleting and trying again.");
outputDir.Delete(true);
}
}
Logger.Info("Enqueuing file " + inputFiles[calcidx].Name);
_calculationsToProcess.Enqueue(new CalcJobQueueEntry(inputFiles[calcidx], calcidx));
_totalCalculations++;
}
}
private static JsonReference CreateSingleHouse(
[NotNull] HouseCreationAndCalculationJob hj,
[NotNull] Simulator sim,
GeographicLocation geoloc,
TemperatureProfile temperatureProfile,
Random r)
{
if (hj.House == null)
{
throw new LPGException("No house data was set in the file");
}
//string name = hj.House?.Name ?? "";
var housedata = hj.House;
Logger.Info("Creating new house with " + hj.House?.Households.Count + " households...");
//List<ModularHousehold> createdHouseholds = new List<ModularHousehold>();
//make the house
var house = MakeHouse(sim, hj.House);
house.GeographicLocation = geoloc;
house.TemperatureProfile = temperatureProfile;
//add the Households
int householdidx = 1;
foreach (var householdData in housedata.Households)
{
var hhs = MakeHousehold(sim, householdData, r);
var chargingStationSet = sim.ChargingStationSets.FindByJsonReference(householdData.ChargingStationSet);
var travelrouteset = sim.TravelRouteSets.FindByJsonReference(householdData.TravelRouteSet);
var transportationDeviceSet = sim.TransportationDeviceSets.FindByJsonReference(householdData.TransportationDeviceSet);
if (householdData.TransportationDistanceModifiers != null && travelrouteset != null && householdData.TransportationDistanceModifiers.Count > 0
)
{
Logger.Info("Settings new travel distances for " + hhs.Name + " " + "");
travelrouteset = AdjustTravelDistancesBasedOnModifiers(travelrouteset, sim, house,
householdData, householdidx++);
Logger.Info("Name of the new travel route set to be used is " + travelrouteset.Name);
}
house.AddHousehold(hhs, chargingStationSet, travelrouteset, transportationDeviceSet);
//createdHouseholds.Add(hhs);
}
/*
* if (createdHouseholds.Count == 0) {
* sim.Houses.DeleteItem(house);
* continue;
* }*/
house.SaveToDB();
Logger.Info("Successfully created house.");
//saving matching calculation file
Logger.Info("Creating calculation file.");
if (house == null)
{
throw new LPGException("House generation failed");
}
return(house.GetJsonReference());
}
public bool PrepareLoadProfileIfNeeded([NotNull] ProviderParameterDto parameters)
{
if (parameters.HouseComponent.HouseComponentType != HouseComponentType.Household)
{
throw new FlaException("Was not household: " + parameters.HouseComponent.HouseComponentType);
}
Household hh = (Household)parameters.HouseComponent;
Hausanschluss ha = _dbDto.Hausanschlusse.Single(x => x.Guid == hh.HausAnschlussGuid);
var houseJob = _housesToBeCreated.FirstOrDefault(x => x.House.HouseGuid == hh.HouseGuid);
if (houseJob == null)
{
var flahouse = _dbDto.Houses.Single(x => x.Guid == hh.HouseGuid);
houseJob = new HouseCreationAndCalculationJob(Slice.DstScenario.ToString(), Slice.DstYear.ToString(), ha.Trafokreis);
houseJob.House = new HouseData(hh.HouseGuid, "HT01", 0, 0, flahouse.ComplexName);
_housesToBeCreated.Add(houseJob);
}
HouseholdData hd = new HouseholdData(hh.HouseholdKey,
hh.EffectiveEnergyDemand,
ElectricCarUse.NoElectricCar,
hh.Name,
null,
null,
null,
null,
HouseholdDataSpecifictionType.ByPersons);
hd.IsCarProfileCalculated = true;
hd.HouseholdDataPersonSpecification = new HouseholdDataPersonSpecification(new List <PersonData>());
if (hh.Occupants.Count == 0)
{
throw new FlaException("No occupants in the household " + hh.Name);
}
foreach (var occupant in hh.Occupants)
{
hd.HouseholdDataPersonSpecification.Persons.Add(new PersonData(occupant.Age, (Gender)occupant.Gender));
}
House house = _dbDto.Houses.First(x => x.Guid == hh.HouseGuid);
if (Services.RunningConfig.LpgPrepareMode == LpgPrepareMode.PrepareWithFullLpgLoad)
{
Profile lpgProfile = null;
try {
lpgProfile = _lpgloader.LoadLPGProfile(parameters,
ha.Trafokreis,
LoadtypetoSearchFor,
_saveableEntry,
hh.HouseholdKey,
out _,
house.ComplexName,
Services.RunningConfig);
}
#pragma warning disable CA1031 // Do not catch general exception types
catch (Exception ex) {
#pragma warning restore CA1031 // Do not catch general exception types
parameters.HouseComponentResultObject.LPGErrors = "trying to load lpg profile failed: " + ex.Message;
Error("trying to load lpg profile: " + ex.Message);
}
if (lpgProfile != null)
{
hd.IsHouseholdProfileCalculated = true;
}
else
{
hd.IsHouseholdProfileCalculated = false;
}
}
else if (Services.RunningConfig.LpgPrepareMode == LpgPrepareMode.PrepareWithOnlyNamecheck)
{
if (_saveableEntry.CheckForName(hh.HouseholdKey, Services.Logger))
{
hd.IsHouseholdProfileCalculated = true;
}
else
{
hd.IsHouseholdProfileCalculated = false;
}
}
else
{
throw new FlaException("Unknown lpg prepare mode");
}
houseJob.House.Households.Add(hd);
return(true);
}
