public void Run()
{
Random rnd = new Random();
// ReSharper disable twice AssignNullToNotNullAttribute
ServiceRepository services = new ServiceRepository(null, null, Logger, Config, rnd);
ProsumerComponentResultArchiver pra = new ProsumerComponentResultArchiver(Stage.Testing, Constants.PresentSlice, HouseProcessor.ProcessingMode.Collecting,
services);
Prosumer prosumer = new Prosumer("houseguid", "housename", HouseComponentType.Household, "sourceguid", 1,
"hausanschlussguid", "haussanschlusskey", GenerationOrLoad.Load, "trafokreis", "providername", "profileSource");
Stopwatch sw = Stopwatch.StartNew();
var createdProsumers = new List <Prosumer>();
for (int i = 0; i < 200; i++)
{
Profile prof = Profile.MakeRandomProfile(rnd, "name", Profile.ProfileResolution.QuarterHour);
prosumer.Profile = prof;
pra.Archive(prosumer);
createdProsumers.Add(prosumer);
}
Info("Creating and queueing: " + sw.Elapsed.ToString());
pra.FinishSavingEverything();
pra.Dispose();
sw.Stop();
Info("total: " + sw.Elapsed);
var loadSa = SaveableEntry <Prosumer> .GetSaveableEntry(services.SqlConnectionPreparer.GetDatabaseConnection(Stage.Testing, Constants.PresentSlice, DatabaseCode.HouseProfiles),
SaveableEntryTableType.HouseLoad, Logger);
var loadedProsumers = loadSa.LoadAllOrMatching();
Info("checking profile counts");
foreach (var loadedProsumer in loadedProsumers)
{
loadedProsumer.Profile?.Values.Count.Should().Be(8760 * 4);
}
Info("profile counts were ok, checking prosumers");
sw = Stopwatch.StartNew();
for (int i = 0; i < createdProsumers.Count && i < 1; i++)
{
loadedProsumers[i].Should().BeEquivalentTo(createdProsumers[i]);
}
sw.Stop();
Info("checking prosumers took " + sw.Elapsed.ToString());
}
public void ProcessAllHouses([NotNull] ScenarioSliceParameters parameters,
[NotNull] Func <string, ScenarioSliceParameters, bool, string> makeAndRegisterFullFilename,
ProcessingMode processingMode,
[NotNull][ItemNotNull] List <string> developmentStatus)
{
if (!Directory.Exists(_processingResultPathForProfiles))
{
Directory.CreateDirectory(_processingResultPathForProfiles);
Thread.Sleep(500);
}
else
{
var dstDi = new DirectoryInfo(_processingResultPathForProfiles);
var files = dstDi.GetFiles();
Info("Cleaning " + files.Length + " files from result directory " + _processingResultPathForProfiles);
foreach (var file in files)
{
file.Delete();
}
}
var dbHouses = _services.SqlConnectionPreparer.GetDatabaseConnection(Stage.Houses, parameters);
Info("using house db in " + dbHouses.ConnectionString);
var houses = dbHouses.Fetch <House>();
HouseComponentRepository hcr = new HouseComponentRepository(dbHouses);
var households = dbHouses.Fetch <Household>();
var hausanschlusses = dbHouses.Fetch <Hausanschluss>();
var cars = dbHouses.Fetch <Car>();
var dbRaw = _services.SqlConnectionPreparer.GetDatabaseConnection(Stage.Raw, Constants.PresentSlice);
var measuredRlmProfiles = dbRaw.Fetch <RlmProfile>();
DBDto dbdto = new DBDto(houses, hausanschlusses, cars, households, measuredRlmProfiles);
ProsumerComponentResultArchiver pra = new ProsumerComponentResultArchiver(Stage.ProfileGeneration, parameters, processingMode, _services);
var trafokreiseToProcess = PrepareListOfTrafokreiseToProcess(hausanschlusses);
List <HouseCreationAndCalculationJob> districts = new List <HouseCreationAndCalculationJob>();
var vdewValues = dbRaw.Fetch <VDEWProfileValue>();
var feiertage = dbRaw.Fetch <FeiertagImport>();
var slpProvider = new SLPProvider(parameters.DstYear, vdewValues, feiertage);
var loadProfileProviders = MakeDiContrainer(parameters, developmentStatus, hausanschlusses, houses, districts, slpProvider, dbdto);
if (processingMode == ProcessingMode.Collecting)
{
ClearAllExistingExportProfiles();
}
var lpgDirectoryInfo = GetLPGCalcDirectoryInfo(parameters);
ProfileGenerationRo pgRo = new ProfileGenerationRo();
DateTime startTime = DateTime.Now;
DateTime lastLog = DateTime.Now;
Info("Processing " + houses.Count + " houses.");
Info("Processing mode is " + processingMode);
List <string> brokenLpgDirectories = new List <string>();
List <string> brokenLpgJsons = new List <string>();
Dictionary <string, List <HouseComponentEntry> > houseComponentsByObjectID = BuildDictionaryByObjektID(houses,
hcr,
hausanschlusses,
pgRo,
out var numberOfcomponents);
int processedComponents = 0;
Stopwatch swCollecting = new Stopwatch();
Stopwatch swWriting = new Stopwatch();
Dictionary <string, int> numberOfEmptyProsumers = new Dictionary <string, int>();
HashSet <string> validHouseGuids = houses.Select(x => x.Guid).ToHashSet();
foreach (KeyValuePair <string, List <HouseComponentEntry> > pair in houseComponentsByObjectID)
{
var houseProsumers = new List <Prosumer>();
//erst alle profile einsammeln / vorbereiten
swCollecting.Start();
foreach (var component in pair.Value)
{
processedComponents++;
if (processedComponents % 1000 == 0)
{
Info(processingMode + " processed Components " + processedComponents + " / " + numberOfcomponents);
}
ProviderParameterDto ppdto = new ProviderParameterDto(component.Component, lpgDirectoryInfo.FullName, pgRo[component.Component]);
var provider = loadProfileProviders.GetCorrectProvider(component.Component);
pgRo[component.Component].UsedProvider = provider.Name;
if (processingMode == ProcessingMode.Preparing)
{
provider.PrepareLoadProfileIfNeeded(ppdto);
}
if (processingMode == ProcessingMode.Collecting)
{
if (trafokreiseToProcess.Contains(component.Hausanschluss.Trafokreis))
{
Prosumer p = provider.ProvideProfile(ppdto);
//Todo: add up profile per trafokreis, profile per provider etc right here
if (p != null)
{
// some providers that are not ready will return null
if (p.Profile?.EnergyOrPower != EnergyOrPower.Energy)
{
throw new FlaException("Got a power profile from " + provider.Name);
}
CheckProfileIntegrity(p, provider, component.Component, validHouseGuids);
// ReSharper disable once AssignNullToNotNullAttribute
pgRo[component.Component].AddProsumerInformation(p);
houseProsumers.Add(p);
pra.Archive(p);
}
else
{
if (!numberOfEmptyProsumers.ContainsKey(provider.Name))
{
numberOfEmptyProsumers.Add(provider.Name, 0);
}
numberOfEmptyProsumers[provider.Name]++;
}
}
}
}
swCollecting.Stop();
//dann alles rausschreiben
swWriting.Start();
if (processingMode == ProcessingMode.Collecting)
{
var generationProsumers = houseProsumers.Where(x => x.GenerationOrLoad == GenerationOrLoad.Generation).ToList();
var loadProsumers = houseProsumers.Where(x => x.GenerationOrLoad == GenerationOrLoad.Load).ToList();
var component = pair.Value[0];
if (loadProsumers.Count > 0)
{
Prosumer summedProsumer = GetSummedProsumer(loadProsumers, component.Hausanschluss.Trafokreis);
var haros = pgRo.HausanschlussByObjectId(pair.Key);
double maxPower = summedProsumer.Profile?.MaxPower() ?? 0;
foreach (var haro in haros)
{
haro.MaximumPower = maxPower;
}
WriteSumLineToCsv(summedProsumer, component.Hausanschluss.Trafokreis, GenerationOrLoad.Load);
}
if (generationProsumers.Count > 0)
{
var sumProfile = GetSummedProsumer(generationProsumers, component.Hausanschluss.Trafokreis);
WriteSumLineToCsv(sumProfile, component.Hausanschluss.Trafokreis, GenerationOrLoad.Generation);
// ReSharper disable once PossibleNullReferenceException
}
}
swWriting.Stop();
ReportProgress(startTime, processedComponents, numberOfcomponents, parameters, ref lastLog, swCollecting, swWriting);
}
Info("Finished processing all components, finishing up now. Duration: " + (DateTime.Now - startTime).TotalMinutes.ToString("F2") +
" minutes");
Info("collecting took " + swCollecting.Elapsed + " and writing took " + swWriting.Elapsed);
foreach (var pair in numberOfEmptyProsumers)
{
Info("NumberOfEmptyProsumers for " + pair.Key + " was " + pair.Value);
}
if (processingMode == ProcessingMode.Preparing)
{
DateTime endtime = new DateTime(parameters.DstYear, 12, 31);
string houseJobDirectory = Path.Combine(_services.RunningConfig.Directories.HouseJobsDirectory,
parameters.DstScenario.ToString(),
parameters.DstYear.ToString());
DirectoryInfo houseJobDi = new DirectoryInfo(houseJobDirectory);
WriteDistrictsForLPG(districts, houseJobDi, _services.Logger, parameters, endtime, pgRo);
}
else
{
DateTime startSavingToDB = DateTime.Now;
pra.FinishSavingEverything();
Info("Finished writing prosumers to db. Duration: " + (DateTime.Now - startSavingToDB).TotalMinutes.ToString("F2") + " minutes");
}
pra.Dispose();
var excelFiles = WriteExcelResultFiles(parameters, makeAndRegisterFullFilename, processingMode, pgRo);
if (_services.RunningConfig.CollectFilesForArchive)
{
SaveToArchiveDirectory(parameters, excelFiles, _services.StartingTime, _services.RunningConfig);
}
WriteBrokenLpgCalcCleanupBatch(parameters, makeAndRegisterFullFilename, processingMode, brokenLpgDirectories, brokenLpgJsons);
if (processingMode == ProcessingMode.Collecting)
{
foreach (var provider in loadProfileProviders.Providers)
{
provider.DoFinishCheck();
}
}
}