public void Run()
        {
            Random rnd = new Random();
            // ReSharper disable twice AssignNullToNotNullAttribute
            ServiceRepository services          = new ServiceRepository(null, null, Logger, Config, rnd);
            ProsumerComponentResultArchiver pra = new ProsumerComponentResultArchiver(Stage.Testing, Constants.PresentSlice, HouseProcessor.ProcessingMode.Collecting,
                                                                                      services);
            Prosumer prosumer = new Prosumer("houseguid", "housename", HouseComponentType.Household, "sourceguid", 1,
                                             "hausanschlussguid", "haussanschlusskey", GenerationOrLoad.Load, "trafokreis", "providername", "profileSource");
            Stopwatch sw = Stopwatch.StartNew();
            var       createdProsumers = new List <Prosumer>();

            for (int i = 0; i < 200; i++)
            {
                Profile prof = Profile.MakeRandomProfile(rnd, "name", Profile.ProfileResolution.QuarterHour);
                prosumer.Profile = prof;
                pra.Archive(prosumer);
                createdProsumers.Add(prosumer);
            }
            Info("Creating and queueing: " + sw.Elapsed.ToString());
            pra.FinishSavingEverything();
            pra.Dispose();
            sw.Stop();
            Info("total: " + sw.Elapsed);
            var loadSa = SaveableEntry <Prosumer> .GetSaveableEntry(services.SqlConnectionPreparer.GetDatabaseConnection(Stage.Testing, Constants.PresentSlice, DatabaseCode.HouseProfiles),
                                                                    SaveableEntryTableType.HouseLoad, Logger);

            var loadedProsumers = loadSa.LoadAllOrMatching();

            Info("checking profile counts");
            foreach (var loadedProsumer in loadedProsumers)
            {
                loadedProsumer.Profile?.Values.Count.Should().Be(8760 * 4);
            }
            Info("profile counts were ok, checking prosumers");
            sw = Stopwatch.StartNew();
            for (int i = 0; i < createdProsumers.Count && i < 1; i++)
            {
                loadedProsumers[i].Should().BeEquivalentTo(createdProsumers[i]);
            }
            sw.Stop();
            Info("checking prosumers took " + sw.Elapsed.ToString());
        }
        public void ProcessAllHouses([NotNull] ScenarioSliceParameters parameters,
                                     [NotNull] Func <string, ScenarioSliceParameters, bool, string> makeAndRegisterFullFilename,
                                     ProcessingMode processingMode,
                                     [NotNull][ItemNotNull] List <string> developmentStatus)
        {
            if (!Directory.Exists(_processingResultPathForProfiles))
            {
                Directory.CreateDirectory(_processingResultPathForProfiles);
                Thread.Sleep(500);
            }
            else
            {
                var dstDi = new DirectoryInfo(_processingResultPathForProfiles);
                var files = dstDi.GetFiles();
                Info("Cleaning " + files.Length + " files from result directory " + _processingResultPathForProfiles);
                foreach (var file in files)
                {
                    file.Delete();
                }
            }

            var dbHouses = _services.SqlConnectionPreparer.GetDatabaseConnection(Stage.Houses, parameters);

            Info("using house db in  " + dbHouses.ConnectionString);
            var houses = dbHouses.Fetch <House>();
            HouseComponentRepository hcr = new HouseComponentRepository(dbHouses);
            var   households             = dbHouses.Fetch <Household>();
            var   hausanschlusses        = dbHouses.Fetch <Hausanschluss>();
            var   cars  = dbHouses.Fetch <Car>();
            var   dbRaw = _services.SqlConnectionPreparer.GetDatabaseConnection(Stage.Raw, Constants.PresentSlice);
            var   measuredRlmProfiles = dbRaw.Fetch <RlmProfile>();
            DBDto dbdto = new DBDto(houses, hausanschlusses, cars, households, measuredRlmProfiles);
            ProsumerComponentResultArchiver pra = new ProsumerComponentResultArchiver(Stage.ProfileGeneration, parameters, processingMode, _services);
            var trafokreiseToProcess            = PrepareListOfTrafokreiseToProcess(hausanschlusses);

            List <HouseCreationAndCalculationJob> districts = new List <HouseCreationAndCalculationJob>();
            var vdewValues           = dbRaw.Fetch <VDEWProfileValue>();
            var feiertage            = dbRaw.Fetch <FeiertagImport>();
            var slpProvider          = new SLPProvider(parameters.DstYear, vdewValues, feiertage);
            var loadProfileProviders = MakeDiContrainer(parameters, developmentStatus, hausanschlusses, houses, districts, slpProvider, dbdto);

            if (processingMode == ProcessingMode.Collecting)
            {
                ClearAllExistingExportProfiles();
            }

            var lpgDirectoryInfo          = GetLPGCalcDirectoryInfo(parameters);
            ProfileGenerationRo pgRo      = new ProfileGenerationRo();
            DateTime            startTime = DateTime.Now;
            DateTime            lastLog   = DateTime.Now;

            Info("Processing " + houses.Count + " houses.");
            Info("Processing mode is " + processingMode);
            List <string> brokenLpgDirectories = new List <string>();
            List <string> brokenLpgJsons       = new List <string>();
            Dictionary <string, List <HouseComponentEntry> > houseComponentsByObjectID = BuildDictionaryByObjektID(houses,
                                                                                                                   hcr,
                                                                                                                   hausanschlusses,
                                                                                                                   pgRo,
                                                                                                                   out var numberOfcomponents);
            int       processedComponents = 0;
            Stopwatch swCollecting        = new Stopwatch();
            Stopwatch swWriting           = new Stopwatch();
            Dictionary <string, int> numberOfEmptyProsumers = new Dictionary <string, int>();
            HashSet <string>         validHouseGuids        = houses.Select(x => x.Guid).ToHashSet();

            foreach (KeyValuePair <string, List <HouseComponentEntry> > pair in houseComponentsByObjectID)
            {
                var houseProsumers = new List <Prosumer>();
                //erst alle profile einsammeln / vorbereiten
                swCollecting.Start();
                foreach (var component in pair.Value)
                {
                    processedComponents++;
                    if (processedComponents % 1000 == 0)
                    {
                        Info(processingMode + " processed Components " + processedComponents + " / " + numberOfcomponents);
                    }

                    ProviderParameterDto ppdto = new ProviderParameterDto(component.Component, lpgDirectoryInfo.FullName, pgRo[component.Component]);
                    var provider = loadProfileProviders.GetCorrectProvider(component.Component);
                    pgRo[component.Component].UsedProvider = provider.Name;
                    if (processingMode == ProcessingMode.Preparing)
                    {
                        provider.PrepareLoadProfileIfNeeded(ppdto);
                    }

                    if (processingMode == ProcessingMode.Collecting)
                    {
                        if (trafokreiseToProcess.Contains(component.Hausanschluss.Trafokreis))
                        {
                            Prosumer p = provider.ProvideProfile(ppdto);
                            //Todo: add up profile per trafokreis, profile per provider etc right here
                            if (p != null)
                            {
                                // some providers that are not ready will return null
                                if (p.Profile?.EnergyOrPower != EnergyOrPower.Energy)
                                {
                                    throw new FlaException("Got a power profile from " + provider.Name);
                                }

                                CheckProfileIntegrity(p, provider, component.Component, validHouseGuids);
                                // ReSharper disable once AssignNullToNotNullAttribute
                                pgRo[component.Component].AddProsumerInformation(p);
                                houseProsumers.Add(p);
                                pra.Archive(p);
                            }
                            else
                            {
                                if (!numberOfEmptyProsumers.ContainsKey(provider.Name))
                                {
                                    numberOfEmptyProsumers.Add(provider.Name, 0);
                                }

                                numberOfEmptyProsumers[provider.Name]++;
                            }
                        }
                    }
                }

                swCollecting.Stop();
                //dann alles rausschreiben
                swWriting.Start();
                if (processingMode == ProcessingMode.Collecting)
                {
                    var generationProsumers = houseProsumers.Where(x => x.GenerationOrLoad == GenerationOrLoad.Generation).ToList();
                    var loadProsumers       = houseProsumers.Where(x => x.GenerationOrLoad == GenerationOrLoad.Load).ToList();
                    var component           = pair.Value[0];
                    if (loadProsumers.Count > 0)
                    {
                        Prosumer summedProsumer = GetSummedProsumer(loadProsumers, component.Hausanschluss.Trafokreis);
                        var      haros          = pgRo.HausanschlussByObjectId(pair.Key);
                        double   maxPower       = summedProsumer.Profile?.MaxPower() ?? 0;
                        foreach (var haro in haros)
                        {
                            haro.MaximumPower = maxPower;
                        }

                        WriteSumLineToCsv(summedProsumer, component.Hausanschluss.Trafokreis, GenerationOrLoad.Load);
                    }

                    if (generationProsumers.Count > 0)
                    {
                        var sumProfile = GetSummedProsumer(generationProsumers, component.Hausanschluss.Trafokreis);
                        WriteSumLineToCsv(sumProfile, component.Hausanschluss.Trafokreis, GenerationOrLoad.Generation);
                        // ReSharper disable once PossibleNullReferenceException
                    }
                }

                swWriting.Stop();
                ReportProgress(startTime, processedComponents, numberOfcomponents, parameters, ref lastLog, swCollecting, swWriting);
            }

            Info("Finished processing all components, finishing up now. Duration: " + (DateTime.Now - startTime).TotalMinutes.ToString("F2") +
                 " minutes");
            Info("collecting took " + swCollecting.Elapsed + " and writing took " + swWriting.Elapsed);
            foreach (var pair in numberOfEmptyProsumers)
            {
                Info("NumberOfEmptyProsumers for " + pair.Key + " was " + pair.Value);
            }

            if (processingMode == ProcessingMode.Preparing)
            {
                DateTime endtime           = new DateTime(parameters.DstYear, 12, 31);
                string   houseJobDirectory = Path.Combine(_services.RunningConfig.Directories.HouseJobsDirectory,
                                                          parameters.DstScenario.ToString(),
                                                          parameters.DstYear.ToString());
                DirectoryInfo houseJobDi = new DirectoryInfo(houseJobDirectory);
                WriteDistrictsForLPG(districts, houseJobDi, _services.Logger, parameters, endtime, pgRo);
            }
            else
            {
                DateTime startSavingToDB = DateTime.Now;
                pra.FinishSavingEverything();
                Info("Finished writing prosumers to db. Duration: " + (DateTime.Now - startSavingToDB).TotalMinutes.ToString("F2") + " minutes");
            }

            pra.Dispose();
            var excelFiles = WriteExcelResultFiles(parameters, makeAndRegisterFullFilename, processingMode, pgRo);

            if (_services.RunningConfig.CollectFilesForArchive)
            {
                SaveToArchiveDirectory(parameters, excelFiles, _services.StartingTime, _services.RunningConfig);
            }

            WriteBrokenLpgCalcCleanupBatch(parameters, makeAndRegisterFullFilename, processingMode, brokenLpgDirectories, brokenLpgJsons);
            if (processingMode == ProcessingMode.Collecting)
            {
                foreach (var provider in loadProfileProviders.Providers)
                {
                    provider.DoFinishCheck();
                }
            }
        }