static void Main(string[] args) { DateTime start = DateTime.Now; System.Console.WriteLine("Kørsel starter " + start.ToShortTimeString()); var logRepository = LogManager.GetRepository(Assembly.GetEntryAssembly()); System.Console.WriteLine(Assembly.GetEntryAssembly().Location); XmlConfigurator.Configure(logRepository, new FileInfo("log4net.config")); _log.Info("### Regnskabsindlæsning startet. ###"); RegnskabConfig config = new RegnskabConfig(); if (!config.InitializeProgram(args)) { return; } ; config.WriteUsage(); RunSimpleExtraction(config); DateTime s**t = DateTime.Now; _log.Info("### Kørsel varede " + s**t.Subtract(start).TotalSeconds + " sekunder ###"); System.Console.WriteLine("Kørsel slutter " + s**t.ToShortTimeString() + " ialt: " + s**t.Subtract(start).TotalSeconds + " sekunder"); }
public static List <InfoLine> FetchAndTreatDocuments(RegnskabConfig config, DirectoryInfo koerselskatalog) { List <Indberetning> docs = GetAllDocumentsInIndex(config.StartDato, config.SlutDato, config.UseIndexDate); List <InfoLine> resultList = new List <InfoLine>(); _log.Info("Modtaget fra ES: " + docs.Count + " dokumenter, for datoer: " + config.StartDato + "/" + config.SlutDato); if (config.UseMax && docs.Count > config.Max) { dontStop = false; System.Console.WriteLine("For mange dokumenter modtaget, stopper kørsel, sæt use_max=false for at køre alligevel"); _log.Info("For mange dokumenter modtaget, stopper kørsel, sæt use_max=false for at køre alligevel"); return(resultList); } for (int i = 0; i < docs.Count; i += config.Chunks) { int chunk = i + config.Chunks <= docs.Count ? config.Chunks : docs.Count % config.Chunks; _log.Info("Downloader næste " + chunk + "/" + i); IEnumerable <Task <InfoLine> > downloadTasks = from virksomhed in docs.GetRange(i, chunk) select FetchDocument(virksomhed, koerselskatalog); Task <InfoLine>[] downloads = downloadTasks.ToArray(); // Await the completion of all the running tasks. List <InfoLine> list = Task.WhenAll(downloads).Result.ToList(); resultList.AddRange(list); } return(resultList); }
private static void RunSimpleExtraction(RegnskabConfig config) { HttpClientHandler handler = new HttpClientHandler() { AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate }; client = new HttpClient(handler); client.BaseAddress = config.RegnskabsUri; ConnectionSettings con_settings = new ConnectionSettings(config.ErstDistUri) .DefaultIndex("offentliggoerelser") .DefaultTypeName("_doc"); es_client = new ElasticClient(con_settings); Guid koerselsId = Guid.NewGuid(); DateTime startDay = config.StartDato; DateTime endDay = config.SlutDato; dontStop = true; while (endDay.Subtract(startDay).TotalDays > 0 && dontStop) { System.Console.Write(string.Format("Dag: {0} ", startDay.ToString("yyyy-MM-dd"))); var koerselskatalog = Directory.CreateDirectory(config.TempDirectory + "/" + startDay.ToString("yyyy-MM-dd") + "-" + config.RunName + "-" + config.TypeName + "-" + koerselsId); _log.Info("Dannet temp-katalog: " + koerselskatalog.Name); StreamWriter writer = File.CreateText(path: koerselskatalog.FullName + "/index.csv"); using (var csv = new CsvWriter(writer)) { config.StartDato = startDay; config.SlutDato = startDay.AddDays(1); List <InfoLine> lines = FetchAndTreatDocuments(config, koerselskatalog); System.Console.WriteLine(string.Format("Modtaget: {0} regnskaber", lines.Count)); csv.WriteRecords(lines); } _log.Info("Zipper: " + koerselskatalog.FullName); ZipFile.CreateFromDirectory(koerselskatalog.FullName, koerselskatalog.FullName + ".zip"); _log.Info("Flytter til: " + config.TargetDirectory); File.Copy(koerselskatalog.FullName + ".zip", config.TargetDirectory + koerselskatalog.Name + ".zip"); File.SetAttributes(koerselskatalog.FullName, FileAttributes.Normal); Directory.Delete(koerselskatalog.FullName, true); File.Delete(koerselskatalog.FullName + ".zip"); _log.Info(string.Format("### Slettet temp-filer og afslutter for {0} ###", startDay.ToShortDateString())); startDay = startDay.AddDays(1); } }