public bool Save(Uri url) { using (Devmasters.Net.HttpClient.URLContent net = new Devmasters.Net.HttpClient.URLContent(url.AbsoluteUri)) { net.Timeout = 180 * 1000; net.Tries = 3; net.TimeInMsBetweenTries = 10000; var data = net.GetBinary().Binary; var fn = net.ResponseParams.Headers["Content-Disposition"]; if (!string.IsNullOrWhiteSpace(fn)) { fn = Devmasters.RegexUtil.GetRegexGroupValue(fn, "filename=\"(?<fn>.*)\"", "fn"); } var attrs = new FileAttributes() { Downloaded = DateTime.UtcNow, Source = url.AbsoluteUri, ContentType = net.ContentType, Size = data.Length, Filename = fn }; return(Save(ref data, url, attrs)); } }
private static byte[] GetBinaryDataFromUrl(KeyAndId ki) { byte[] data = null; try { using (Devmasters.Net.HttpClient.URLContent net = new Devmasters.Net.HttpClient.URLContent(ki.ValueForData)) { net.Timeout = 7000; net.IgnoreHttpErrors = true; data = net.GetBinary().Binary; } } catch (Exception e) { HlidacStatu.Util.Consts.Logger.Error("Manager Save", e); } if (data == null || data.Length == 0) { return(System.IO.File.ReadAllBytes(HlidacStatu.Lib.Init.WebAppRoot + @"content\icons\largetile.png")); } else { return(data); } }
public static async Task <Result> TextFromUrlAsync(string apikey, Uri url, string client, int priority, MiningIntensity intensity, string origFilename = null, TimeSpan?maxWaitingTime = null, TimeSpan?restartTaskAfterTime = null /*, Api.CallbackData callBackData = null*/) { string fullUrl = null; string taskId = null; var tmpFile = TempIO.GetTemporaryFilename(); try { using (Devmasters.Net.HttpClient.URLContent net = new Devmasters.Net.HttpClient.URLContent(url.AbsoluteUri)) { net.TimeInMsBetweenTries = 2000; net.Timeout = 60000; net.Tries = 5; net.IgnoreHttpErrors = true; System.IO.File.WriteAllBytes(tmpFile, net.GetBinary().Binary); } return(await TextFromFileAsync(apikey, tmpFile, client, priority, intensity, origFilename, maxWaitingTime, restartTaskAfterTime)); } catch (Exception e) { throw new ApiException("exception API TextFromFileAsync ", e); //return new Result() { Id = taskId, IsValid = Result.ResultStatus.Invalid, Error = json["error"].Value<string>() }; } finally { TempIO.DeleteFile(tmpFile); } }
private static string DownloadStatement(string statementUrl, string root) { var pdfTmpFile = Path.Combine(root, "statement.pdf"); using (var net = new Devmasters.Net.HttpClient.URLContent(statementUrl)) { File.WriteAllBytes(pdfTmpFile, net.GetBinary().Binary); } return(pdfTmpFile); }
static void Main(string[] args) { var DArgs = args .Select(m => m.Split('=')) .ToDictionary(m => m[0].ToLower(), v => v.Length == 1 ? "" : v[1]); CreateDataset(DArgs); //find xls url string openDataPage = "https://dip.mzcr.cz/api/v1/kapacity-intenzivni-pece-zdravotnicke-zarizeni.csv"; Uri xlsUrl = null; DateTime fromD = DateTime.Now.Date.AddDays(-30); Devmasters.Logging.Logger.Root.Info("Getting URL of csv from " + openDataPage); using (Devmasters.Net.HttpClient.URLContent net = new Devmasters.Net.HttpClient.URLContent(openDataPage)) { Devmasters.Logging.Logger.Root.Info("Getting csv"); var fn = "kapacity-intenzivni-pece-zdravotnicke-zarizeni.csv"; System.IO.File.WriteAllBytes(fn, net.GetBinary().Binary); using (var reader = new StreamReader(fn)) //new StringReader(html)) { using (var csv = new CsvReader(reader, new CsvHelper.Configuration.CsvConfiguration(System.Globalization.CultureInfo.InvariantCulture) { Delimiter = ",", IgnoreBlankLines = true, HasHeaderRecord = true, TrimOptions = CsvHelper.Configuration.TrimOptions.Trim })) { //csv.Context.RegisterClassMap<NemocniceMap>(); //var records = csv.GetRecords<Nemocnice>().ToArray(); List <Nemocnice> recs = new List <Nemocnice>(); if (true) { csv.Read(); csv.ReadHeader(); while (csv.Read()) { var record = new Nemocnice { crrt_kapacita_celkem = csv.GetField <int>("crrt_kapacita_celkem"), crrt_kapacita_volna = csv.GetField <int>("crrt_kapacita_volna"), ecmo_kapacita_celkem = csv.GetField <int>("ecmo_kapacita_celkem"), ecmo_kapacita_volna = csv.GetField <int>("ecmo_kapacita_volna"), ihd_kapacita_celkem = csv.GetField <int>("ihd_kapacita_celkem"), ihd_kapacita_volna = csv.GetField <int>("ihd_kapacita_volna"), kraj_nazev = csv.GetField <string>("kraj_nazev"), kraj_nuts_kod = csv.GetField <string>("kraj_nuts_kod"), luzka_aro_jip_kapacita_celkem = csv.GetField <int>("luzka_aro_jip_kapacita_celkem"), luzka_aro_jip_kapacita_volna_covid_negativni = csv.GetField <int>("luzka_aro_jip_kapacita_volna_covid_negativni"), luzka_aro_jip_kapacita_volna_covid_pozitivni = csv.GetField <int>("luzka_aro_jip_kapacita_volna_covid_pozitivni"), luzka_standard_kyslik_kapacita_celkem = csv.GetField <int>("luzka_standard_kyslik_kapacita_celkem"), luzka_standard_kyslik_kapacita_volna_covid_negativni = csv.GetField <int>("luzka_standard_kyslik_kapacita_volna_covid_negativni"), luzka_standard_kyslik_kapacita_volna_covid_pozitivni = csv.GetField <int>("luzka_standard_kyslik_kapacita_volna_covid_pozitivni"), reprofilizovana_kapacita_luzka_aro_jip_kapacita_celkem = csv.GetField <int?>("reprofilizovana_kapacita_luzka_aro_jip_kapacita_celkem") ?? 0, reprofilizovana_kapacita_luzka_aro_jip_kapacita_planovana = csv.GetField <int?>("reprofilizovana_kapacita_luzka_aro_jip_kapacita_planovana") ?? 0, reprofilizovana_kapacita_luzka_aro_jip_kapacita_volna = csv.GetField <int?>("reprofilizovana_kapacita_luzka_aro_jip_kapacita_volna") ?? 0, reprofilizovana_kapacita_luzka_standard_kyslik_kapacita_celkem = csv.GetField <int?>("reprofilizovana_kapacita_luzka_standard_kyslik_kapacita_celkem") ?? 0, reprofilizovana_kapacita_luzka_standard_kyslik_kapacita_planovana = csv.GetField <int?>("reprofilizovana_kapacita_luzka_standard_kyslik_kapacita_planovana") ?? 0, reprofilizovana_kapacita_luzka_standard_kyslik_kapacita_volna = csv.GetField <int?>("reprofilizovana_kapacita_luzka_standard_kyslik_kapacita_volna") ?? 0, upv_kapacita_celkem = csv.GetField <int>("upv_kapacita_celkem"), upv_kapacita_volna = csv.GetField <int>("upv_kapacita_volna"), ventilatory_operacni_sal_kapacita_celkem = csv.GetField <int>("ventilatory_operacni_sal_kapacita_celkem"), ventilatory_operacni_sal_kapacita_volna = csv.GetField <int>("ventilatory_operacni_sal_kapacita_volna"), ventilatory_prenosne_kapacita_celkem = csv.GetField <int>("ventilatory_prenosne_kapacita_celkem"), ventilatory_prenosne_kapacita_volna = csv.GetField <int>("ventilatory_prenosne_kapacita_volna"), zz_kod = csv.GetField <string>("zz_kod"), zz_nazev = csv.GetField <string>("zz_nazev"), datum = DateTime.ParseExact(csv.GetField <string>("datum"), "yyyy-MM-dd", System.Globalization.CultureInfo.CurrentCulture) }; var iis = record.id; if (record.datum > fromD) { recs.Add(record); } } } //ds.AddOrRewriteItems(recs); Devmasters.Batch.Manager.DoActionForAll <Nemocnice>(recs.OrderByDescending(o => o.datum), n => { Console.Write("."); ds.AddOrUpdateItem(n, HlidacStatu.Api.V2.Dataset.Typed.ItemInsertMode.skip); return(new Devmasters.Batch.ActionOutputData()); }, false); } } } }
private static void GetExcelFromUzisZIP_Old() { string fn = GetExecutingDirectoryName() + $"\\dip-report-kraje-{DateTime.Now:yyyyMMdd-HHmmss}.xlsx"; string fnTemp = System.IO.Path.GetTempFileName(); //nejnovejsi ZIP for (int i = 0; i < 7; i++) { DateTime dt = DateTime.Now.Date.AddDays(-1 * i); string zipUrl = $"https://share.uzis.cz/s/fbCgFKagS6fCrzc/download?path=%2F{dt.Year}-{dt.ToString("MM")}%20({dt.ToString("MMMM", System.Globalization.CultureInfo.GetCultureInfo("cs"))}%20{dt.Year})&files={dt:yyyy-MM-dd}-dostupnost-kapacit.zip"; //$"https://share.uzis.cz/s/fbCgFKagS6fCrzc/download?path=%2F&files={dt:yyyy-MM-dd}-dostupnost-kapacit.zip"; Devmasters.Logging.Logger.Root.Info($"Getting ZIP url {zipUrl}"); using (Devmasters.Net.HttpClient.URLContent net = new Devmasters.Net.HttpClient.URLContent(zipUrl)) { try { System.IO.File.WriteAllBytes(fnTemp, net.GetBinary().Binary); break; } catch (Exception e) { } } } try { Devmasters.Logging.Logger.Root.Info("Getting Excel from ZIP"); //get xlsx from ZIP using (ZipArchive archive = ZipFile.OpenRead(fnTemp)) { foreach (ZipArchiveEntry entry in archive.Entries) { if (entry.FullName.EndsWith(".xlsx", StringComparison.OrdinalIgnoreCase)) { entry.ExtractToFile(fn); } } } if (false) //download xls from web { //find xls url string openDataPage = "https://onemocneni-aktualne.mzcr.cz/api/v2/covid-19"; Uri xlsUrl = null; Devmasters.Logging.Logger.Root.Info("Getting URL of XLS from " + openDataPage); using (Devmasters.Net.HttpClient.URLContent net = new Devmasters.Net.HttpClient.URLContent(openDataPage)) { Devmasters.Logging.Logger.Root.Info("Getting Excel URL"); var html = net.GetContent().Text; Devmasters.XPath xp = new Devmasters.XPath(html); var node = xp.GetNode("//a[contains(@href,'dip-report-kraje.xlsx')]"); if (node != null) { xlsUrl = new Uri("https://onemocneni-aktualne.mzcr.cz" + node.Attributes["href"].Value); } } if (xlsUrl == null) { Devmasters.Logging.Logger.Root.Fatal("No URL to download"); return; } using (Devmasters.Net.HttpClient.URLContent net = new Devmasters.Net.HttpClient.URLContent(xlsUrl.AbsoluteUri)) { Devmasters.Logging.Logger.Root.Info("Getting Excel"); System.IO.File.WriteAllBytes(fn, net.GetBinary().Binary); } } //debug //fn = @"c:\!!\ONLINE_DISPECINK_IP_dostupne_kapacity_20201014_05-50.xlsx"; ExcelPackage.LicenseContext = LicenseContext.NonCommercial; using (var p = new ExcelPackage(new System.IO.FileInfo(fn))) { ExcelWorksheet ws = p.Workbook.Worksheets[1]; //find date //Analýza provedena z exportu 01.10.2020 for (int row = 1; row < 100000; row++) { Console.Write("."); var txt = ws.Cells[row, 1].GetValue <string>(); if (txt != null && txt.StartsWith("Stav k datu:")) { string head = txt.Replace("Stav k datu: ", ""); string sdate = Devmasters.RegexUtil.GetRegexGroupValue(head, @" \s* (?<dt>\d{1,2}\s*\.\s*\d{1,2}\s*\.\s*\d{4} )", "dt"); DateTime dt = Devmasters.DT.Util.ToDate(sdate).Value; string id = "id_" + dt.ToString("yyyy-MM-dd"); NemocniceData nd = null; try { nd = ds.GetItem(id); // new NemocniceData(); } catch (Exception) { } if (nd == null) { nd = new NemocniceData(); nd.regions = new List <NemocniceData.Region>(); } nd.lastUpdated = dt; nd.id = id; Console.WriteLine("."); Devmasters.Logging.Logger.Root.Info(nd.lastUpdated.ToString()); row = row + 4; List <NemocniceData.Region> finalRegs = new List <NemocniceData.Region>(); for (int regs = 0; regs < 14; regs++) { string region = ws.Cells[row + regs, 1].GetValue <string>(); NemocniceData.Region r = nd.regions.FirstOrDefault(m => m.region == region); //new NemocniceData.Region(); if (r == null) { r = new NemocniceData.Region(); } r.lastModified = nd.lastUpdated; r.region = region; r.UPV_celkem = ws.Cells[row + regs, 4].GetValue <int>(); r.UPV_volna = ws.Cells[row + regs, 5].GetValue <int>(); r.ECMO_celkem = ws.Cells[row + regs, 5].GetValue <int>(); r.ECMO_volna = ws.Cells[row + regs, 6].GetValue <int>(); r.CRRT_celkem = ws.Cells[row + regs, 8].GetValue <int>(); r.CRRT_volna = ws.Cells[row + regs, 9].GetValue <int>(); r.IHD_celkem = ws.Cells[row + regs, 11].GetValue <int>(); r.IHD_volna = ws.Cells[row + regs, 12].GetValue <int>(); r.AROJIP_luzka_celkem = ws.Cells[row + regs, 14].GetValue <int>(); r.AROJIP_luzka_covid = ws.Cells[row + regs, 15].GetValue <int>(); r.AROJIP_luzka_necovid = ws.Cells[row + regs, 16].GetValue <int>(); r.Standard_luzka_s_kyslikem_celkem = ws.Cells[row + regs, 18].GetValue <int>(); r.Standard_luzka_s_kyslikem_covid = ws.Cells[row + regs, 19].GetValue <int>(); r.Standard_luzka_s_kyslikem_necovid = ws.Cells[row + regs, 20].GetValue <int>(); r.Lekari_AROJIP_celkem = ws.Cells[row + regs, 22].GetValue <int>(); r.Lekari_AROJIP_dostupni = ws.Cells[row + regs, 23].GetValue <int>(); r.Sestry_AROJIP_celkem = ws.Cells[row + regs, 25].GetValue <int>(); r.Sestry_AROJIP_dostupni = ws.Cells[row + regs, 26].GetValue <int>(); r.Ventilatory_prenosne_celkem = ws.Cells[row + regs, 28].GetValue <int>(); r.Ventilatory_operacnisal_celkem = ws.Cells[row + regs, 29].GetValue <int>(); r.Standard_luzka_celkem = ws.Cells[row + regs, 30].GetValue <int>(); r.Standard_luzka_s_monitor_celkem = ws.Cells[row + regs, 31].GetValue <int>(); finalRegs.Add(r); } nd.regions = finalRegs; row = row + 16; Devmasters.Logging.Logger.Root.Info("Saving"); ds.AddOrUpdateItem(nd, HlidacStatu.Api.V2.Dataset.Typed.ItemInsertMode.rewrite); } } } } catch (Exception e) { Devmasters.Logging.Logger.Root.Error("Processing ZIP XLS error", e); SendMail("*****@*****.**", "Selhalo zpracovani dat z UZIS", e.ToString(), ""); } }
public static string GetFileFromPrilohaRepository(HlidacStatu.Lib.Data.Smlouva.Priloha att, Lib.Data.Smlouva smlouva) { var ext = ".pdf"; try { ext = new System.IO.FileInfo(att.nazevSouboru).Extension; } catch (Exception) { HlidacStatu.Util.Consts.Logger.Warning("invalid file name " + (att?.nazevSouboru ?? "(null)")); } string localFile = Lib.Init.PrilohaLocalCopy.GetFullPath(smlouva, att); var tmpPath = System.IO.Path.GetTempPath(); Devmasters.IO.IOTools.DeleteFile(tmpPath); if (!System.IO.Directory.Exists(tmpPath)) { try { System.IO.Directory.CreateDirectory(tmpPath); } catch { } } string tmpFnSystem = System.IO.Path.GetTempFileName(); string tmpFn = tmpFnSystem + HlidacStatu.Lib.OCR.DocTools.PrepareFilenameForOCR(att.nazevSouboru); try { //System.IO.File.Delete(fn); if (System.IO.File.Exists(localFile)) { //do local copy Consts.Logger.Debug($"Copying priloha {att.nazevSouboru} for smlouva {smlouva.Id} from local disk {localFile}"); System.IO.File.Copy(localFile, tmpFn, true); } else { try { Consts.Logger.Debug($"Downloading priloha {att.nazevSouboru} for smlouva {smlouva.Id} from URL {att.odkaz}"); byte[] data = null; using (Devmasters.Net.HttpClient.URLContent web = new Devmasters.Net.HttpClient.URLContent(att.odkaz)) { web.Timeout = web.Timeout * 10; data = web.GetBinary().Binary; System.IO.File.WriteAllBytes(tmpFn, data); } Consts.Logger.Debug($"Downloaded priloha {att.nazevSouboru} for smlouva {smlouva.Id} from URL {att.odkaz}"); } catch (Exception) { try { if (Uri.TryCreate(att.odkaz, UriKind.Absolute, out var urlTmp)) { byte[] data = null; Consts.Logger.Debug($"Second try: Downloading priloha {att.nazevSouboru} for smlouva {smlouva.Id} from URL {att.odkaz}"); using (Devmasters.Net.HttpClient.URLContent web = new Devmasters.Net.HttpClient.URLContent(att.odkaz)) { web.Tries = 5; web.IgnoreHttpErrors = true; web.TimeInMsBetweenTries = 1000; web.Timeout = web.Timeout * 20; data = web.GetBinary().Binary; System.IO.File.WriteAllBytes(tmpFn, data); } Consts.Logger.Debug($"Second try: Downloaded priloha {att.nazevSouboru} for smlouva {smlouva.Id} from URL {att.odkaz}"); return(tmpFn); } return(null); } catch (Exception e) { HlidacStatu.Util.Consts.Logger.Error(att.odkaz, e); return(null); } } } } catch (Exception e) { HlidacStatu.Util.Consts.Logger.Error(att.odkaz, e); throw; } finally { Devmasters.IO.IOTools.DeleteFile(tmpFnSystem); } return(tmpFn); }