private static string Request(string url)
{
using (Devmasters.Net.Web.URLContent http = new Devmasters.Net.Web.URLContent(url))
{
http.Timeout = 30000;
http.RequestParams.Headers.Add("Authorization", "Token " + token);
try
{
var a = http.GetContent(System.Text.Encoding.UTF8).Text;
return(a);
}
catch (Devmasters.Net.Web.UrlContentException e)
{
HlidacStatu.Util.Consts.Logger.Error("Merk request " + url, e);
//if (e.StatusCode == System.Net.HttpStatusCode.NotFound)
// return string.Empty;
//else
if (e.InnerException != null && e.InnerException.Message.Contains("204"))
{
return(emptyJson);
}
return(emptyJson);
}
catch (Exception e)
{
HlidacStatu.Util.Consts.Logger.Error("Merk request " + url, e);
//Console.Write(e.ToString());
return(emptyJson);
}
}
}
public bool PostToTopic(int topicId, string post)
{
using (Devmasters.Net.Web.URLContent net = new Devmasters.Net.Web.URLContent(GetUrl("posts", false)))
{
try
{
//net.ContentType = "application/json";
net.Method = Devmasters.Net.Web.MethodEnum.POST;
net.RequestParams.Form.Add("api_key", apiKey);
net.RequestParams.Form.Add("api_username", apiUsername);
net.RequestParams.Form.Add("topic_id", topicId.ToString());
net.RequestParams.Form.Add("raw", post);
var s = net.GetContent().Text;
var ret = Newtonsoft.Json.Linq.JObject.Parse(s);
return(true);
}
catch (Exception e)
{
HlidacStatu.Util.Consts.Logger.Error("PostToTopic error", e);
return(false);
}
}
}
private static byte[] GetBinaryDataFromUrl(KeyAndId ki)
{
byte[] data = null;
try
{
using (Devmasters.Net.Web.URLContent net = new Devmasters.Net.Web.URLContent(ki.ValueForData))
{
net.Timeout = 7000;
net.IgnoreHttpErrors = true;
data = net.GetBinary().Binary;
}
}
catch (Exception e)
{
HlidacStatu.Util.Consts.Logger.Error("Manager Save", e);
}
if (data == null || data.Length == 0)
{
return(System.IO.File.ReadAllBytes(HlidacStatu.Lib.Init.WebAppRoot + @"content\icons\largetile.png"));
}
else
{
return(data);
}
}
private static string DownloadStatement(string statementUrl, string root)
{
var pdfTmpFile = Path.Combine(root, "statement.pdf");
using (var net = new Devmasters.Net.Web.URLContent(statementUrl))
{
File.WriteAllBytes(pdfTmpFile, net.GetBinary().Binary);
}
return(pdfTmpFile);
}
public ActionResult Index()
{
//global::hlst
ViewBag.Token = HlidacStatu.Lib.Data.AspNetUserToken.GetToken(this.User.Identity.Name).Token.ToString("N");
if (!string.IsNullOrEmpty(Request.QueryString["getocr"]))
{
using (Devmasters.Net.Web.URLContent url = new Devmasters.Net.Web.URLContent(
$"https://ocr.hlidacstatu.cz/AddApi.ashx?apikey={Devmasters.Core.Util.Config.GetConfigValue("OCRServerApiKey")}&email={this.User.Identity.Name}"
))
{
var json = Newtonsoft.Json.Linq.JToken.Parse(url.GetContent().Text);
ViewBag.OcrToken = json.Value <string>("apikey");
}
}
return(View());
}
private Dictionary <string, DateTime> GetBankStatementLinks()
{
using (var url = new Devmasters.Net.Web.URLContent(Ucet.Url))
{
var doc = new XPath(url.GetContent().Text);
return(doc.GetNodes(
"//div[@class='npw-transaction-group']/ul[@class='npw-documents']//a[text()[contains(.,'Transakce')]]")
?.Select(n => new
{
url = "https://www.csob.cz" + n.Attributes["href"].Value,
month = "01-" + n.InnerText.Replace("Transakce ", "").Replace("/", "-").Trim()
}
)
?.ToDictionary(k => k.url,
v => DateTime.ParseExact(v.month, "dd-MM-yyyy", Consts.czCulture))
?? new Dictionary <string, DateTime>());
;
}
}
public bool InviteNewUser(string email)
{
using (Devmasters.Net.Web.URLContent net = new Devmasters.Net.Web.URLContent(GetUrl("invites", false)))
{
try
{
//net.ContentType = "application/json";
net.Method = Devmasters.Net.Web.MethodEnum.POST;
net.RequestParams.Form.Add("api_key", apiKey);
net.RequestParams.Form.Add("api_username", apiUsername);
net.RequestParams.Form.Add("group_names", "Hlidac-team-members");
net.RequestParams.Form.Add("email", email);
net.RequestParams.Form.Add("custom_message", @"Ahoj.
Toto je pozvámka do Platforma.Hlidacstatu.cz - veřejné diskuzní platformy s privátní částí pro naše teamové diskuze.
Interně je to vhodný systém pro diskuzi nad konkrétními projekty, nápady, kdy jsou jednotlivá témata diskuze od sebe oddělena (narozdíl od Slacku).\n
Současně je to platforma pro veřejnou diskuzi a kontakt s veřejností.
Michal Bláha");
var s = net.GetContent().Text;
var ret = Newtonsoft.Json.Linq.JObject.Parse(s);
if (ret["success"].ToObject <string>() == "OK")
{
return(true);
}
else
{
return(false);
}
}
catch (Exception e)
{
HlidacStatu.Util.Consts.Logger.Error("platforma invitation error", e);
return(false);
}
}
}
private static GetInfoResponse GetResponse(string request)
{
try
{
using (Devmasters.Net.Web.URLContent net = new Devmasters.Net.Web.URLContent("https://www.mojedatovaschranka.cz/sds/ws/call"))
{
net.Method = Devmasters.Net.Web.MethodEnum.POST;
net.RequestParams.RawContent = request;
var resp = net.GetContent().Text;
XmlSerializer serializer = new XmlSerializer(typeof(GetInfoResponse));
GetInfoResponse obj = serializer.Deserialize(new System.IO.StringReader(resp)) as GetInfoResponse;
return(obj);
}
}
catch (Exception e)
{
HlidacStatu.Util.Consts.Logger.Error("GetDatoveSchranky request error", e);
return(null);
}
}
TimeSpan.FromDays(365 * 10)); //10 years
private static byte[] getRawStemsFromServer(KeyAndId smlouvaKeyId)
{
Smlouva s = Smlouva.Load(smlouvaKeyId.ValueForData);
if (s == null)
{
return(null);
}
var settings = new JsonSerializerSettings();
settings.ContractResolver = new HlidacStatu.Util.FirstCaseLowercaseContractResolver();
using (Devmasters.Net.Web.URLContent stem = new Devmasters.Net.Web.URLContent(classificationBaseUrl() + "/stemmer"))
{
stem.Method = Devmasters.Net.Web.MethodEnum.POST;
stem.Tries = 3;
stem.TimeInMsBetweenTries = 5000;
stem.Timeout = 1000 * 60 * 30; //30min
stem.ContentType = "application/json; charset=utf-8";
stem.RequestParams.RawContent = Newtonsoft.Json.JsonConvert.SerializeObject(s, settings);
Devmasters.Net.Web.BinaryContentResult stems = null;
try
{
Util.Consts.Logger.Debug("Getting stems from " + stem.Url);
stems = stem.GetBinary();
return(stems.Binary);
}
catch (Exception e)
{
Util.Consts.Logger.Error("Classification Stemmer API error " + stem.Url, e);
throw;
}
}
}
public static Dictionary <Lib.Data.Smlouva.SClassification.ClassificationsTypes, decimal> GetClassificationFromServer(Smlouva s, bool rewriteStems = false)
{
Dictionary <Lib.Data.Smlouva.SClassification.ClassificationsTypes, decimal> data = new Dictionary <Smlouva.SClassification.ClassificationsTypes, decimal>();
if (s.Prilohy.All(m => m.EnoughExtractedText) == false)
{
return(null);
}
var settings = new JsonSerializerSettings();
settings.ContractResolver = new HlidacStatu.Util.FirstCaseLowercaseContractResolver();
var stems = GetRawStems(s, rewriteStems);
if (string.IsNullOrEmpty(stems))
{
return(data);
}
using (Devmasters.Net.Web.URLContent classif = new Devmasters.Net.Web.URLContent(classificationBaseUrl() + "/classifier"))
{
classif.Method = Devmasters.Net.Web.MethodEnum.POST;
classif.Tries = 3;
classif.TimeInMsBetweenTries = 5000;
classif.Timeout = 180000;
classif.ContentType = "application/json; charset=utf-8";
classif.RequestParams.RawContent = stems;
Devmasters.Net.Web.TextContentResult classifier = null;
try
{
Util.Consts.Logger.Debug("Getting classification from " + classif.Url);
classifier = classif.GetContent();
}
catch (Exception e)
{
Util.Consts.Logger.Error("Classification Classifier API " + classif.Url, e);
throw;
}
using (Devmasters.Net.Web.URLContent fin = new Devmasters.Net.Web.URLContent(classificationBaseUrl() + "/finalizer"))
{
fin.Method = Devmasters.Net.Web.MethodEnum.POST;
fin.Tries = 3;
fin.TimeInMsBetweenTries = 5000;
fin.Timeout = 30000;
fin.ContentType = "application/json; charset=utf-8";
fin.RequestParams.RawContent = classifier.Text;
Devmasters.Net.Web.TextContentResult res = null;
try
{
Util.Consts.Logger.Debug("Getting classification finalizer from " + classif.Url);
res = fin.GetContent();
}
catch (Exception e)
{
Util.Consts.Logger.Error("Classification finalizer API " + classif.Url, e);
throw;
}
var jsonData = Newtonsoft.Json.Linq.JObject.Parse(res.Text);
foreach (JProperty item in jsonData.Children())
{
string key = item.Name.Replace("-", "_");// jsonData[i][0].Value<string>().Replace("-", "_");
decimal prob = HlidacStatu.Util.ParseTools.ToDecimal(item.Value.ToString()) ?? 0;
if (Enum.TryParse <Smlouva.SClassification.ClassificationsTypes>(key, out var typ))
{
if (!data.ContainsKey(typ))
{
data.Add(typ, prob);
}
else if (typ == SClassification.ClassificationsTypes.OSTATNI)
{
Util.Consts.Logger.Warning($"Classification type lookup failure : { key }");
}
}
else
{
Util.Consts.Logger.Warning("Classification type lookup failure - Invalid key " + key);
data.Add(Smlouva.SClassification.ClassificationsTypes.OSTATNI, prob);
}
}
}
}
return(data);
}
protected override IEnumerable <IBankovniPolozka> DoParse(DateTime?fromDate = null, DateTime?toDate = null)
{
List <IBankovniPolozka> polozky = new List <IBankovniPolozka>();
if (!fromDate.HasValue)
{
fromDate = DateTime.Now.Date.AddYears(-1).AddDays(1);
}
if (!toDate.HasValue)
{
toDate = DateTime.Now.Date;
}
int page = 0;
//https://www.rb.cz/o-nas/povinne-uverejnovane-informace/transparentni-ucty?p_p_id=Transparentaccountportlet_WAR_Transparentaccountportlet_INSTANCE_e6cf4781&p_p_lifecycle=2&p_p_state=normal&p_p_mode=view&p_p_resource_id=nextTransactions&p_p_cacheability=cacheLevelPage&p_p_col_id=_DynamicNestedPortlet_INSTANCE_f5c4beca__column-1-1&p_p_col_count=1&idBankAccount=24389217&fromIndex=51&dateFrom=2016-3-1&dateTo=2018-3-9&q=
//https://www.rb.cz/o-nas/povinne-uverejnovane-informace/transparentni-ucty?p_p_id=Transparentaccountportlet_WAR_Transparentaccountportlet_INSTANCE_e6cf4781&p_p_lifecycle=2&p_p_state=normal&p_p_mode=view&p_p_resource_id=nextTransactions&p_p_cacheability=cacheLevelPage&p_p_col_id=_DynamicNestedPortlet_INSTANCE_f5c4beca__column-1-1&p_p_col_count=1&idBankAccount=24389217&fromIndex=0&dateFrom=2016-3-1&dateTo=2018-3-17&q=
using (Devmasters.Net.Web.URLContent baseUrl = new Devmasters.Net.Web.URLContent(this.Ucet.Url))
{
baseUrl.IgnoreHttpErrors = true;
var html = baseUrl.GetContent(Encoding.UTF8);
var webReqInstance = HlidacStatu.Util.ParseTools.GetRegexGroupValue(html.Text, "Transparentaccountportlet_INSTANCE_(?<inst>[a-z0-9]*)_", "inst");
var dynamicInst = HlidacStatu.Util.ParseTools.GetRegexGroupValue(html.Text, "p_p_id_DynamicNestedPortlet_INSTANCE_(?<inst>[a-z0-9]*)_", "inst");
var internalIdBankAccount = HlidacStatu.Util.ParseTools.GetRegexGroupValue(html.Text, @"idBankAccount=(?<id>\d*)", "id");
if (!string.IsNullOrEmpty(webReqInstance))
{
bool getSomeData = true;
string cisloUctu = this.Ucet.CisloUctu.Split('/')[0];
do
{
string url = string.Format(@"https://www.rb.cz/o-nas/povinne-uverejnovane-informace/transparentni-ucty?" +
"p_p_id=Transparentaccountportlet_WAR_Transparentaccountportlet_INSTANCE_{0}&p_p_lifecycle=2&p_p_state=normal"
+ "&p_p_mode=view&p_p_resource_id=nextTransactions&p_p_cacheability=cacheLevelPage"
+ "&p_p_col_id=_DynamicNestedPortlet_INSTANCE_{1}__column-1-1&p_p_col_count=1"
+ "&idBankAccount={2}&fromIndex={3}&dateFrom={4}&dateTo={5}&q="
, webReqInstance, dynamicInst, internalIdBankAccount, page * 20 + 1, fromDate.Value.ToString("yyyy-M-d"), toDate.Value.ToString("yyyy-M-d"));
using (Devmasters.Net.Web.URLContent net = new Devmasters.Net.Web.URLContent(url, html.Context))
{
net.IgnoreHttpErrors = true;
var json = net.GetContent().Text;
try
{
RBData data = Newtonsoft.Json.JsonConvert.DeserializeObject <RBData>(json);
page++;
if (data.transactions != null && data.transactions.Count() > 0)
{
getSomeData = true;
polozky.AddRange(
data.transactions
.Select(m => new SimpleBankovniPolozka()
{
Castka = HlidacStatu.Util.ParseTools.ToDecimal(m.amount) ?? 0,
CisloProtiuctu = "",
CisloUctu = this.Ucet.CisloUctu,
Datum = HlidacStatu.Util.ParseTools.ToDateTime(m.datumDate, "dd.MM.yyyy").Value,
KS = m.constSymbol,
NazevProtiuctu = m.accountName,
PopisTransakce = m.type,
SS = m.specSymbol,
VS = m.varSymbol,
ZdrojUrl = baseUrl.Url,
ZpravaProPrijemce = m.info,
})
);
}
else
{
getSomeData = false;
}
}
catch (Exception e)
{
TULogger.Error("RB parser JSON error", e);
return(polozky);
}
}
} while (getSomeData);
}
}
return(polozky);
}
public void SaveAttachmentsToDisk(bool rewriteExisting = false)
{
//string root = AppDomain.CurrentDomain.BaseDirectory + "\\Prilohy\\";
//if (!System.IO.Directory.Exists(root))
// System.IO.Directory.CreateDirectory(root);
//string dir = root + item.Id;
//if (!System.IO.Directory.Exists(dir))
//{
// System.IO.Directory.CreateDirectory(dir);
//}
var io = Lib.Init.PrilohaLocalCopy;
int count = 0;
string listing = "";
if (this.Prilohy != null)
{
if (!System.IO.Directory.Exists(io.GetFullDir(this)))
{
System.IO.Directory.CreateDirectory(io.GetFullDir(this));
}
foreach (var p in this.Prilohy)
{
string attUrl = p.odkaz;
if (string.IsNullOrEmpty(attUrl))
{
continue;
}
count++;
string fullPath = io.GetFullPath(this, p);
listing = listing + string.Format("{0} : {1} \n", count, System.Net.WebUtility.UrlDecode(attUrl));
if (!System.IO.File.Exists(fullPath) || rewriteExisting)
{
try
{
using (Devmasters.Net.Web.URLContent url = new Devmasters.Net.Web.URLContent(attUrl))
{
url.Timeout = url.Timeout * 10;
byte[] data = url.GetBinary().Binary;
System.IO.File.WriteAllBytes(fullPath, data);
//p.LocalCopy = System.Text.UTF8Encoding.UTF8.GetBytes(io.GetRelativePath(item, p));
}
}
catch (Exception e)
{
HlidacStatu.Util.Consts.Logger.Error(attUrl, e);
}
}
if (p.hash == null)
{
using (FileStream filestream = new FileStream(fullPath, FileMode.Open))
{
using (SHA256 mySHA256 = SHA256Managed.Create())
{
filestream.Position = 0;
byte[] hashValue = mySHA256.ComputeHash(filestream);
p.hash = new Lib.XSD.tHash()
{
algoritmus = "sha256",
Value = BitConverter.ToString(hashValue).Replace("-", String.Empty)
};
}
}
}
//System.IO.File.WriteAllText(dir + "\\" + "content.nfo", listing);
}
}
}
private Lib.Data.Logs.ProfilZadavateleDownload _processReqProfiluZadavatel(VZ.ProfilZadavatele profil, DateTime from, DateTime to)
{
string xmlUrlTemp = profil.Url;
if (profil.Url?.EndsWith("/") == true)
{
xmlUrlTemp = xmlUrlTemp + "XMLdataVZ?od={0:ddMMyyy}&do={1:ddMMyyyy}";
}
else
{
xmlUrlTemp = xmlUrlTemp + "/XMLdataVZ?od={0:ddMMyyy}&do={1:ddMMyyyy}";
}
var xml = "";
Devmasters.Core.StopWatchEx sw = new StopWatchEx();
sw.Start();
var surl = string.Format(xmlUrlTemp, from, to);
var ReqLog = new Lib.Data.Logs.ProfilZadavateleDownload()
{
Date = DateTime.Now, ProfileId = profil.Id, RequestedUrl = surl
};
try
{
sem.WaitOne();
using (Devmasters.Net.Web.URLContent net = new Devmasters.Net.Web.URLContent(surl))
{
//net.TimeInMsBetweenTries = 20*1000;
//net.Tries = 1;
net.Timeout = 60 * 1000;
xml = net.GetContent().Text;
ReqLog.HttpValid = true;
}
}
catch (Devmasters.Net.Web.UrlContentException ex)
{
ReqLog.HttpValid = false;
ReqLog.HttpError = ex.ToString();
if (ex.InnerException != null && ex.InnerException.GetType() == typeof(System.Net.WebException))
{
var wex = (System.Net.WebException)ex.InnerException;
ReqLog.HttpError = wex.ToString();
if (wex.Status == WebExceptionStatus.ProtocolError && wex.Response != null)
{
ReqLog.HttpErrorCode = (int)(((HttpWebResponse)wex.Response).StatusCode);
}
}
ReqLog.Save();
profil.LastAccessResult = VZ.ProfilZadavatele.LastAccessResults.HttpError;
profil.LastAccess = DateTime.Now;
profil.Save();
return(ReqLog);
}
catch (System.Net.WebException wex)
{
ReqLog.HttpValid = false;
ReqLog.HttpError = wex.ToString();
if (wex.Status == WebExceptionStatus.ProtocolError && wex.Response != null)
{
ReqLog.HttpErrorCode = (int)(((HttpWebResponse)wex.Response).StatusCode);
}
ReqLog.Save();
profil.LastAccessResult = VZ.ProfilZadavatele.LastAccessResults.HttpError;
profil.LastAccess = DateTime.Now;
profil.Save();
return(ReqLog);
}
catch (Exception e)
{
ReqLog.HttpValid = false;
ReqLog.HttpError = e.ToString();
ReqLog.Save();
profil.LastAccessResult = VZ.ProfilZadavatele.LastAccessResults.HttpError;
profil.LastAccess = DateTime.Now;
profil.Save();
return(ReqLog);
}
finally
{
sem.Release();
sw.Stop();
ReqLog.ResponseMs = sw.ElapsedMilliseconds;
}
Lib.Data.External.ProfilZadavatelu.ProfilStructure prof = null;
try
{
prof = ParserXml(xml);
ReqLog.XmlValid = true;
}
catch (Exception e)
{
ReqLog.XmlValid = false;
ReqLog.XmlError = e.ToString();
ReqLog.XmlInvalidContent = xml;
ReqLog.Save();
profil.LastAccessResult = VZ.ProfilZadavatele.LastAccessResults.XmlError;
profil.LastAccess = DateTime.Now;
profil.Save();
return(ReqLog);
}
if (prof != null)
{
var cli = Lib.ES.Manager.GetESClient_VerejneZakazkyNaProfiluRaw();
foreach (var zak in prof.zakazka)
{
Lib.Data.External.ProfilZadavatelu.ZakazkaRaw myZak = new Lib.Data.External.ProfilZadavatelu.ZakazkaRaw(zak, profil);
myZak.Save();
}
ReqLog.Save();
profil.LastAccessResult = VZ.ProfilZadavatele.LastAccessResults.OK;
profil.LastAccess = DateTime.Now;
profil.Save();
}
return(ReqLog);
}
static string ParseTiskovku(DatasetConnector dsc, zapis zap)
{
string html = "";
using (var net = new Devmasters.Net.Web.URLContent(zap.url))
{
html = net.GetContent().Text;
}
var xp = new XPath(html);
var casti = xp.GetNodes("//div[@class='content-main']//div[@class='detail']/p");
if (casti == null)
{
return(""); // zatim neni prepsana, jdi na dalsi
}
List <zapis.vstup> vyjadreni = new List <zapis.vstup>();
int poradi = 1;
zapis.vstup vyj = new zapis.vstup();
vyj.poradi = poradi;
string prevMluvci = "~~~";
foreach (var cast in casti)
{
string text = System.Net.WebUtility.HtmlDecode(cast.InnerText);
string mluvci = XPath.Tools.GetNodeText(cast, "./strong");
if (string.IsNullOrWhiteSpace(mluvci))
{
vyj.text = vyj.text + "\n" + text;
}
else if (mluvci != prevMluvci)
{
if (prevMluvci != "~~~" && !string.IsNullOrEmpty(vyj.jmeno) && !string.IsNullOrEmpty(vyj.prijmeni))
{
vyjadreni.Add(vyj);
poradi++;
vyj = new zapis.vstup();
vyj.poradi = poradi;
}
using (var net = new Devmasters.Net.Web.URLContent($"https://www.hlidacstatu.cz/api/v1/PolitikFromText?text={System.Net.WebUtility.UrlEncode(mluvci)}&Authorization={System.Configuration.ConfigurationManager.AppSettings["apikey"]}"))
{
net.Timeout = 60 * 1000;
var osobahtml = net.GetContent().Text;
var osoba = Newtonsoft.Json.Linq.JObject.Parse(osobahtml);
if (!string.IsNullOrEmpty(osoba.Value <string>("jmeno")))
{
vyj.jmeno = osoba.Value <string>("jmeno");
vyj.prijmeni = osoba.Value <string>("prijmeni");
vyj.osobaId = osoba.Value <string>("osobaid");
var info = Regex.Replace(mluvci, $@"({vyj.jmeno} \s {vyj.prijmeni} \s* ,?) | ({vyj.prijmeni} \s {vyj.jmeno} \s* ,?)", "", options)
.Replace(":", "");
vyj.osobainfo = info?.Trim();
}
else
{
vyj.osobainfo = mluvci;
}
}
text = text.Replace(mluvci, "").Trim();
vyj.text = text;
prevMluvci = mluvci;
}
}
if (!string.IsNullOrEmpty(vyj.text) && !string.IsNullOrEmpty(vyj.jmeno) && !string.IsNullOrEmpty(vyj.prijmeni))
{
vyjadreni.Add(vyj);
}
////////// END vyjadreni
zap.vstupy = vyjadreni.ToArray();
//mp3
var mp3urls = xp.GetNodes("//div[@class='record']//a");
if (mp3urls != null && mp3urls.Count > 0)
{
var ahref = mp3urls
.FirstOrDefault(m => m.Attributes["href"]?.Value?.ToLower()?.EndsWith(".mp3") == true);
if (ahref != null)
{
zap.audio = new zapis.odkaz()
{
url = "https://www.vlada.cz" + ahref.Attributes["href"].Value, nazev = "Zvukový záznam"
}
}
;
}
////// MP3
//RELATED
var relsdt = xp.GetNodes("//dl[@class='related']/dt");
var relsdd = xp.GetNodes("//dl[@class='related']/dd");
if (relsdd?.Count > 0 && relsdd?.Count == relsdt?.Count)
{
List <zapis.odkaz> odkazy = new List <zapis.odkaz>();
for (int i = 0; i < relsdt.Count; i++)
{
var odkaz = new zapis.odkaz();
odkaz.nazev = System.Net.WebUtility.HtmlDecode(relsdt[i].InnerText) + ": " + System.Net.WebUtility.HtmlDecode(relsdd[i].InnerText);
odkaz.url = "https://www.vlada.cz" + XPath.Tools.GetNodeAttributeValue(relsdd[i], "./a", "href");
odkazy.Add(odkaz);
}
zap.souvisejici = odkazy.ToArray();
}
if (zap.vstupy.Count() == 0)
{
if (html.Contains("<ifram") ||
html.Contains("jwplayer")
)
{
zap.vstupy = new zapis.vstup[] { new zapis.vstup()
{
poradi = 1, text = "Přepis tiskové konference není dostupný, pouze videozáznam."
} };
}
else
{
zap.vstupy = new zapis.vstup[] { new zapis.vstup()
{
poradi = 1, text = "Přepis tiskové konference není dostupný."
} };
}
}
Console.WriteLine(zap.Id + " " + zap.nazev);
zap.PrepareBeforeSave();
var id = dsc.AddItemToDataset <zapis>(Parse.datasetname, zap, DatasetConnector.AddItemMode.Rewrite).Result;
return(id);
}
public static void DownloadAllData(DatasetConnector dsc, DateTime?from = null)
{
from = from ?? new DateTime(2006, 1, 1);
List <zapis> tiskovky = new List <zapis>();
int page = 1;
bool nextPage = false;
do
{
nextPage = false;
string url = string.Format(Program.startUrl, page);
Console.WriteLine($"Page {page}");
string html = "";
using (var net = new Devmasters.Net.Web.URLContent(url))
{
html = net.GetContent().Text;
}
var xp = new XPath(html);
var items = xp.GetNodes("//div[@class='record-offset']//div[@class='record']");
foreach (var item in items)
{
zapis zap = new zapis();
zap.datum = DateTime.ParseExact(
XPath.Tools.GetNodeText(item, "./p[@class='info']"),
"d. M. yyyy",
System.Globalization.CultureInfo.GetCultureInfo("en-US"), System.Globalization.DateTimeStyles.AssumeLocal);
zap.nazev = XPath.Tools.GetNodeText(item, "./h2/a");
zap.url = "https://www.vlada.cz" + XPath.Tools.GetNodeAttributeValue(item, "./h2/a", "href");
//https://www.vlada.cz/cz/media-centrum/tiskove-konference/videozaznam-z-tiskove-konference-predsedy-vlady-cr-mirka-topolanka-s-predsedou-vlady-sr-robertem-ficem-19001/
//id from unique page id from URL - last integer in url
zap.Id = Regex.Match(zap.url, @"\w* - (?<id>\d{1,6}) / $", options).Groups["id"].Value;
tiskovky.Add(zap);
}
if (tiskovky.Last().datum <= from)
{
goto parse;
}
//Dokumenty 1301 až 1305 z 1305
var counter = xp.GetNodeText("//p[@class='counter']");
var m = Regex.Match(counter, @"\d* \s* až \s* (?<to>\d{1,4})\s* z \s* (?<z>\d{1,4})", options);
if (m.Success)
{
var to = m.Groups["to"].Value;
var z = m.Groups["z"].Value;
nextPage = to != z;
}
page++;
} while (nextPage);
parse:
Devmasters.Core.Batch.Manager.DoActionForAll <zapis>(tiskovky,
(zap) =>
{
try
{
var ret = ParseTiskovku(dsc, zap);
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
}
return(new Devmasters.Core.Batch.ActionOutputData());
}
, Devmasters.Core.Batch.Manager.DefaultOutputWriter
, new Devmasters.Core.Batch.ActionProgressWriter(0.1f).Write
, !System.Diagnostics.Debugger.IsAttached, 20
);
}
public bool SendNotification(WatchDogProcessor.Result result, string specificContact = null)
{
string contact = specificContact;
if (string.IsNullOrEmpty(contact))
{
if (!string.IsNullOrEmpty(this.OrigWD.SpecificContact))
{
contact = this.OrigWD.SpecificContact;
}
else
{
contact = this.OrigWD.User().Email;
}
}
HlidacStatu.Util.Consts.Logger.Debug($"Processing specific watchdog id {this.OrigWD.Id} ({this.OrigWD.dataType}) SENDING to contact {contact}.");
if (Devmasters.Core.TextUtil.IsValidEmail(contact))
{
return(SendEmailNotification(contact, result));
}
if (contact == "HTTPPOSTBACK")
{
// https://daxiel.solcloud.cz/api/match
/*
* {
* “id”:” id je ID hlidace ve vasem systemu”,
* “found”:[{ExportedVZ}, {ExportedVZ
* }]
* }
* kde ExportedVZ je datova struktura zakazky(viz dale)
*/
if (result.dataType == typeof(VZ.VerejnaZakazka).Name.ToLower())
{
var jsonResult = new
{
id = this.OrigWD.Name.Replace(APIID_Prefix, ""),
found = result.Results
.Cast <VZ.VerejnaZakazka>()
.Take(400)
.Select(m => m.Export())
.ToArray()
};
try
{
using (Devmasters.Net.Web.URLContent net = new Devmasters.Net.Web.URLContent("https://daxiel.solcloud.cz/api/match"))
{
net.Tries = 5;
net.TimeInMsBetweenTries = 2000;
net.Timeout = 30000;
net.Method = Devmasters.Net.Web.MethodEnum.POST;
net.RequestParams.RawContent = Newtonsoft.Json.JsonConvert.SerializeObject(jsonResult);
net.ContentType = "application/json; charset=utf-8";
var apires = net.GetContent();
}
}
catch (Exception ex)
{
HlidacStatu.Util.Consts.Logger.Error("sending data to Solidis error", ex);
throw;
}
}
}
var platformaTopicId = HlidacStatu.Util.ParseTools.GetRegexGroupValue(contact, @"(platforma\((?<platforma>\d*)\))", "platforma");
if (Devmasters.Core.TextUtil.IsNumeric(platformaTopicId))
{
return(SendPlatformaTopicAnswer(Convert.ToInt32(platformaTopicId), result));
}
var twitterCred = HlidacStatu.Util.ParseTools.GetRegexGroupValue(contact, @"^twitter\(\s*(?<json>{ .* })\s* \)", "json");
if (!string.IsNullOrEmpty(twitterCred))
{
//return SendTwitter(twitterCred, result);
}
return(false);
}
public static string GetFileFromPrilohaRepository(HlidacStatu.Lib.Data.Smlouva.Priloha att,
Lib.Data.Smlouva smlouva)
{
var ext = ".pdf";
try
{
ext = new System.IO.FileInfo(att.nazevSouboru).Extension;
}
catch (Exception e)
{
HlidacStatu.Util.Consts.Logger.Warning("invalid file name " + (att?.nazevSouboru ?? "(null)"));
}
string localFile = Lib.Init.PrilohaLocalCopy.GetFullPath(smlouva, att);
var tmpPath = System.IO.Path.GetTempPath();
HlidacStatu.Util.IOTools.DeleteFile(tmpPath);
if (!System.IO.Directory.Exists(tmpPath))
{
try
{
System.IO.Directory.CreateDirectory(tmpPath);
}
catch
{
}
}
string tmpFnSystem = System.IO.Path.GetTempFileName();
string tmpFn = tmpFnSystem + HlidacStatu.Lib.OCR.DocTools.PrepareFilenameForOCR(att.nazevSouboru);
try
{
//System.IO.File.Delete(fn);
if (System.IO.File.Exists(localFile))
{
//do local copy
Consts.Logger.Debug($"Copying priloha {att.nazevSouboru} for smlouva {smlouva.Id} from local disk {localFile}");
System.IO.File.Copy(localFile, tmpFn, true);
}
else
{
try
{
Consts.Logger.Debug($"Downloading priloha {att.nazevSouboru} for smlouva {smlouva.Id} from URL {att.odkaz}");
byte[] data = null;
using (Devmasters.Net.Web.URLContent web = new Devmasters.Net.Web.URLContent(att.odkaz))
{
web.Timeout = web.Timeout * 10;
data = web.GetBinary().Binary;
System.IO.File.WriteAllBytes(tmpFn, data);
}
Consts.Logger.Debug($"Downloaded priloha {att.nazevSouboru} for smlouva {smlouva.Id} from URL {att.odkaz}");
}
catch (Exception)
{
try
{
byte[] data = null;
Consts.Logger.Debug($"Second try: Downloading priloha {att.nazevSouboru} for smlouva {smlouva.Id} from URL {att.odkaz}");
using (Devmasters.Net.Web.URLContent web = new Devmasters.Net.Web.URLContent(att.odkaz))
{
web.Tries = 5;
web.IgnoreHttpErrors = true;
web.TimeInMsBetweenTries = 1000;
web.Timeout = web.Timeout * 20;
data = web.GetBinary().Binary;
System.IO.File.WriteAllBytes(tmpFn, data);
}
Consts.Logger.Debug($"Second try: Downloaded priloha {att.nazevSouboru} for smlouva {smlouva.Id} from URL {att.odkaz}");
return(tmpFn);
}
catch (Exception e)
{
HlidacStatu.Util.Consts.Logger.Error(att.odkaz, e);
return(null);
}
}
}
}
catch (Exception e)
{
HlidacStatu.Util.Consts.Logger.Error(att.odkaz, e);
throw;
}
finally
{
HlidacStatu.Util.IOTools.DeleteFile(tmpFnSystem);
}
return(tmpFn);
}
private IEnumerable <IBankovniPolozka> ParseStatement(string url)
{
var polozky = new HashSet <IBankovniPolozka>();
using (var net = new Devmasters.Net.Web.URLContent(url))
{
net.IgnoreHttpErrors = true;
var content = net.GetContent(Encoding.UTF8).Text;
if (content.Contains("Některé pohyby nemusí být zobrazeny. Zmenšete datumový rozsah."))
{
throw new StatementTooLongException();
}
var doc = new XPath(content);
var xoverviewRows = "//div[contains(@class, 'pohybySum')]/table/tbody/tr";
var overviewRows = doc.GetNodes(xoverviewRows)?.Count ?? 0;
if (overviewRows == 0)
{
TULogger.Warning($"FIO: Account statement page was not found for account {Ucet.CisloUctu}. Account has been probably canceled. Url: {url}");
return(new List <IBankovniPolozka>());
}
var overview = new StatementOverview
{
OpeningBalance = parseAmount(doc.GetNodeText(xoverviewRows + "/td[1]")),
FinalBalance = parseAmount(doc.GetNodeText(xoverviewRows + "/td[2]")),
CreditSum = parseAmount(doc.GetNodeText(xoverviewRows + "/td[3]")),
DebitSum = parseAmount(doc.GetNodeText(xoverviewRows + "/td[4]"))
};
var xrows = "//table[@class='table' and starts-with(@id,'id')]/tbody/tr";
var rows = doc.GetNodes(xrows)?.Count ?? 0;
for (var row = 1; row <= rows; row++)
{
var xroot = xrows + "[" + row + "]";
var p = new SimpleBankovniPolozka
{
CisloUctu = Ucet.CisloUctu,
Datum = ParseTools.ToDateTime(doc.GetNodeText(xroot + "/td[1]"), "dd.MM.yyyy").Value,
Castka = parseAmount(System.Net.WebUtility.HtmlDecode(doc.GetNodeText(xroot + "/td[2]"))),
PopisTransakce = System.Net.WebUtility.HtmlDecode(doc.GetNodeText(xroot + "/td[3]")),
NazevProtiuctu = System.Net.WebUtility.HtmlDecode(doc.GetNodeText(xroot + "/td[4]")),
ZpravaProPrijemce = Devmasters.Core.TextUtil.NormalizeToBlockText(
System.Net.WebUtility.HtmlDecode(doc.GetNodeHtml(xroot + "/td[5]"))
?.Replace("<br>", " \n")
)
};
var poznamka = Devmasters.Core.TextUtil.NormalizeToBlockText(
System.Net.WebUtility.HtmlDecode(doc.GetNodeHtml(xroot + "/td[9]"))
?.Replace("<br>", " \n")
);
if (poznamka != p.ZpravaProPrijemce)
{
p.ZpravaProPrijemce += " " + poznamka;
}
p.KS = doc.GetNodeText(xroot + "/td[6]");
p.VS = doc.GetNodeText(xroot + "/td[7]");
p.SS = doc.GetNodeText(xroot + "/td[8]");
p.ZdrojUrl = net.Url;
p.CisloProtiuctu = ""; //neni k dispozici
if (!polozky.Contains(p))
{
polozky.Add(p);
}
}
ValidateParsedItems(polozky, overview);
}
return(polozky);
}
