/// <summary>
/// Gets classification
/// </summary>
/// <param name="idSmlouvy"></param>
/// <returns>Classification json</returns>
public static string GetClassificationExplanation(string idSmlouvy)
{
if (string.IsNullOrWhiteSpace(idSmlouvy))
{
return(null);
}
Smlouva s = Smlouva.Load(idSmlouvy);
if (s == null)
{
return(null);
}
var settings = new JsonSerializerSettings();
settings.ContractResolver = new Util.FirstCaseLowercaseContractResolver();
var response = CallEndpoint("explain_json",
JsonConvert.SerializeObject(s, settings),
idSmlouvy,
1000 * 60 * 10);
return(System.Text.RegularExpressions.Regex.Unescape(response));
}
public static Smlouva PrepareForDump(Smlouva s)
{
if (s.znepristupnenaSmlouva() && s.Prilohy != null)
{
foreach (var p in s.Prilohy)
{
p.PlainTextContent = "-- anonymizovano serverem HlidacStatu.cz --";
p.odkaz = "";
}
}
if (s.Prilohy != null)
{
foreach (var p in s.Prilohy)
{
p.DatlClassification = null;
p.FileMetadata = null;
}
}
s.SVazbouNaPolitiky = null;
s.SVazbouNaPolitikyAktualni = null;
s.SVazbouNaPolitikyNedavne = null;
return(s);
}
TimeSpan.FromDays(365 * 10)); //10 years
private static byte[] getRawStemsFromServer(KeyAndId smlouvaKeyId)
{
Smlouva s = Smlouva.Load(smlouvaKeyId.ValueForData);
if (s == null)
{
return(null);
}
var settings = new JsonSerializerSettings();
settings.ContractResolver = new HlidacStatu.Util.FirstCaseLowercaseContractResolver();
string stemmerResponse = CallEndpoint("stemmer",
JsonConvert.SerializeObject(s, settings),
s.Id,
1000 * 60 * 30);
try
{
var jtoken = JToken.Parse(stemmerResponse);
}
catch (JsonReaderException e)
{
Util.Consts.Logger.Error($"Stemmer returned incomplete json for {smlouvaKeyId.ValueForData}", e);
throw;
}
return(Encoding.UTF8.GetBytes(stemmerResponse));
}
TimeSpan.FromDays(365 * 10)); //10 years
private static byte[] getRawStemsFromServer(KeyAndId smlouvaKeyId)
{
Smlouva s = Smlouva.Load(smlouvaKeyId.ValueForData);
if (s == null)
{
return(null);
}
var settings = new JsonSerializerSettings();
settings.ContractResolver = new HlidacStatu.Util.FirstCaseLowercaseContractResolver();
using (Devmasters.Net.Web.URLContent stem = new Devmasters.Net.Web.URLContent(classificationBaseUrl() + "/stemmer"))
{
stem.Method = Devmasters.Net.Web.MethodEnum.POST;
stem.Tries = 3;
stem.TimeInMsBetweenTries = 5000;
stem.Timeout = 1000 * 60 * 30; //30min
stem.ContentType = "application/json; charset=utf-8";
stem.RequestParams.RawContent = Newtonsoft.Json.JsonConvert.SerializeObject(s, settings);
Devmasters.Net.Web.BinaryContentResult stems = null;
try
{
Util.Consts.Logger.Debug("Getting stems from " + stem.Url);
stems = stem.GetBinary();
return(stems.Binary);
}
catch (Exception e)
{
Util.Consts.Logger.Error("Classification Stemmer API error " + stem.Url, e);
throw;
}
}
}
public static string GetRawStems(Smlouva s, bool rewriteStems = false)
{
if (s == null)
{
return(null);
}
var key = new KeyAndId()
{
ValueForData = s.Id, CacheNameOnDisk = $"stem_smlouva_{s.Id}"
};
if (rewriteStems)
{
InvalidateStemCache(s.Id);
}
var data = stemCacheManager.Get(key);
if (data == null)
{
return(null);
}
return(System.Text.Encoding.UTF8.GetString(data));
}
public static Dictionary <ClassificationsTypes, decimal> GetClassificationFromServer(Smlouva s, bool rewriteStems = false)
{
Dictionary <ClassificationsTypes, decimal> data = new Dictionary <ClassificationsTypes, decimal>();
if (s.Prilohy.Any(m => m.EnoughExtractedText) == false)
{
return(null);
}
var settings = new JsonSerializerSettings();
settings.ContractResolver = new HlidacStatu.Util.FirstCaseLowercaseContractResolver();
var stems = GetRawStems(s, rewriteStems);
if (string.IsNullOrEmpty(stems))
{
return(data);
}
string classifierResponse = "";
try
{
classifierResponse = CallEndpoint("classifier",
stems,
s.Id,
180000);
}
catch
{
//retry once with new stems
stems = GetRawStems(s, true);
classifierResponse = CallEndpoint("classifier",
stems,
s.Id,
180000);
}
//string finalizerResponse = CallEndpoint("finalizer",
// classifierResponse,
// s.Id,
// 30000);
var jsonData = Newtonsoft.Json.Linq.JObject.Parse(classifierResponse);
if (jsonData.Children().Count() == 0)
{
data.Add(ClassificationsTypes.OSTATNI, 0.6M);
return(data);
}
foreach (JProperty item in jsonData.Children())
{
string key = item.Name.Replace("-", "_").Replace("_generic", "_obecne");// jsonData[i][0].Value<string>().Replace("-", "_");
decimal prob = HlidacStatu.Util.ParseTools.ToDecimal(item.Value.ToString()) ?? 0;
if (prob > 0)
{
if (Enum.TryParse <Smlouva.SClassification.ClassificationsTypes>(key, out var typ))
{
if (!data.ContainsKey(typ))
{
data.Add(typ, prob);
}
else if (typ == SClassification.ClassificationsTypes.OSTATNI)
{
Util.Consts.Logger.Warning($"Classification type lookup failure : { key }");
}
}
else
{
Util.Consts.Logger.Warning("Classification type lookup failure - Invalid key " + key);
if (!data.ContainsKey(Smlouva.SClassification.ClassificationsTypes.OSTATNI))
{
data.Add(Smlouva.SClassification.ClassificationsTypes.OSTATNI, prob);
}
}
}
}
return(data);
}
private static string PrintFlatRelations(Graph.Edge parent, int level, IEnumerable <Graph.Edge> relations, TiskEnum typ,
List <string> renderedIds, bool withStats = true, string highlightSubjId = null)
{
int space = 2;
string horizLine = "--"; //new string('\u2500',2);
string vertLine = "|"; //new string('\u2502',1);
string cross = "+"; //new string('\u251C', 1);
if (renderedIds == null)
{
renderedIds = new List <string>();
}
var rels = relations
.Where(m =>
(
(parent != null && m.From?.UniqId == parent.To?.UniqId)
||
(parent == null && !relations.Any(r => r.To?.UniqId == m.From?.UniqId)) //do root urovne pridej vazby, ktere jsou sirotci bez parenta
)
)
.Distinct()
.GroupBy(k => new { id = k.To.UniqId, type = k.To.Type }, (k, v) =>
{
Graph.Edge withChildren = v.Where(r => relations.Any(m => m.From?.UniqId == r.To?.UniqId)).FirstOrDefault();
if (withChildren == null)
{
withChildren = v.First();
}
return(withChildren);
})
.OrderBy(m => m.To.PrintName(typ == TiskEnum.Html))
.ToArray();
if (rels.Count() == 0)
{
return(string.Empty);
}
StringBuilder sb = new StringBuilder(512);
List <Graph.Edge> deepRels = new List <Graph.Edge>();
switch (typ)
{
case TiskEnum.Text:
break;
case TiskEnum.Html:
case TiskEnum.Checkbox:
sb.AppendLine("<ul>");
break;
case TiskEnum.Json:
break;
}
for (int i = 0; i < rels.Count(); i++)
{
var rel = rels[i];
if (renderedIds.Contains(rel.To.UniqId))
{
continue;
}
var last = i == (rels.Count() - 1);
Analysis.SubjectStatistic stat = null;
if (withStats && rel.To.Type == Graph.Node.NodeType.Company)
{
stat = new Analysis.SubjectStatistic(rel.To.Id);
}
string subjId = rel.To.Type == Graph.Node.NodeType.Company ? rel.To.Id : "Osoba";
string subjName = rel.To.PrintName();
renderedIds.Add(rel.To.UniqId);
switch (typ)
{
case TiskEnum.Text:
sb.AppendLine(string.Concat(Enumerable.Repeat(vertLine + new string(' ', space), level + 1)));
sb.Append(
string.Concat(
Enumerable.Repeat(
vertLine + new string(' ', space)
, (level))
)
);
if (rel.To.Highlighted)
{
subjName = string.Format("!!{0}!!", subjName);
}
sb.AppendFormat("{0}{1}:{2} {3}\n",
cross + horizLine + " ",
subjId,
subjName,
rel.Doba()
);
sb.Append(PrintFlatRelations(rel, level + 1, relations, typ, renderedIds, withStats));
break;
case TiskEnum.Html:
if (withStats && stat != null)
{
sb.AppendFormat("<li class='{3} {6}'><a href='/subjekt/{0}'>{0}:{1}</a>{7}; {4}, celkem {5}. {2}</li>",
subjId,
subjName,
PrintFlatRelations(rel, level + 1, relations, typ, renderedIds, withStats),
last ? "" : "connect",
PluralForm.Get((int)stat.BasicStatPerYear.Summary.Pocet, "{0} smlouva;{0} smlouvy;{0} smluv"),
Smlouva.NicePrice(stat.BasicStatPerYear.Summary.CelkemCena, html: true, shortFormat: true),
"aktualnost" + ((int)rel.Aktualnost).ToString(),
(rel.Aktualnost < AktualnostType.Aktualni) ? rel.Doba("/{0}/") : string.Empty
);
}
else
{
sb.AppendFormat("<li class='{3} {4}'><a href='/subjekt/{0}'><span class=''>{0}:{1}</span></a>{5}. {2}</li>",
subjId,
subjName,
PrintFlatRelations(rel, level + 1, relations, typ, renderedIds, withStats),
last ? "" : "connect",
"aktualnost" + ((int)rel.Aktualnost).ToString(),
(rel.Aktualnost < AktualnostType.Aktualni) ? rel.Doba("/{0}/") : string.Empty,
(!string.IsNullOrEmpty(highlightSubjId) && subjId == highlightSubjId) ? "highlighted" : ""
);
}
break;
case TiskEnum.Checkbox:
sb.AppendFormat(@"<li class=""{0} {1}""><input type=""checkbox"" name=""ico"" value=""{2}"" /> <span><b>{2}</b> {3}</span>{4}</li>"
, (last ? "" : "connect"),
("aktualnost" + ((int)rel.Aktualnost).ToString()),
subjId, subjName,
PrintFlatRelations(rel, level + 1, relations, typ, renderedIds, withStats)
);
break;
case TiskEnum.Json:
break;
}
}
switch (typ)
{
case TiskEnum.Text:
break;
case TiskEnum.Html:
case TiskEnum.Checkbox:
sb.AppendLine("</ul>");
break;
case TiskEnum.Json:
break;
}
return(sb.ToString());
}
public string RenderHtml(string delimeter = ", ")
{
string zdroj = "";
if (!string.IsNullOrEmpty(this.Zdroj))
{
if (this.Zdroj.ToLower().StartsWith("http"))
{
zdroj = string.Format(" <a target='_blank' href='{0}'>{1}</a>", this.Zdroj, "<span class='text-muted' title='Jedná se o peněžní nebo nepeněžní dar' alt='Jedná se o peněžní nebo nepeněžní dar'>(<span class='glyphicon glyphicon-link' aria-hidden='true'></span> zdroj</span>)");
}
else
{
zdroj = string.Format(" ({0})", this.Zdroj);
}
}
StringBuilder sb = new StringBuilder();
switch ((Types)this.Type)
{
case Types.Sponzor:
return(Title + " v " + this.RenderDatum() + (AddInfoNum.HasValue ? ", " + Smlouva.NicePrice(AddInfoNum) : "") + zdroj);
case Types.SponzorZuctu:
return(""); // Title + " v " + this.RenderDatum() + (AddInfoNum.HasValue ? ", " + Smlouva.NicePrice(AddInfoNum) : "") + " (z transp.účtu)" + zdroj;
case Types.OsobniVztah:
if (!string.IsNullOrEmpty(AddInfo) && Devmasters.Core.TextUtil.IsNumeric(AddInfo))
{
Osoba o = Osoby.GetById.Get(Convert.ToInt32(AddInfo));
if (o != null)
{
return(this.Title + " s " + string.Format("<a href=\"{0}\">{1}</a>", o.GetUrlOnWebsite(), o.FullName()) + zdroj);
}
else
{
return(this.Title + " " + Description + zdroj);
}
}
else
{
return(this.Title + " " + Description + zdroj);
}
case Types.Popis:
default:
if (!string.IsNullOrEmpty(this.Title) && !string.IsNullOrEmpty(this.Description))
{
return(this.Title + delimeter + this.Description + zdroj);
}
else if (!string.IsNullOrEmpty(this.Title))
{
return(this.Title + zdroj);
}
else if (!string.IsNullOrEmpty(this.Description))
{
return(this.Description + zdroj);
}
else
{
return(string.Empty);
}
}
}
public string RenderText(string delimeter = "\n")
{
StringBuilder sb = new StringBuilder();
switch ((Types)this.Type)
{
case Types.Sponzor:
return(Title + " v " + this.RenderDatum() + (AddInfoNum.HasValue ? ", " + Smlouva.NicePrice(AddInfoNum) : ""));
case Types.SponzorZuctu:
return(""); // Title + " v " + this.RenderDatum() + (AddInfoNum.HasValue ? ", " + Smlouva.NicePrice(AddInfoNum) : "") + " (z transp.účtu)";
case Types.OsobniVztah:
if (!string.IsNullOrEmpty(AddInfo) && Devmasters.Core.TextUtil.IsNumeric(AddInfo))
{
Osoba o = Osoby.GetById.Get(Convert.ToInt32(AddInfo));
if (o != null)
{
return(this.Title + " s " + o.FullName());
}
else
{
return(this.Title + " " + Description);
}
}
else
{
return(this.Title + " " + Description);
}
case Types.Popis:
default:
if (!string.IsNullOrEmpty(this.Title) && !string.IsNullOrEmpty(this.Description))
{
return(this.Title + delimeter + this.Description);
}
else if (!string.IsNullOrEmpty(this.Title))
{
return(this.Title);
}
else if (!string.IsNullOrEmpty(this.Description))
{
return(this.Description);
}
else
{
return(string.Empty);
}
}
}
public static Dictionary <Lib.Data.Smlouva.SClassification.ClassificationsTypes, decimal> GetClassificationFromServer(Smlouva s, bool rewriteStems = false)
{
Dictionary <Lib.Data.Smlouva.SClassification.ClassificationsTypes, decimal> data = new Dictionary <Smlouva.SClassification.ClassificationsTypes, decimal>();
if (s.Prilohy.All(m => m.EnoughExtractedText) == false)
{
return(null);
}
var settings = new JsonSerializerSettings();
settings.ContractResolver = new HlidacStatu.Util.FirstCaseLowercaseContractResolver();
var stems = GetRawStems(s, rewriteStems);
if (string.IsNullOrEmpty(stems))
{
return(data);
}
using (Devmasters.Net.Web.URLContent classif = new Devmasters.Net.Web.URLContent(classificationBaseUrl() + "/classifier"))
{
classif.Method = Devmasters.Net.Web.MethodEnum.POST;
classif.Tries = 3;
classif.TimeInMsBetweenTries = 5000;
classif.Timeout = 180000;
classif.ContentType = "application/json; charset=utf-8";
classif.RequestParams.RawContent = stems;
Devmasters.Net.Web.TextContentResult classifier = null;
try
{
Util.Consts.Logger.Debug("Getting classification from " + classif.Url);
classifier = classif.GetContent();
}
catch (Exception e)
{
Util.Consts.Logger.Error("Classification Classifier API " + classif.Url, e);
throw;
}
using (Devmasters.Net.Web.URLContent fin = new Devmasters.Net.Web.URLContent(classificationBaseUrl() + "/finalizer"))
{
fin.Method = Devmasters.Net.Web.MethodEnum.POST;
fin.Tries = 3;
fin.TimeInMsBetweenTries = 5000;
fin.Timeout = 30000;
fin.ContentType = "application/json; charset=utf-8";
fin.RequestParams.RawContent = classifier.Text;
Devmasters.Net.Web.TextContentResult res = null;
try
{
Util.Consts.Logger.Debug("Getting classification finalizer from " + classif.Url);
res = fin.GetContent();
}
catch (Exception e)
{
Util.Consts.Logger.Error("Classification finalizer API " + classif.Url, e);
throw;
}
var jsonData = Newtonsoft.Json.Linq.JObject.Parse(res.Text);
foreach (JProperty item in jsonData.Children())
{
string key = item.Name.Replace("-", "_");// jsonData[i][0].Value<string>().Replace("-", "_");
decimal prob = HlidacStatu.Util.ParseTools.ToDecimal(item.Value.ToString()) ?? 0;
if (Enum.TryParse <Smlouva.SClassification.ClassificationsTypes>(key, out var typ))
{
if (!data.ContainsKey(typ))
{
data.Add(typ, prob);
}
else if (typ == SClassification.ClassificationsTypes.OSTATNI)
{
Util.Consts.Logger.Warning($"Classification type lookup failure : { key }");
}
}
else
{
Util.Consts.Logger.Warning("Classification type lookup failure - Invalid key " + key);
data.Add(Smlouva.SClassification.ClassificationsTypes.OSTATNI, prob);
}
}
}
}
return(data);
}
public Smlouva[] GetPodobneSmlouvy(IEnumerable <QueryContainer> mandatory, IEnumerable <QueryContainer> optional = null, IEnumerable <string> exceptIds = null, int numOfResults = 50)
{
optional = optional ?? new QueryContainer[] { };
exceptIds = exceptIds ?? new string[] { };
Smlouva[] _result = null;
int tryNum = optional.Count();
while (_podobneSmlouvy == null && tryNum >= 0)
{
var query = mandatory.Concat(optional.Take(tryNum)).ToArray();
tryNum--;
var tmpResult = new List <Smlouva>();
var res = ES.SearchTools.RawSearch(
new QueryContainerDescriptor <Lib.Data.Smlouva>().Bool(b => b.Must(query)),
1, numOfResults, ES.SearchTools.OrderResult.DateAddedDesc, null
);
var resN = ES.SearchTools.RawSearch(
new QueryContainerDescriptor <Lib.Data.Smlouva>().Bool(b => b.Must(query)),
1, numOfResults, ES.SearchTools.OrderResult.DateAddedDesc, null, platnyZaznam: 0
);
if (res.IsValid == false)
{
HlidacStatu.Lib.ES.Manager.LogQueryError <Smlouva>(res);
}
else
{
tmpResult.AddRange(res.Hits.Select(m => m.Source).Where(m => m.Id != this.Id));
}
if (resN.IsValid == false)
{
HlidacStatu.Lib.ES.Manager.LogQueryError <Smlouva>(resN);
}
else
{
tmpResult.AddRange(resN.Hits.Select(m => m.Source).Where(m => m.Id != this.Id));
}
if (tmpResult.Count > 0)
{
var resSml = tmpResult.Where(m =>
m.Id != this.Id &&
!exceptIds.Any(id => id == m.Id)
).ToArray();
if (resSml.Length > 0)
{
_result = resSml;
}
}
}
;
if (_result == null)
{
_result = new Smlouva[] { }
}
; //not found anything
return(_result.Take(numOfResults).ToArray());
}
public string RenderText(string delimeter = "\n")
{
StringBuilder sb = new StringBuilder();
switch ((Types)this.Type)
{
case Types.ClenStrany:
sb.AppendFormat("Člen strany {1} {0} ", this.RenderDatum(), this.AddInfo);
return(sb.ToString());
case Types.Poslanec:
sb.AppendFormat("Poslanec {0} ", this.RenderDatum());
if (!string.IsNullOrEmpty(this.AddInfo))
{
sb.Append(" za " + AddInfo);
}
return(sb.ToString());
case Types.Senator:
sb.AppendFormat("Senator {0} ", this.RenderDatum());
if (!string.IsNullOrEmpty(this.AddInfo))
{
sb.Append(" za " + AddInfo);
}
return(sb.ToString());
case Types.Sponzor:
return(Title + " v " + this.RenderDatum() + (AddInfoNum.HasValue ? ", hodnota daru " + Smlouva.NicePrice(AddInfoNum) : ""));
case Types.OsobniVztah:
case Types.Pribuzny:
if (!string.IsNullOrEmpty(AddInfo) && Devmasters.Core.TextUtil.IsNumeric(AddInfo))
{
Osoba o = Osoby.GetById.Get(Convert.ToInt32(AddInfo));
if (o != null)
{
return(this.Title + " s " + o.FullName());
}
else
{
return(this.Title + " " + Description);
}
}
else
{
return(this.Title + " " + Description);
}
case Types.Popis:
default:
if (!string.IsNullOrEmpty(this.Title) && !string.IsNullOrEmpty(this.Description))
{
return(this.Title + delimeter + this.Description);
}
else if (!string.IsNullOrEmpty(this.Title))
{
return(this.Title);
}
else if (!string.IsNullOrEmpty(this.Description))
{
return(this.Description);
}
else
{
return(string.Empty);
}
}
}
