Beispiel #1
0
            /// <summary>
            /// Gets classification
            /// </summary>
            /// <param name="idSmlouvy"></param>
            /// <returns>Classification json</returns>
            public static string GetClassificationExplanation(string idSmlouvy)
            {
                if (string.IsNullOrWhiteSpace(idSmlouvy))
                {
                    return(null);
                }

                Smlouva s = Smlouva.Load(idSmlouvy);

                if (s == null)
                {
                    return(null);
                }

                var settings = new JsonSerializerSettings();

                settings.ContractResolver = new Util.FirstCaseLowercaseContractResolver();

                var response = CallEndpoint("explain_json",
                                            JsonConvert.SerializeObject(s, settings),
                                            idSmlouvy,
                                            1000 * 60 * 10);

                return(System.Text.RegularExpressions.Regex.Unescape(response));
            }
Beispiel #2
0
        public static Smlouva PrepareForDump(Smlouva s)
        {
            if (s.znepristupnenaSmlouva() && s.Prilohy != null)
            {
                foreach (var p in s.Prilohy)
                {
                    p.PlainTextContent = "-- anonymizovano serverem HlidacStatu.cz --";
                    p.odkaz            = "";
                }
            }

            if (s.Prilohy != null)
            {
                foreach (var p in s.Prilohy)
                {
                    p.DatlClassification = null;
                    p.FileMetadata       = null;
                }
            }
            s.SVazbouNaPolitiky         = null;
            s.SVazbouNaPolitikyAktualni = null;
            s.SVazbouNaPolitikyNedavne  = null;

            return(s);
        }
Beispiel #3
0
                                                   TimeSpan.FromDays(365 * 10)); //10 years

            private static byte[] getRawStemsFromServer(KeyAndId smlouvaKeyId)
            {
                Smlouva s = Smlouva.Load(smlouvaKeyId.ValueForData);

                if (s == null)
                {
                    return(null);
                }

                var settings = new JsonSerializerSettings();

                settings.ContractResolver = new HlidacStatu.Util.FirstCaseLowercaseContractResolver();

                string stemmerResponse = CallEndpoint("stemmer",
                                                      JsonConvert.SerializeObject(s, settings),
                                                      s.Id,
                                                      1000 * 60 * 30);

                try
                {
                    var jtoken = JToken.Parse(stemmerResponse);
                }
                catch (JsonReaderException e)
                {
                    Util.Consts.Logger.Error($"Stemmer returned incomplete json for {smlouvaKeyId.ValueForData}", e);
                    throw;
                }

                return(Encoding.UTF8.GetBytes(stemmerResponse));
            }
                                                   TimeSpan.FromDays(365 * 10)); //10 years

            private static byte[] getRawStemsFromServer(KeyAndId smlouvaKeyId)
            {
                Smlouva s = Smlouva.Load(smlouvaKeyId.ValueForData);

                if (s == null)
                {
                    return(null);
                }

                var settings = new JsonSerializerSettings();

                settings.ContractResolver = new HlidacStatu.Util.FirstCaseLowercaseContractResolver();


                using (Devmasters.Net.Web.URLContent stem = new Devmasters.Net.Web.URLContent(classificationBaseUrl() + "/stemmer"))
                {
                    stem.Method = Devmasters.Net.Web.MethodEnum.POST;
                    stem.Tries  = 3;
                    stem.TimeInMsBetweenTries = 5000;
                    stem.Timeout     = 1000 * 60 * 30; //30min
                    stem.ContentType = "application/json; charset=utf-8";
                    stem.RequestParams.RawContent = Newtonsoft.Json.JsonConvert.SerializeObject(s, settings);
                    Devmasters.Net.Web.BinaryContentResult stems = null;
                    try
                    {
                        Util.Consts.Logger.Debug("Getting stems from " + stem.Url);

                        stems = stem.GetBinary();
                        return(stems.Binary);
                    }
                    catch (Exception e)
                    {
                        Util.Consts.Logger.Error("Classification Stemmer API error " + stem.Url, e);
                        throw;
                    }
                }
            }
Beispiel #5
0
            public static string GetRawStems(Smlouva s, bool rewriteStems = false)
            {
                if (s == null)
                {
                    return(null);
                }
                var key = new KeyAndId()
                {
                    ValueForData = s.Id, CacheNameOnDisk = $"stem_smlouva_{s.Id}"
                };

                if (rewriteStems)
                {
                    InvalidateStemCache(s.Id);
                }
                var data = stemCacheManager.Get(key);

                if (data == null)
                {
                    return(null);
                }

                return(System.Text.Encoding.UTF8.GetString(data));
            }
Beispiel #6
0
            public static Dictionary <ClassificationsTypes, decimal> GetClassificationFromServer(Smlouva s, bool rewriteStems = false)
            {
                Dictionary <ClassificationsTypes, decimal> data = new Dictionary <ClassificationsTypes, decimal>();

                if (s.Prilohy.Any(m => m.EnoughExtractedText) == false)
                {
                    return(null);
                }

                var settings = new JsonSerializerSettings();

                settings.ContractResolver = new HlidacStatu.Util.FirstCaseLowercaseContractResolver();

                var stems = GetRawStems(s, rewriteStems);

                if (string.IsNullOrEmpty(stems))
                {
                    return(data);
                }

                string classifierResponse = "";

                try
                {
                    classifierResponse = CallEndpoint("classifier",
                                                      stems,
                                                      s.Id,
                                                      180000);
                }
                catch
                {
                    //retry once with new stems
                    stems = GetRawStems(s, true);
                    classifierResponse = CallEndpoint("classifier",
                                                      stems,
                                                      s.Id,
                                                      180000);
                }

                //string finalizerResponse = CallEndpoint("finalizer",
                //        classifierResponse,
                //        s.Id,
                //        30000);

                var jsonData = Newtonsoft.Json.Linq.JObject.Parse(classifierResponse);

                if (jsonData.Children().Count() == 0)
                {
                    data.Add(ClassificationsTypes.OSTATNI, 0.6M);
                    return(data);
                }

                foreach (JProperty item in jsonData.Children())
                {
                    string  key  = item.Name.Replace("-", "_").Replace("_generic", "_obecne");// jsonData[i][0].Value<string>().Replace("-", "_");
                    decimal prob = HlidacStatu.Util.ParseTools.ToDecimal(item.Value.ToString()) ?? 0;
                    if (prob > 0)
                    {
                        if (Enum.TryParse <Smlouva.SClassification.ClassificationsTypes>(key, out var typ))
                        {
                            if (!data.ContainsKey(typ))
                            {
                                data.Add(typ, prob);
                            }
                            else if (typ == SClassification.ClassificationsTypes.OSTATNI)
                            {
                                Util.Consts.Logger.Warning($"Classification type lookup failure : { key }");
                            }
                        }
                        else
                        {
                            Util.Consts.Logger.Warning("Classification type lookup failure - Invalid key " + key);
                            if (!data.ContainsKey(Smlouva.SClassification.ClassificationsTypes.OSTATNI))
                            {
                                data.Add(Smlouva.SClassification.ClassificationsTypes.OSTATNI, prob);
                            }
                        }
                    }
                }

                return(data);
            }
Beispiel #7
0
        private static string PrintFlatRelations(Graph.Edge parent, int level, IEnumerable <Graph.Edge> relations, TiskEnum typ,
                                                 List <string> renderedIds, bool withStats = true, string highlightSubjId = null)
        {
            int    space     = 2;
            string horizLine = "--"; //new string('\u2500',2);
            string vertLine  = "|";  //new string('\u2502',1);
            string cross     = "+";  //new string('\u251C', 1);

            if (renderedIds == null)
            {
                renderedIds = new List <string>();
            }

            var rels = relations
                       .Where(m =>
                              (
                                  (parent != null && m.From?.UniqId == parent.To?.UniqId)
                                  ||
                                  (parent == null && !relations.Any(r => r.To?.UniqId == m.From?.UniqId)) //do root urovne pridej vazby, ktere jsou sirotci bez parenta
                              )
                              )
                       .Distinct()
                       .GroupBy(k => new { id = k.To.UniqId, type = k.To.Type }, (k, v) =>
            {
                Graph.Edge withChildren = v.Where(r => relations.Any(m => m.From?.UniqId == r.To?.UniqId)).FirstOrDefault();
                if (withChildren == null)
                {
                    withChildren = v.First();
                }

                return(withChildren);
            })
                       .OrderBy(m => m.To.PrintName(typ == TiskEnum.Html))
                       .ToArray();

            if (rels.Count() == 0)
            {
                return(string.Empty);
            }

            StringBuilder     sb       = new StringBuilder(512);
            List <Graph.Edge> deepRels = new List <Graph.Edge>();

            switch (typ)
            {
            case TiskEnum.Text:
                break;

            case TiskEnum.Html:
            case TiskEnum.Checkbox:
                sb.AppendLine("<ul>");
                break;

            case TiskEnum.Json:
                break;
            }
            for (int i = 0; i < rels.Count(); i++)
            {
                var rel = rels[i];
                if (renderedIds.Contains(rel.To.UniqId))
                {
                    continue;
                }

                var last = i == (rels.Count() - 1);
                Analysis.SubjectStatistic stat = null;
                if (withStats && rel.To.Type == Graph.Node.NodeType.Company)
                {
                    stat = new Analysis.SubjectStatistic(rel.To.Id);
                }
                string subjId   = rel.To.Type == Graph.Node.NodeType.Company ? rel.To.Id : "Osoba";
                string subjName = rel.To.PrintName();
                renderedIds.Add(rel.To.UniqId);
                switch (typ)
                {
                case TiskEnum.Text:
                    sb.AppendLine(string.Concat(Enumerable.Repeat(vertLine + new string(' ', space), level + 1)));
                    sb.Append(
                        string.Concat(
                            Enumerable.Repeat(
                                vertLine + new string(' ', space)
                                , (level))
                            )
                        );
                    if (rel.To.Highlighted)
                    {
                        subjName = string.Format("!!{0}!!", subjName);
                    }

                    sb.AppendFormat("{0}{1}:{2} {3}\n",
                                    cross + horizLine + " ",
                                    subjId,
                                    subjName,
                                    rel.Doba()
                                    );
                    sb.Append(PrintFlatRelations(rel, level + 1, relations, typ, renderedIds, withStats));
                    break;

                case TiskEnum.Html:
                    if (withStats && stat != null)
                    {
                        sb.AppendFormat("<li class='{3} {6}'><a href='/subjekt/{0}'>{0}:{1}</a>{7}; {4}, celkem {5}.   {2}</li>",
                                        subjId,
                                        subjName,
                                        PrintFlatRelations(rel, level + 1, relations, typ, renderedIds, withStats),
                                        last ? "" : "connect",
                                        PluralForm.Get((int)stat.BasicStatPerYear.Summary.Pocet, "{0} smlouva;{0} smlouvy;{0} smluv"),
                                        Smlouva.NicePrice(stat.BasicStatPerYear.Summary.CelkemCena, html: true, shortFormat: true),
                                        "aktualnost" + ((int)rel.Aktualnost).ToString(),
                                        (rel.Aktualnost < AktualnostType.Aktualni) ? rel.Doba("/{0}/") : string.Empty
                                        );
                    }
                    else
                    {
                        sb.AppendFormat("<li class='{3} {4}'><a href='/subjekt/{0}'><span class=''>{0}:{1}</span></a>{5}.  {2}</li>",
                                        subjId,
                                        subjName,
                                        PrintFlatRelations(rel, level + 1, relations, typ, renderedIds, withStats),
                                        last ? "" : "connect",
                                        "aktualnost" + ((int)rel.Aktualnost).ToString(),
                                        (rel.Aktualnost < AktualnostType.Aktualni) ? rel.Doba("/{0}/") : string.Empty,
                                        (!string.IsNullOrEmpty(highlightSubjId) && subjId == highlightSubjId) ? "highlighted" : ""
                                        );
                    }

                    break;

                case TiskEnum.Checkbox:
                    sb.AppendFormat(@"<li class=""{0} {1}""><input type=""checkbox"" name=""ico"" value=""{2}"" /> <span><b>{2}</b> {3}</span>{4}</li>"
                                    , (last ? "" : "connect"),
                                    ("aktualnost" + ((int)rel.Aktualnost).ToString()),
                                    subjId, subjName,
                                    PrintFlatRelations(rel, level + 1, relations, typ, renderedIds, withStats)
                                    );

                    break;

                case TiskEnum.Json:
                    break;
                }
            }
            switch (typ)
            {
            case TiskEnum.Text:
                break;

            case TiskEnum.Html:
            case TiskEnum.Checkbox:
                sb.AppendLine("</ul>");
                break;

            case TiskEnum.Json:
                break;
            }

            return(sb.ToString());
        }
        public string RenderHtml(string delimeter = ", ")
        {
            string zdroj = "";

            if (!string.IsNullOrEmpty(this.Zdroj))
            {
                if (this.Zdroj.ToLower().StartsWith("http"))
                {
                    zdroj = string.Format(" <a target='_blank' href='{0}'>{1}</a>", this.Zdroj, "<span class='text-muted' title='Jedná se o peněžní nebo nepeněžní dar' alt='Jedná se o peněžní nebo nepeněžní dar'>(<span class='glyphicon glyphicon-link' aria-hidden='true'></span> zdroj</span>)");
                }
                else
                {
                    zdroj = string.Format(" ({0})", this.Zdroj);
                }
            }
            StringBuilder sb = new StringBuilder();

            switch ((Types)this.Type)
            {
            case Types.Sponzor:
                return(Title + " v " + this.RenderDatum() + (AddInfoNum.HasValue ? ", " + Smlouva.NicePrice(AddInfoNum) : "") + zdroj);

            case Types.SponzorZuctu:
                return("");    // Title + " v " + this.RenderDatum() + (AddInfoNum.HasValue ? ", " + Smlouva.NicePrice(AddInfoNum) : "") + " (z transp.účtu)" + zdroj;

            case Types.OsobniVztah:
                if (!string.IsNullOrEmpty(AddInfo) && Devmasters.Core.TextUtil.IsNumeric(AddInfo))
                {
                    Osoba o = Osoby.GetById.Get(Convert.ToInt32(AddInfo));
                    if (o != null)
                    {
                        return(this.Title + " s " + string.Format("<a href=\"{0}\">{1}</a>", o.GetUrlOnWebsite(), o.FullName()) + zdroj);
                    }
                    else
                    {
                        return(this.Title + " " + Description + zdroj);
                    }
                }
                else
                {
                    return(this.Title + " " + Description + zdroj);
                }

            case Types.Popis:
            default:
                if (!string.IsNullOrEmpty(this.Title) && !string.IsNullOrEmpty(this.Description))
                {
                    return(this.Title + delimeter + this.Description + zdroj);
                }
                else if (!string.IsNullOrEmpty(this.Title))
                {
                    return(this.Title + zdroj);
                }
                else if (!string.IsNullOrEmpty(this.Description))
                {
                    return(this.Description + zdroj);
                }
                else
                {
                    return(string.Empty);
                }
            }
        }
        public string RenderText(string delimeter = "\n")
        {
            StringBuilder sb = new StringBuilder();

            switch ((Types)this.Type)
            {
            case Types.Sponzor:
                return(Title + " v " + this.RenderDatum() + (AddInfoNum.HasValue ? ", " + Smlouva.NicePrice(AddInfoNum) : ""));

            case Types.SponzorZuctu:
                return("");    // Title + " v " + this.RenderDatum() + (AddInfoNum.HasValue ? ", " + Smlouva.NicePrice(AddInfoNum) : "") + " (z transp.účtu)";

            case Types.OsobniVztah:
                if (!string.IsNullOrEmpty(AddInfo) && Devmasters.Core.TextUtil.IsNumeric(AddInfo))
                {
                    Osoba o = Osoby.GetById.Get(Convert.ToInt32(AddInfo));
                    if (o != null)
                    {
                        return(this.Title + " s " + o.FullName());
                    }
                    else
                    {
                        return(this.Title + " " + Description);
                    }
                }
                else
                {
                    return(this.Title + " " + Description);
                }

            case Types.Popis:
            default:
                if (!string.IsNullOrEmpty(this.Title) && !string.IsNullOrEmpty(this.Description))
                {
                    return(this.Title + delimeter + this.Description);
                }
                else if (!string.IsNullOrEmpty(this.Title))
                {
                    return(this.Title);
                }
                else if (!string.IsNullOrEmpty(this.Description))
                {
                    return(this.Description);
                }
                else
                {
                    return(string.Empty);
                }
            }
        }
            public static Dictionary <Lib.Data.Smlouva.SClassification.ClassificationsTypes, decimal> GetClassificationFromServer(Smlouva s, bool rewriteStems = false)
            {
                Dictionary <Lib.Data.Smlouva.SClassification.ClassificationsTypes, decimal> data = new Dictionary <Smlouva.SClassification.ClassificationsTypes, decimal>();

                if (s.Prilohy.All(m => m.EnoughExtractedText) == false)
                {
                    return(null);
                }

                var settings = new JsonSerializerSettings();

                settings.ContractResolver = new HlidacStatu.Util.FirstCaseLowercaseContractResolver();

                var stems = GetRawStems(s, rewriteStems);

                if (string.IsNullOrEmpty(stems))
                {
                    return(data);
                }


                using (Devmasters.Net.Web.URLContent classif = new Devmasters.Net.Web.URLContent(classificationBaseUrl() + "/classifier"))
                {
                    classif.Method = Devmasters.Net.Web.MethodEnum.POST;
                    classif.Tries  = 3;
                    classif.TimeInMsBetweenTries = 5000;
                    classif.Timeout     = 180000;
                    classif.ContentType = "application/json; charset=utf-8";
                    classif.RequestParams.RawContent = stems;
                    Devmasters.Net.Web.TextContentResult classifier = null;
                    try
                    {
                        Util.Consts.Logger.Debug("Getting classification from " + classif.Url);
                        classifier = classif.GetContent();
                    }
                    catch (Exception e)
                    {
                        Util.Consts.Logger.Error("Classification Classifier API " + classif.Url, e);
                        throw;
                    }

                    using (Devmasters.Net.Web.URLContent fin = new Devmasters.Net.Web.URLContent(classificationBaseUrl() + "/finalizer"))
                    {
                        fin.Method = Devmasters.Net.Web.MethodEnum.POST;
                        fin.Tries  = 3;
                        fin.TimeInMsBetweenTries = 5000;
                        fin.Timeout     = 30000;
                        fin.ContentType = "application/json; charset=utf-8";
                        fin.RequestParams.RawContent = classifier.Text;
                        Devmasters.Net.Web.TextContentResult res = null;
                        try
                        {
                            Util.Consts.Logger.Debug("Getting classification finalizer from " + classif.Url);
                            res = fin.GetContent();
                        }
                        catch (Exception e)
                        {
                            Util.Consts.Logger.Error("Classification finalizer API " + classif.Url, e);
                            throw;
                        }


                        var jsonData = Newtonsoft.Json.Linq.JObject.Parse(res.Text);

                        foreach (JProperty item in jsonData.Children())
                        {
                            string  key  = item.Name.Replace("-", "_");// jsonData[i][0].Value<string>().Replace("-", "_");
                            decimal prob = HlidacStatu.Util.ParseTools.ToDecimal(item.Value.ToString()) ?? 0;
                            if (Enum.TryParse <Smlouva.SClassification.ClassificationsTypes>(key, out var typ))
                            {
                                if (!data.ContainsKey(typ))
                                {
                                    data.Add(typ, prob);
                                }
                                else if (typ == SClassification.ClassificationsTypes.OSTATNI)
                                {
                                    Util.Consts.Logger.Warning($"Classification type lookup failure : { key }");
                                }
                            }
                            else
                            {
                                Util.Consts.Logger.Warning("Classification type lookup failure - Invalid key " + key);
                                data.Add(Smlouva.SClassification.ClassificationsTypes.OSTATNI, prob);
                            }
                        }
                    }
                }
                return(data);
            }
Beispiel #11
0
        public Smlouva[] GetPodobneSmlouvy(IEnumerable <QueryContainer> mandatory, IEnumerable <QueryContainer> optional = null, IEnumerable <string> exceptIds = null, int numOfResults = 50)
        {
            optional  = optional ?? new QueryContainer[] { };
            exceptIds = exceptIds ?? new string[] { };
            Smlouva[] _result = null;

            int tryNum = optional.Count();

            while (_podobneSmlouvy == null && tryNum >= 0)
            {
                var query = mandatory.Concat(optional.Take(tryNum)).ToArray();
                tryNum--;

                var tmpResult = new List <Smlouva>();
                var res       = ES.SearchTools.RawSearch(
                    new QueryContainerDescriptor <Lib.Data.Smlouva>().Bool(b => b.Must(query)),
                    1, numOfResults, ES.SearchTools.OrderResult.DateAddedDesc, null
                    );
                var resN = ES.SearchTools.RawSearch(
                    new QueryContainerDescriptor <Lib.Data.Smlouva>().Bool(b => b.Must(query)),
                    1, numOfResults, ES.SearchTools.OrderResult.DateAddedDesc, null, platnyZaznam: 0
                    );

                if (res.IsValid == false)
                {
                    HlidacStatu.Lib.ES.Manager.LogQueryError <Smlouva>(res);
                }
                else
                {
                    tmpResult.AddRange(res.Hits.Select(m => m.Source).Where(m => m.Id != this.Id));
                }
                if (resN.IsValid == false)
                {
                    HlidacStatu.Lib.ES.Manager.LogQueryError <Smlouva>(resN);
                }
                else
                {
                    tmpResult.AddRange(resN.Hits.Select(m => m.Source).Where(m => m.Id != this.Id));
                }

                if (tmpResult.Count > 0)
                {
                    var resSml = tmpResult.Where(m =>
                                                 m.Id != this.Id &&
                                                 !exceptIds.Any(id => id == m.Id)
                                                 ).ToArray();
                    if (resSml.Length > 0)
                    {
                        _result = resSml;
                    }
                }
            }
            ;
            if (_result == null)
            {
                _result = new Smlouva[] { }
            }
            ;                                //not found anything

            return(_result.Take(numOfResults).ToArray());
        }
        public string RenderText(string delimeter = "\n")
        {
            StringBuilder sb = new StringBuilder();

            switch ((Types)this.Type)
            {
            case Types.ClenStrany:
                sb.AppendFormat("Člen strany {1} {0} ", this.RenderDatum(), this.AddInfo);
                return(sb.ToString());

            case Types.Poslanec:
                sb.AppendFormat("Poslanec {0} ", this.RenderDatum());
                if (!string.IsNullOrEmpty(this.AddInfo))
                {
                    sb.Append(" za " + AddInfo);
                }
                return(sb.ToString());

            case Types.Senator:
                sb.AppendFormat("Senator {0} ", this.RenderDatum());
                if (!string.IsNullOrEmpty(this.AddInfo))
                {
                    sb.Append(" za " + AddInfo);
                }
                return(sb.ToString());

            case Types.Sponzor:
                return(Title + " v " + this.RenderDatum() + (AddInfoNum.HasValue ? ", hodnota daru " + Smlouva.NicePrice(AddInfoNum)  : ""));

            case Types.OsobniVztah:
            case Types.Pribuzny:
                if (!string.IsNullOrEmpty(AddInfo) && Devmasters.Core.TextUtil.IsNumeric(AddInfo))
                {
                    Osoba o = Osoby.GetById.Get(Convert.ToInt32(AddInfo));
                    if (o != null)
                    {
                        return(this.Title + " s " + o.FullName());
                    }
                    else
                    {
                        return(this.Title + " " + Description);
                    }
                }
                else
                {
                    return(this.Title + " " + Description);
                }

            case Types.Popis:
            default:
                if (!string.IsNullOrEmpty(this.Title) && !string.IsNullOrEmpty(this.Description))
                {
                    return(this.Title + delimeter + this.Description);
                }
                else if (!string.IsNullOrEmpty(this.Title))
                {
                    return(this.Title);
                }
                else if (!string.IsNullOrEmpty(this.Description))
                {
                    return(this.Description);
                }
                else
                {
                    return(string.Empty);
                }
            }
        }