示例#1
0
        public static DataSearchResult SearchData(DataSet ds, string queryString, int page, int pageSize,
                                                  string sort            = null, bool excludeBigProperties = true, bool withHighlighting = false,
                                                  bool exactNumOfResults = false)
        {
            Devmasters.DT.StopWatchEx sw = new Devmasters.DT.StopWatchEx();

            sw.Start();
            var query = Lib.Searching.Tools.FixInvalidQuery(queryString, queryShorcuts, queryOperators);

            var res = _searchData(ds, query, page, pageSize, sort, excludeBigProperties, withHighlighting, exactNumOfResults);

            sw.Stop();
            if (!res.IsValid)
            {
                throw DataSetException.GetExc(
                          ds.DatasetId,
                          ApiResponseStatus.InvalidSearchQuery.error.number,
                          ApiResponseStatus.InvalidSearchQuery.error.description,
                          queryString
                          );
            }

            if (res.Total > 0)
            {
                var expConverter = new Newtonsoft.Json.Converters.ExpandoObjectConverter();

                return(new DataSearchResult()
                {
                    ElapsedTime = sw.Elapsed,
                    Q = queryString,
                    IsValid = true,
                    Total = res.Total,
                    Result = res.Hits
                             .Select(m => Newtonsoft.Json.JsonConvert.SerializeObject(m.Source))
                             .Select(s => (dynamic)Newtonsoft.Json.JsonConvert.DeserializeObject <System.Dynamic.ExpandoObject>(s, expConverter)),

                    Page = page,
                    PageSize = pageSize,
                    DataSet = ds,
                    ElasticResultsRaw = res,
                });
            }
            else
            {
                return new DataSearchResult()
                       {
                           ElapsedTime       = sw.Elapsed,
                           Q                 = queryString,
                           IsValid           = true,
                           Total             = 0,
                           Result            = new dynamic[] { },
                           Page              = page,
                           PageSize          = pageSize,
                           DataSet           = ds,
                           ElasticResultsRaw = res,
                       }
            };
        }
示例#2
0
        public static DataSearchResult SearchData(DataSet ds, string queryString, int page, int pageSize, string sort = null, bool excludeBigProperties = true)
        {
            Devmasters.Core.StopWatchEx sw = new Devmasters.Core.StopWatchEx();

            sw.Start();
            var query = Lib.ES.SearchTools.FixInvalidQuery(queryString, queryShorcuts, queryOperators);

            var res = _searchData(ds, query, page, pageSize, sort, excludeBigProperties);

            sw.Stop();
            if (!res.IsValid)
            {
                throw DataSetException.GetExc(
                          ds.DatasetId,
                          ApiResponseStatus.InvalidSearchQuery.error.number,
                          ApiResponseStatus.InvalidSearchQuery.error.description,
                          queryString
                          );
            }

            if (res.Total > 0)
            {
                return new DataSearchResult()
                       {
                           ElapsedTime = sw.Elapsed,
                           Q           = queryString,
                           IsValid     = true,
                           Total       = res.Total,
                           Result      = res.Hits
                                         .Select(m => m.Source.ToString())
                                         .Select(s => (dynamic)Newtonsoft.Json.Linq.JObject.Parse(s)),
                           Page     = page,
                           PageSize = pageSize,
                           DataSet  = ds,
                       }
            }
            ;
            else
            {
                return new DataSearchResult()
                       {
                           ElapsedTime = sw.Elapsed,
                           Q           = queryString,
                           IsValid     = true,
                           Total       = 0,
                           Result      = new dynamic[] { },
                           Page        = page,
                           PageSize    = pageSize,
                           DataSet     = ds,
                       }
            };
        }
示例#3
0
        public static DataSearchRawResult SearchDataRaw(DataSet ds, string queryString, int page, int pageSize,
                                                        string sort            = null, bool excludeBigProperties = true, bool withHighlighting = false,
                                                        bool exactNumOfResults = false)
        {
            var query = Lib.Searching.Tools.FixInvalidQuery(queryString, queryShorcuts, queryOperators);
            var res   = _searchData(ds, query, page, pageSize, sort, excludeBigProperties, withHighlighting, exactNumOfResults);

            if (!res.IsValid)
            {
                throw DataSetException.GetExc(ds.DatasetId,
                                              ApiResponseStatus.InvalidSearchQuery.error.number,
                                              ApiResponseStatus.InvalidSearchQuery.error.description,
                                              queryString
                                              );
            }

            if (res.Total > 0)
            {
                return new DataSearchRawResult()
                       {
                           Q                 = queryString,
                           IsValid           = true,
                           Total             = res.Total,
                           Result            = res.Hits.Select(m => new Tuple <string, string>(m.Id, Newtonsoft.Json.JsonConvert.SerializeObject(m.Source))),
                           Page              = page,
                           PageSize          = pageSize,
                           DataSet           = ds,
                           ElasticResultsRaw = res,
                           Order             = sort ?? "0"
                       }
            }
            ;
            else
            {
                return new DataSearchRawResult()
                       {
                           Q                 = queryString,
                           IsValid           = true,
                           Total             = 0,
                           Result            = new List <Tuple <string, string> >(),
                           ElasticResultsRaw = res,
                           Page              = page,
                           PageSize          = pageSize,
                           DataSet           = ds,
                           Order             = sort ?? "0"
                       }
            };
        }
示例#4
0
        public static DataSearchRawResult SearchDataRaw(DataSet ds, string queryString, int page, int pageSize, string sort = null, bool excludeBigProperties = true)
        {
            var query = Lib.ES.SearchTools.FixInvalidQuery(queryString, queryShorcuts, queryOperators);
            var res   = _searchData(ds, queryString, page, pageSize, sort, excludeBigProperties);

            if (!res.IsValid)
            {
                throw DataSetException.GetExc(ds.DatasetId,
                                              ApiResponseStatus.InvalidSearchQuery.error.number,
                                              ApiResponseStatus.InvalidSearchQuery.error.description,
                                              queryString
                                              );
            }

            if (res.Total > 0)
            {
                return new DataSearchRawResult()
                       {
                           Q        = queryString,
                           IsValid  = true,
                           Total    = res.Total,
                           Result   = res.Hits.Select(m => new Tuple <string, string>(m.Id, m.Source.ToString())),
                           Page     = page,
                           PageSize = pageSize,
                           DataSet  = ds,
                           Order    = sort ?? "0"
                       }
            }
            ;
            else
            {
                return new DataSearchRawResult()
                       {
                           Q        = queryString,
                           IsValid  = true,
                           Total    = 0,
                           Result   = new List <Tuple <string, string> >(),
                           Page     = page,
                           PageSize = pageSize,
                           DataSet  = ds,
                           Order    = sort ?? "0"
                       }
            };
        }
示例#5
0
        public virtual string AddData(string data, string id, string createdBy, bool validateSchema = true, bool skipOCR = false)
        {
            Newtonsoft.Json.Linq.JObject obj = Newtonsoft.Json.Linq.JObject.Parse(data);
            dynamic objDyn = Newtonsoft.Json.Linq.JObject.Parse(data);

            if (validateSchema)
            {
                Newtonsoft.Json.Schema.JSchema schema = DataSetDB.Instance.GetRegistration(this.datasetId).GetSchema();

                if (schema != null)
                {
                    IList <string> errors;
                    if (!obj.IsValid(schema, out errors))
                    {
                        if (errors == null || errors?.Count == 0)
                        {
                            errors = new string[] { "", "" }
                        }
                        ;

                        throw DataSetException.GetExc(this.datasetId,
                                                      ApiResponseStatus.DatasetItemInvalidFormat.error.number,
                                                      ApiResponseStatus.DatasetItemInvalidFormat.error.description,
                                                      errors.Aggregate((f, s) => f + ";" + s)
                                                      );
                    }
                }
            }
            if (string.IsNullOrEmpty(id))
            {
                throw new DataSetException(this.datasetId, ApiResponseStatus.DatasetItemNoSetID);
            }

            if (objDyn.Id == null
                &&
                objDyn.id == null)
            {
                throw new DataSetException(this.datasetId, ApiResponseStatus.DatasetItemNoSetID);
            }
            else
            {
                id = objDyn.Id == null ? (string)objDyn.id : (string)objDyn.Id;
            }

            objDyn.DbCreated   = DateTime.UtcNow;
            objDyn.DbCreatedBy = createdBy;



            //check special HsProcessType
            var jobj   = (Newtonsoft.Json.Linq.JObject)objDyn;
            var jpaths = jobj
                         .SelectTokens("$..HsProcessType")
                         .ToArray();
            var jpathObjs = jpaths.Select(j => j.Parent.Parent).ToArray();

            if (this.DatasetId == DataSetDB.DataSourcesDbName) //don't analyze for registration of new dataset
            {
                jpathObjs = new JContainer[] { }
            }
            ;

            foreach (var jo in jpathObjs)
            {
                if (jo["HsProcessType"].Value <string>() == "person")
                {
                    var jmenoAttrName = jo.Children()
                                        .Select(c => c as JProperty)
                                        .Where(c => c != null)
                                        .Where(c => c.Name.ToLower() == "jmeno" ||
                                               c.Name.ToLower() == "name")
                                        .FirstOrDefault()?.Name;
                    var prijmeniAttrName = jo.Children()
                                           .Select(c => c as JProperty)
                                           .Where(c => c != null)
                                           .Where(c => c.Name.ToLower() == "prijmeni" ||
                                                  c.Name.ToLower() == "surname")
                                           .FirstOrDefault()?.Name;
                    var narozeniAttrName = jo.Children()
                                           .Select(c => c as JProperty)
                                           .Where(c => c != null)
                                           .Where(c => c.Name.ToLower() == "narozeni" ||
                                                  c.Name.ToLower() == "birthdate")
                                           .FirstOrDefault()?.Name;
                    var osobaIdAttrName = jo.Children()
                                          .Select(c => c as JProperty)
                                          .Where(c => c != null)
                                          .Where(c => c.Name.ToLower() == "osobaid")
                                          .FirstOrDefault()?.Name ?? "OsobaId";

                    var celejmenoAttrName = jo.Children()
                                            .Select(c => c as JProperty)
                                            .Where(c => c != null)
                                            .Where(c => c.Name.ToLower() == "celejmeno" ||
                                                   c.Name.ToLower() == "fullname")
                                            .FirstOrDefault()?.Name;


                    #region FindOsobaId
                    if (jmenoAttrName != null && prijmeniAttrName != null && narozeniAttrName != null)
                    {
                        if (string.IsNullOrEmpty(jo["OsobaId"]?.Value <string>()) &&
                            jo[narozeniAttrName] != null && jo[narozeniAttrName].Value <DateTime?>().HasValue
                            ) //pokud OsobaId je vyplnena, nehledej jinou
                        {
                            string osobaId   = null;
                            var    osobaInDb = Osoba.GetByName(
                                jo[jmenoAttrName].Value <string>(),
                                jo[prijmeniAttrName].Value <string>(),
                                jo[narozeniAttrName].Value <DateTime>()
                                );
                            if (osobaInDb == null)
                            {
                                osobaInDb = Osoba.GetByNameAscii(
                                    jo[jmenoAttrName].Value <string>(),
                                    jo[prijmeniAttrName].Value <string>(),
                                    jo[narozeniAttrName].Value <DateTime>()
                                    );
                            }

                            if (osobaInDb != null && string.IsNullOrEmpty(osobaInDb.NameId))
                            {
                                osobaInDb.NameId = osobaInDb.GetUniqueNamedId();
                                osobaInDb.Save();
                            }
                            osobaId       = osobaInDb?.NameId;
                            jo["OsobaId"] = osobaId;
                        }
                    }
                    else if (celejmenoAttrName != null && narozeniAttrName != null)
                    {
                        if (string.IsNullOrEmpty(jo["OsobaId"]?.Value <string>()) &&
                            jo[narozeniAttrName].Value <DateTime?>().HasValue
                            ) //pokud OsobaId je vyplnena, nehledej jinou
                        {
                            string         osobaId      = null;
                            Lib.Data.Osoba osobaZeJmena = Lib.Validators.OsobaInText(jo[celejmenoAttrName].Value <string>());
                            if (osobaZeJmena != null)
                            {
                                var osobaInDb = Osoba.GetByName(
                                    osobaZeJmena.Jmeno,
                                    osobaZeJmena.Prijmeni,
                                    jo[narozeniAttrName].Value <DateTime>()
                                    );

                                if (osobaInDb == null)
                                {
                                    osobaInDb = Osoba.GetByNameAscii(
                                        osobaZeJmena.Jmeno,
                                        osobaZeJmena.Prijmeni,
                                        jo[narozeniAttrName].Value <DateTime>()
                                        );
                                }

                                if (osobaInDb != null && string.IsNullOrEmpty(osobaInDb.NameId))
                                {
                                    osobaInDb.NameId = osobaInDb.GetUniqueNamedId();
                                    osobaInDb.Save();
                                }
                                osobaId = osobaInDb?.NameId;
                            }
                            jo["OsobaId"] = osobaId;
                        }
                    }

                    #endregion
                }
            }


            string   updatedData = Newtonsoft.Json.JsonConvert.SerializeObject(objDyn);
            PostData pd          = PostData.String(updatedData);

            var tres = client.LowLevel.Index <StringResponse>(client.ConnectionSettings.DefaultIndex, id, pd); //todo: es7 check

            if (tres.Success)
            {
                Newtonsoft.Json.Linq.JObject jobject = Newtonsoft.Json.Linq.JObject.Parse(tres.Body);

                string finalId = jobject.Value <string>("_id");

                //do DocumentMining after successfull save
                //record must exists before document mining
                bool needsOCR = false;
                if (skipOCR == false)
                {
                    foreach (var jo in jpathObjs)
                    {
                        if (jo["HsProcessType"].Value <string>() == "document")
                        {
                            if (jo["DocumentUrl"] != null && string.IsNullOrEmpty(jo["DocumentPlainText"].Value <string>()))
                            {
                                if (Uri.TryCreate(jo["DocumentUrl"].Value <string>(), UriKind.Absolute, out var uri2Ocr))
                                {
                                    //get text from document
                                    //var url = Devmasters.Core.Util.Config.GetConfigValue("ESConnection");
                                    //url = url + $"/{client.ConnectionSettings.DefaultIndex}/data/{finalId}/_update";
                                    //string callback = HlidacStatu.Lib.OCR.Api.CallbackData.PrepareElasticCallbackDataForOCRReq($"{jo.Path}.DocumentPlainText", false);
                                    //var ocrCallBack = new HlidacStatu.Lib.OCR.Api.CallbackData(new Uri(url), callback, HlidacStatu.Lib.OCR.Api.CallbackData.CallbackType.LocalElastic);
                                    //HlidacStatu.Lib.OCR.Api.Client.TextFromUrl(
                                    //    Devmasters.Core.Util.Config.GetConfigValue("OCRServerApiKey"),
                                    //    uri2Ocr, "Dataset+" + createdBy,
                                    //    HlidacStatu.Lib.OCR.Api.Client.TaskPriority.Standard, HlidacStatu.Lib.OCR.Api.Client.MiningIntensity.Maximum
                                    //    ); //TODOcallBackData: ocrCallBack);

                                    needsOCR = true;
                                }
                            }
                        }
                    }
                }
                if (needsOCR)
                {
                    Lib.Data.ItemToOcrQueue.AddNewTask(ItemToOcrQueue.ItemToOcrType.Dataset, finalId, this.datasetId, OCR.Api.Client.TaskPriority.Standard);
                }

                return(finalId);
            }
            else
            {
                var status = ApiResponseStatus.DatasetItemSaveError;
                if (tres.TryGetServerError(out var servererr))
                {
                    status.error.errorDetail = servererr.Error.ToString();
                }
                throw new DataSetException(this.datasetId, status);
            }

            //return res.ToString();
            //ElasticsearchResponse<string> result = this.client.Raw.Index(document.Index, document.Type, document.Id, documentJson);
        }