public static DataSearchResult SearchData(DataSet ds, string queryString, int page, int pageSize,
string sort = null, bool excludeBigProperties = true, bool withHighlighting = false,
bool exactNumOfResults = false)
{
Devmasters.DT.StopWatchEx sw = new Devmasters.DT.StopWatchEx();
sw.Start();
var query = Lib.Searching.Tools.FixInvalidQuery(queryString, queryShorcuts, queryOperators);
var res = _searchData(ds, query, page, pageSize, sort, excludeBigProperties, withHighlighting, exactNumOfResults);
sw.Stop();
if (!res.IsValid)
{
throw DataSetException.GetExc(
ds.DatasetId,
ApiResponseStatus.InvalidSearchQuery.error.number,
ApiResponseStatus.InvalidSearchQuery.error.description,
queryString
);
}
if (res.Total > 0)
{
var expConverter = new Newtonsoft.Json.Converters.ExpandoObjectConverter();
return(new DataSearchResult()
{
ElapsedTime = sw.Elapsed,
Q = queryString,
IsValid = true,
Total = res.Total,
Result = res.Hits
.Select(m => Newtonsoft.Json.JsonConvert.SerializeObject(m.Source))
.Select(s => (dynamic)Newtonsoft.Json.JsonConvert.DeserializeObject <System.Dynamic.ExpandoObject>(s, expConverter)),
Page = page,
PageSize = pageSize,
DataSet = ds,
ElasticResultsRaw = res,
});
}
else
{
return new DataSearchResult()
{
ElapsedTime = sw.Elapsed,
Q = queryString,
IsValid = true,
Total = 0,
Result = new dynamic[] { },
Page = page,
PageSize = pageSize,
DataSet = ds,
ElasticResultsRaw = res,
}
};
}
public static DataSearchResult SearchData(DataSet ds, string queryString, int page, int pageSize, string sort = null, bool excludeBigProperties = true)
{
Devmasters.Core.StopWatchEx sw = new Devmasters.Core.StopWatchEx();
sw.Start();
var query = Lib.ES.SearchTools.FixInvalidQuery(queryString, queryShorcuts, queryOperators);
var res = _searchData(ds, query, page, pageSize, sort, excludeBigProperties);
sw.Stop();
if (!res.IsValid)
{
throw DataSetException.GetExc(
ds.DatasetId,
ApiResponseStatus.InvalidSearchQuery.error.number,
ApiResponseStatus.InvalidSearchQuery.error.description,
queryString
);
}
if (res.Total > 0)
{
return new DataSearchResult()
{
ElapsedTime = sw.Elapsed,
Q = queryString,
IsValid = true,
Total = res.Total,
Result = res.Hits
.Select(m => m.Source.ToString())
.Select(s => (dynamic)Newtonsoft.Json.Linq.JObject.Parse(s)),
Page = page,
PageSize = pageSize,
DataSet = ds,
}
}
;
else
{
return new DataSearchResult()
{
ElapsedTime = sw.Elapsed,
Q = queryString,
IsValid = true,
Total = 0,
Result = new dynamic[] { },
Page = page,
PageSize = pageSize,
DataSet = ds,
}
};
}
public static DataSearchRawResult SearchDataRaw(DataSet ds, string queryString, int page, int pageSize,
string sort = null, bool excludeBigProperties = true, bool withHighlighting = false,
bool exactNumOfResults = false)
{
var query = Lib.Searching.Tools.FixInvalidQuery(queryString, queryShorcuts, queryOperators);
var res = _searchData(ds, query, page, pageSize, sort, excludeBigProperties, withHighlighting, exactNumOfResults);
if (!res.IsValid)
{
throw DataSetException.GetExc(ds.DatasetId,
ApiResponseStatus.InvalidSearchQuery.error.number,
ApiResponseStatus.InvalidSearchQuery.error.description,
queryString
);
}
if (res.Total > 0)
{
return new DataSearchRawResult()
{
Q = queryString,
IsValid = true,
Total = res.Total,
Result = res.Hits.Select(m => new Tuple <string, string>(m.Id, Newtonsoft.Json.JsonConvert.SerializeObject(m.Source))),
Page = page,
PageSize = pageSize,
DataSet = ds,
ElasticResultsRaw = res,
Order = sort ?? "0"
}
}
;
else
{
return new DataSearchRawResult()
{
Q = queryString,
IsValid = true,
Total = 0,
Result = new List <Tuple <string, string> >(),
ElasticResultsRaw = res,
Page = page,
PageSize = pageSize,
DataSet = ds,
Order = sort ?? "0"
}
};
}
public static DataSearchRawResult SearchDataRaw(DataSet ds, string queryString, int page, int pageSize, string sort = null, bool excludeBigProperties = true)
{
var query = Lib.ES.SearchTools.FixInvalidQuery(queryString, queryShorcuts, queryOperators);
var res = _searchData(ds, queryString, page, pageSize, sort, excludeBigProperties);
if (!res.IsValid)
{
throw DataSetException.GetExc(ds.DatasetId,
ApiResponseStatus.InvalidSearchQuery.error.number,
ApiResponseStatus.InvalidSearchQuery.error.description,
queryString
);
}
if (res.Total > 0)
{
return new DataSearchRawResult()
{
Q = queryString,
IsValid = true,
Total = res.Total,
Result = res.Hits.Select(m => new Tuple <string, string>(m.Id, m.Source.ToString())),
Page = page,
PageSize = pageSize,
DataSet = ds,
Order = sort ?? "0"
}
}
;
else
{
return new DataSearchRawResult()
{
Q = queryString,
IsValid = true,
Total = 0,
Result = new List <Tuple <string, string> >(),
Page = page,
PageSize = pageSize,
DataSet = ds,
Order = sort ?? "0"
}
};
}
public virtual string AddData(string data, string id, string createdBy, bool validateSchema = true, bool skipOCR = false)
{
Newtonsoft.Json.Linq.JObject obj = Newtonsoft.Json.Linq.JObject.Parse(data);
dynamic objDyn = Newtonsoft.Json.Linq.JObject.Parse(data);
if (validateSchema)
{
Newtonsoft.Json.Schema.JSchema schema = DataSetDB.Instance.GetRegistration(this.datasetId).GetSchema();
if (schema != null)
{
IList <string> errors;
if (!obj.IsValid(schema, out errors))
{
if (errors == null || errors?.Count == 0)
{
errors = new string[] { "", "" }
}
;
throw DataSetException.GetExc(this.datasetId,
ApiResponseStatus.DatasetItemInvalidFormat.error.number,
ApiResponseStatus.DatasetItemInvalidFormat.error.description,
errors.Aggregate((f, s) => f + ";" + s)
);
}
}
}
if (string.IsNullOrEmpty(id))
{
throw new DataSetException(this.datasetId, ApiResponseStatus.DatasetItemNoSetID);
}
if (objDyn.Id == null
&&
objDyn.id == null)
{
throw new DataSetException(this.datasetId, ApiResponseStatus.DatasetItemNoSetID);
}
else
{
id = objDyn.Id == null ? (string)objDyn.id : (string)objDyn.Id;
}
objDyn.DbCreated = DateTime.UtcNow;
objDyn.DbCreatedBy = createdBy;
//check special HsProcessType
var jobj = (Newtonsoft.Json.Linq.JObject)objDyn;
var jpaths = jobj
.SelectTokens("$..HsProcessType")
.ToArray();
var jpathObjs = jpaths.Select(j => j.Parent.Parent).ToArray();
if (this.DatasetId == DataSetDB.DataSourcesDbName) //don't analyze for registration of new dataset
{
jpathObjs = new JContainer[] { }
}
;
foreach (var jo in jpathObjs)
{
if (jo["HsProcessType"].Value <string>() == "person")
{
var jmenoAttrName = jo.Children()
.Select(c => c as JProperty)
.Where(c => c != null)
.Where(c => c.Name.ToLower() == "jmeno" ||
c.Name.ToLower() == "name")
.FirstOrDefault()?.Name;
var prijmeniAttrName = jo.Children()
.Select(c => c as JProperty)
.Where(c => c != null)
.Where(c => c.Name.ToLower() == "prijmeni" ||
c.Name.ToLower() == "surname")
.FirstOrDefault()?.Name;
var narozeniAttrName = jo.Children()
.Select(c => c as JProperty)
.Where(c => c != null)
.Where(c => c.Name.ToLower() == "narozeni" ||
c.Name.ToLower() == "birthdate")
.FirstOrDefault()?.Name;
var osobaIdAttrName = jo.Children()
.Select(c => c as JProperty)
.Where(c => c != null)
.Where(c => c.Name.ToLower() == "osobaid")
.FirstOrDefault()?.Name ?? "OsobaId";
var celejmenoAttrName = jo.Children()
.Select(c => c as JProperty)
.Where(c => c != null)
.Where(c => c.Name.ToLower() == "celejmeno" ||
c.Name.ToLower() == "fullname")
.FirstOrDefault()?.Name;
#region FindOsobaId
if (jmenoAttrName != null && prijmeniAttrName != null && narozeniAttrName != null)
{
if (string.IsNullOrEmpty(jo["OsobaId"]?.Value <string>()) &&
jo[narozeniAttrName] != null && jo[narozeniAttrName].Value <DateTime?>().HasValue
) //pokud OsobaId je vyplnena, nehledej jinou
{
string osobaId = null;
var osobaInDb = Osoba.GetByName(
jo[jmenoAttrName].Value <string>(),
jo[prijmeniAttrName].Value <string>(),
jo[narozeniAttrName].Value <DateTime>()
);
if (osobaInDb == null)
{
osobaInDb = Osoba.GetByNameAscii(
jo[jmenoAttrName].Value <string>(),
jo[prijmeniAttrName].Value <string>(),
jo[narozeniAttrName].Value <DateTime>()
);
}
if (osobaInDb != null && string.IsNullOrEmpty(osobaInDb.NameId))
{
osobaInDb.NameId = osobaInDb.GetUniqueNamedId();
osobaInDb.Save();
}
osobaId = osobaInDb?.NameId;
jo["OsobaId"] = osobaId;
}
}
else if (celejmenoAttrName != null && narozeniAttrName != null)
{
if (string.IsNullOrEmpty(jo["OsobaId"]?.Value <string>()) &&
jo[narozeniAttrName].Value <DateTime?>().HasValue
) //pokud OsobaId je vyplnena, nehledej jinou
{
string osobaId = null;
Lib.Data.Osoba osobaZeJmena = Lib.Validators.OsobaInText(jo[celejmenoAttrName].Value <string>());
if (osobaZeJmena != null)
{
var osobaInDb = Osoba.GetByName(
osobaZeJmena.Jmeno,
osobaZeJmena.Prijmeni,
jo[narozeniAttrName].Value <DateTime>()
);
if (osobaInDb == null)
{
osobaInDb = Osoba.GetByNameAscii(
osobaZeJmena.Jmeno,
osobaZeJmena.Prijmeni,
jo[narozeniAttrName].Value <DateTime>()
);
}
if (osobaInDb != null && string.IsNullOrEmpty(osobaInDb.NameId))
{
osobaInDb.NameId = osobaInDb.GetUniqueNamedId();
osobaInDb.Save();
}
osobaId = osobaInDb?.NameId;
}
jo["OsobaId"] = osobaId;
}
}
#endregion
}
}
string updatedData = Newtonsoft.Json.JsonConvert.SerializeObject(objDyn);
PostData pd = PostData.String(updatedData);
var tres = client.LowLevel.Index <StringResponse>(client.ConnectionSettings.DefaultIndex, id, pd); //todo: es7 check
if (tres.Success)
{
Newtonsoft.Json.Linq.JObject jobject = Newtonsoft.Json.Linq.JObject.Parse(tres.Body);
string finalId = jobject.Value <string>("_id");
//do DocumentMining after successfull save
//record must exists before document mining
bool needsOCR = false;
if (skipOCR == false)
{
foreach (var jo in jpathObjs)
{
if (jo["HsProcessType"].Value <string>() == "document")
{
if (jo["DocumentUrl"] != null && string.IsNullOrEmpty(jo["DocumentPlainText"].Value <string>()))
{
if (Uri.TryCreate(jo["DocumentUrl"].Value <string>(), UriKind.Absolute, out var uri2Ocr))
{
//get text from document
//var url = Devmasters.Core.Util.Config.GetConfigValue("ESConnection");
//url = url + $"/{client.ConnectionSettings.DefaultIndex}/data/{finalId}/_update";
//string callback = HlidacStatu.Lib.OCR.Api.CallbackData.PrepareElasticCallbackDataForOCRReq($"{jo.Path}.DocumentPlainText", false);
//var ocrCallBack = new HlidacStatu.Lib.OCR.Api.CallbackData(new Uri(url), callback, HlidacStatu.Lib.OCR.Api.CallbackData.CallbackType.LocalElastic);
//HlidacStatu.Lib.OCR.Api.Client.TextFromUrl(
// Devmasters.Core.Util.Config.GetConfigValue("OCRServerApiKey"),
// uri2Ocr, "Dataset+" + createdBy,
// HlidacStatu.Lib.OCR.Api.Client.TaskPriority.Standard, HlidacStatu.Lib.OCR.Api.Client.MiningIntensity.Maximum
// ); //TODOcallBackData: ocrCallBack);
needsOCR = true;
}
}
}
}
}
if (needsOCR)
{
Lib.Data.ItemToOcrQueue.AddNewTask(ItemToOcrQueue.ItemToOcrType.Dataset, finalId, this.datasetId, OCR.Api.Client.TaskPriority.Standard);
}
return(finalId);
}
else
{
var status = ApiResponseStatus.DatasetItemSaveError;
if (tres.TryGetServerError(out var servererr))
{
status.error.errorDetail = servererr.Error.ToString();
}
throw new DataSetException(this.datasetId, status);
}
//return res.ToString();
//ElasticsearchResponse<string> result = this.client.Raw.Index(document.Index, document.Type, document.Id, documentJson);
}
