public void Index(string processId, PrcSettingsElastic prcSettings, CancellationToken token)
{
var logPrefix = $"Prc Index {processId}";
try
{
const int parallelMultiplier = 2;
var service = serviceQuery.Get(prcSettings.ServiceId);
redisHandler.Clean(PrcIndexRedisKey.ServiceDeleteKey(prcSettings.ServiceId));
var wordQuery = queryFactory.GetWordQuery(prcSettings.DataSetName);
var lockObject = new object();
var documentQuery = queryFactory.GetDocumentQuery(prcSettings.DataSetName);
var tagsWithDocumentCounts = documentQuery.CountForTags(prcSettings.IndexSettings.FilterTagIdList, GlobalStore.DataSets.Get(prcSettings.DataSetName).DataSet.TagField);
var allDocumentsCount = tagsWithDocumentCounts.Sum(d => d.Value);
logger.LogInformation($"{logPrefix} starts with ParallelLimit: {parallelService.ParallelLimit * parallelMultiplier}, Tags Count: {prcSettings.IndexSettings.FilterTagIdList.Count}, All documents count: {allDocumentsCount}");
var allDocProgress = new Progress(allDocumentsCount);
// lekérjük az aktuális tag-hez(és Filter - hez) tartozó doksikat (elastic - ból).
foreach (var tagId in prcSettings.IndexSettings.FilterTagIdList)
{
if (token.IsCancellationRequested)
{
CleanPrcIndex(prcSettings.ServiceId);
processHandler.Cancelled(processId);
logger.LogInformation($"{logPrefix} cancelled Tag: `{tagId}`");
return;
}
logger.LogTrace($"{logPrefix} preparing Tag: `{tagId}`");
var globalSubset = GlobalStore.ActivatedPrcs.Get(prcSettings.ServiceId).PrcSubsets[tagId];
if (globalSubset.WordsWithOccurences == null)
{
continue;
}
var documentElasticIds = GetDocumentIds(prcSettings.DataSetName, prcSettings.IndexSettings.FilterQuery, new List <string> {
tagId
}, null, prcSettings.IndexSettings.IndexDate).OrderBy(o => o).ToList();
if (documentElasticIds.Count == 0)
{
continue;
}
var wwoDocuments = wordQuery.GetWordsWithOccurencesByDocuments(
documentElasticIds,
prcSettings.FieldsForRecommendation.Select(DocumentQuery.MapDocumentObjectName),
1, parallelLimit: parallelService.ParallelLimit);
var cleanedTextDocuments = wwoDocuments.ToDictionary(w => w.Key, w => GetCleanedText(w.Value));
var docProgress = new Progress(documentElasticIds.Count);
Parallel.ForEach(documentElasticIds, parallelService.ParallelOptions(parallelMultiplier), (documentId, loopState) =>
{
if (token.IsCancellationRequested)
{
loopState.Stop();
return;
}
try
{
logger.LogTrace($"{logPrefix} preparing Document: `{documentId}`/`{tagId}`");
// kiszámoljuk az aktuális doksi base dictionary - jét
var scorer = GetScorer(globalSubset, wwoDocuments[documentId], cleanedTextDocuments[documentId], GlobalStore.ActivatedPrcs.Get(prcSettings.ServiceId).PrcScorers[tagId]);
if (scorer == null)
{
return;
}
var similarDocuments = new List <KeyValuePair <string, double> >();
// documentElastics except document
foreach (var siblingDocumentId in documentElasticIds)
{
if (siblingDocumentId == documentId)
{
continue;
}
var wwoSibling = wwoDocuments[siblingDocumentId];
if (wwoSibling.Keys.Intersect(scorer.BaseDic.Keys).Count() == 0)
{
continue;
}
var finalScore = GetPrcScore(scorer, cleanedTextDocuments[siblingDocumentId]);
if (finalScore > 0)
{
similarDocuments.Add(new KeyValuePair <string, double>(siblingDocumentId, finalScore));
}
}
var redisKey = new PrcIndexRedisKey(prcSettings.ServiceId, tagId, documentId);
redisHandler.AddDocuments(redisKey, similarDocuments);
}
finally
{
logger.LogTrace($"{logPrefix} prepared Document: `{documentId}`/`{tagId}`");
allDocProgress.Step();
var value = docProgress.Step();
if (value % 50 == 0)
{
lock (lockObject)
{
processHandler.Changed(processId, allDocProgress.Percent.Round(6));
}
logger.LogTrace($"{logPrefix} progress {docProgress} in `{tagId}`");
logger.LogTrace($"{logPrefix} total progress is {allDocProgress}");
}
if (value % 1000 == 0)
{
GC.Collect();
}
}
});
if (token.IsCancellationRequested)
{
CleanPrcIndex(prcSettings.ServiceId);
processHandler.Cancelled(processId);
logger.LogInformation($"{logPrefix} cancelled Tag: `{tagId}`");
return;
}
logger.LogInformation($"{logPrefix} prepared Tag: `{tagId}`");
logger.LogInformation($"{logPrefix} total progress is {allDocProgress}");
GC.Collect();
}
processHandler.Finished(processId, string.Format(ServiceResources.SuccessfullyIndexed_0_Service_1, ServiceTypeEnum.Prc, service.Name));
logger.LogInformation($"{logPrefix} finished");
}
catch (Exception ex)
{
logger.LogError($"{logPrefix} failed. {ex.Message} {ex.StackTrace}");
CleanPrcIndex(prcSettings.ServiceId);
processHandler.Interrupted(processId, ex);
}
finally
{
GC.Collect();
}
}
public void IndexPartial(string processId, PrcSettingsElastic prcSettings, CancellationToken token)
{
var logPrefix = $"Prc Partial Index {processId}";
try
{
const int parallelMultiplier = 2;
var partialIndexDate = DateTime.UtcNow;
var service = serviceQuery.Get(prcSettings.ServiceId);
var wordQuery = queryFactory.GetWordQuery(prcSettings.DataSetName);
var tagProgress = new Progress(prcSettings.IndexSettings.FilterTagIdList.Count);
logger.LogInformation($"{logPrefix} starts with ParallelLimit: {parallelService.ParallelLimit * parallelMultiplier}, Tags Count: {prcSettings.IndexSettings.FilterTagIdList.Count}");
//lekérjük a legutóbbi indexelés óta módosult vagy létrehozott doksikat(Filter - t figyelve)
//TODO get IndexFilterTagIds which has changed only
foreach (var tagId in prcSettings.IndexSettings.FilterTagIdList)
{
if (token.IsCancellationRequested)
{
processHandler.Cancelled(processId);
logger.LogInformation($"{logPrefix} cancelled Tag: `{tagId}`");
return;
}
var changedDocumentElasticIds = GetDocumentIds(prcSettings.DataSetName, prcSettings.IndexSettings.FilterQuery, new List <string> {
tagId
}, prcSettings.IndexSettings.IndexDate, partialIndexDate).OrderBy(o => o);
// no changed document found since last index
if (!changedDocumentElasticIds.Any())
{
continue;
}
// kiszámoljuk az aktuális doksi base dictionary - jét
var globalSubset = GlobalStore.ActivatedPrcs.Get(prcSettings.ServiceId).PrcSubsets[tagId];
if (globalSubset.WordsWithOccurences == null)
{
continue;
}
// Kellenek azok a doksik is az indexeléshez amik nem változtak
var documentElasticIds = GetDocumentIds(prcSettings.DataSetName, prcSettings.IndexSettings.FilterQuery, new List <string> {
tagId
}, dateEnd: partialIndexDate);
if (!documentElasticIds.Any())
{
continue;
}
var wwoDocuments = wordQuery.GetWordsWithOccurencesByDocuments(
documentElasticIds,
prcSettings.FieldsForRecommendation.Select(DocumentQuery.MapDocumentObjectName),
1, parallelLimit: parallelService.ParallelLimit);
var cleanedTextDocuments = wwoDocuments.ToDictionary(w => w.Key, w => GetCleanedText(w.Value));
Parallel.ForEach(changedDocumentElasticIds, parallelService.ParallelOptions(parallelMultiplier), (documentId, loopState) =>
{
if (token.IsCancellationRequested)
{
loopState.Stop();
return;
}
try
{
logger.LogTrace($"{logPrefix} preparing Document: `{documentId}`/`{tagId}`");
var scorer = GetScorer(globalSubset, wwoDocuments[documentId], cleanedTextDocuments[documentId], GlobalStore.ActivatedPrcs.Get(prcSettings.ServiceId).PrcScorers[tagId]);
if (scorer == null)
{
return;
}
var similarDocuments = new List <KeyValuePair <string, double> >();
// calculate documentElastics scores except current document
foreach (var siblingDocumentId in documentElasticIds)
{
if (siblingDocumentId == documentId)
{
continue;
}
var wwoSibling = wwoDocuments[siblingDocumentId];
if (!wwoSibling.Any(w => scorer.BaseDic.Keys.Contains(w.Key)))
{
continue;
}
var finalScore = GetPrcScore(scorer, cleanedTextDocuments[siblingDocumentId]);
if (finalScore > 0)
{
similarDocuments.Add(new KeyValuePair <string, double>(siblingDocumentId, finalScore));
}
}
var redisKey = new PrcIndexRedisKey(prcSettings.ServiceId, tagId, documentId);
var indexedDocumentIds = redisHandler.GetDocuments(redisKey);
var unchangedDocumentIds = indexedDocumentIds
.Where(idx => similarDocuments.Any(sim => sim.Key == idx.Element && sim.Value == idx.Score))
.Select(s => s.Element)
.ToList();
var documentIdsToAdjust = indexedDocumentIds.Select(s => s.Element.ToString())
.Union(similarDocuments.Select(s => s.Key))
.Where(w => !unchangedDocumentIds.Contains(w))
.Distinct()
.ToList();
redisHandler.ReplaceDocuments(redisKey, similarDocuments);
//ezekre doksikra (ha már kéznél vannak), visszafelé is kiszámoljuk a prc score-t
foreach (var reverseDocumentId in documentIdsToAdjust)
{
var reverseRedisKey = new PrcIndexRedisKey(prcSettings.ServiceId, tagId, reverseDocumentId);
redisHandler.RemoveDocument(reverseRedisKey, documentId);
// ha van egyezőség kiszámolni a prcscore-t
// és beszúrni a redisbe
var reverseScorer = GetScorer(globalSubset, wwoDocuments[reverseDocumentId], cleanedTextDocuments[reverseDocumentId], GlobalStore.ActivatedPrcs.Get(prcSettings.ServiceId).PrcScorers[tagId]);
if (reverseScorer == null)
{
continue;
}
var wwoReverse = wwoDocuments[documentId];
if (wwoReverse.Keys.Intersect(reverseScorer.BaseDic.Keys).Count() > 0)
{
var finalScore = GetPrcScore(reverseScorer, cleanedTextDocuments[documentId]);
if (finalScore > 0)
{
redisHandler.AddDocument(reverseRedisKey, documentId, finalScore);
}
}
// levágjuk a max listaelemszám felettieket
redisHandler.TrimDocuments(reverseRedisKey);
}
}
finally
{
logger.LogTrace($"{logPrefix} prepared Document: `{documentId}`/`{tagId}`");
}
});
if (token.IsCancellationRequested)
{
processHandler.Cancelled(processId);
logger.LogInformation($"{logPrefix} cancelled Tag: `{tagId}`");
return;
}
tagProgress.Step();
processHandler.Changed(processId, tagProgress.Percent.Round(6));
GC.Collect();
}
prcSettings.IndexSettings.IndexDate = partialIndexDate;
serviceQuery.IndexSettings(prcSettings);
logger.LogInformation($"{logPrefix} finished");
processHandler.Finished(processId, string.Format(ServiceResources.SuccessfullyPartialIndexed_0_Service_1, ServiceTypeEnum.Prc, service.Name));
}
catch (Exception ex)
{
processHandler.Interrupted(processId, ex);
}
finally
{
GC.Collect();
}
}
public IEnumerable <PrcRecommendationResult> RecommendById(string id, PrcSettingsElastic prcSettings, PrcRecommendationByIdRequest request)
{
var result = new List <PrcRecommendationResult>();
var globalStoreDataSet = GlobalStore.DataSets.Get(prcSettings.DataSetName);
var dataSet = globalStoreDataSet.DataSet;
var documentQuery = queryFactory.GetDocumentQuery(dataSet.Name);
var fieldsForRecommendation = prcSettings.FieldsForRecommendation;
var filterOrWeight = !string.IsNullOrEmpty(request.Query) || request?.Weights?.Any() == true;
var tagId = string.Empty;
if (string.IsNullOrEmpty(request.TagId))
{
var documentElastic = documentQuery.Get(request.DocumentId);
if (documentElastic == null)
{
return(result);
}
var tagToken = JTokenHelper.GetToken(documentElastic.DocumentObject).GetPathToken(dataSet.TagField);
tagId = JTokenHelper.GetUnderlyingToken(tagToken)?.ToString();
if (tagId == null)
{
return(result);
}
}
else
{
tagId = request.TagId;
}
var similarDocIdsWithScore = redisHandler.GetTopNDocuments(new PrcIndexRedisKey(id, tagId, request.DocumentId), filterOrWeight ? -1 : request.Count - 1);
Dictionary <string, double> resultDictionary = similarDocIdsWithScore;
var documentElastics = (filterOrWeight || request.NeedDocumentInResult) ?
GetDocuments(dataSet.Name, request.Query, null, fieldsForRecommendation, similarDocIdsWithScore.Keys, request.NeedDocumentInResult) :
null;
// ha a Filter és a Weights is üres, a TOP Count doksi Id - t visszaadjuk score-jaikkal. (ha kell a document is, akkor elastic - tól elkérjük ezeket pluszban)
if (filterOrWeight)
{
// ezekre a doksikra módosítjuk a prc score - t a Weights-el
var docIdsWithScore = documentElastics.ToDictionary(k => k.Id, v => similarDocIdsWithScore[v.Id]);
//súlyozás
if (request?.Weights?.Any() == true)
{
var weightsDic = request.Weights.ToDictionary(key => Guid.NewGuid().ToString(), value => value);
var docIds = docIdsWithScore.Keys.ToList();
var queries = weightsDic.ToDictionary(key => key.Key, value => documentQuery.PrefixQueryFields(value.Value.Query, globalStoreDataSet.DocumentFields));
var ids = documentQuery.GetExistsForQueries(queries, docIds).ToDictionary(k => k.Key, v => v.Value.ToDictionary(ke => ke, va => va));
var allWeightsCount = request.Weights.Count;
foreach (var docId in docIds)
{
var weightsSum = weightsDic.Where(w => ids[w.Key].ContainsKey(docId)).Sum(w => w.Value.Value);
var pow = 1 + (weightsSum / allWeightsCount);
var score = Math.Pow(docIdsWithScore[docId] + 1, pow) - 1;
docIdsWithScore[docId] = score;
}
}
resultDictionary = docIdsWithScore;
}
var recommendation = resultDictionary
.OrderByDescending(o => o.Value)
.Take(request.Count)
.Select(s => new PrcRecommendationResult()
{
DocumentId = s.Key,
Score = s.Value,
Document = request.NeedDocumentInResult ? documentElastics.SingleOrDefault(d => d.Id == s.Key).DocumentObject : null
});
return(recommendation);
}
public IActionResult Prepare(string id, [FromBody] PrcPrepareSettings prcPrepareSettings)
{
//SERVICE VALIDATION
var service = serviceQuery.Get(id);
if (service == null)
{
return(new HttpStatusCodeWithErrorResult(StatusCodes.Status404NotFound, ServiceResources.InvalidIdNotExistingService));
}
if (service.Type != (int)ServiceTypeEnum.Prc)
{
return(new HttpStatusCodeWithErrorResult(StatusCodes.Status400BadRequest, string.Format(ServiceResources.InvalidServiceTypeOnly_0_ServicesAreValidForThisRequest, "Prc")));
}
if (service.Status != (int)ServiceStatusEnum.New)
{
return(new HttpStatusCodeWithErrorResult(StatusCodes.Status400BadRequest, ServiceResources.InvalidStatusOnlyTheServicesWithNewStatusCanBePrepared));
}
//DATASET VALIDATION
if (!GlobalStore.DataSets.IsExist(prcPrepareSettings.DataSetName))
{
return(new HttpStatusCodeWithErrorResult(StatusCodes.Status400BadRequest, string.Format(ServiceResources.DataSet_0_NotFound, prcPrepareSettings.DataSetName)));
}
var globalStoreDataSet = GlobalStore.DataSets.Get(prcPrepareSettings.DataSetName);
var dataSet = globalStoreDataSet.DataSet;
//TAGS VALIDATION
var tagQuery = queryFactory.GetTagQuery(dataSet.Name);
List <TagElastic> tags;
if (prcPrepareSettings?.TagIdList?.Any() == true)
{
tags = tagQuery.Get(prcPrepareSettings.TagIdList).ToList();
if (tags.Count < prcPrepareSettings.TagIdList.Count)
{
var missingTagIds = prcPrepareSettings.TagIdList.Except(tags.Select(t => t.Id)).ToList();
return(new HttpStatusCodeWithErrorResult(StatusCodes.Status400BadRequest,
string.Format(ServiceResources.TheFollowingTagIdsNotExistInTheDataSet_0, string.Join(", ", missingTagIds))));
}
}
else
{
tags = tagQuery.GetAll().Items.Where(i => i.IsLeaf).ToList();
}
//SAVE SETTINGS TO ELASTIC
var serviceSettings = new PrcSettingsElastic
{
DataSetName = globalStoreDataSet.IndexName,
ServiceId = service.Id,
Tags = tags,
CompressSettings = CompressHelper.ToCompressSettingsElastic(prcPrepareSettings.CompressSettings, prcPrepareSettings.CompressLevel)
};
serviceQuery.IndexSettings(serviceSettings);
var process = processHandler.Create(
ProcessTypeEnum.PrcPrepare,
service.Id,
prcPrepareSettings,
string.Format(ServiceResources.Preparing_0_Service_1, ServiceTypeEnum.Prc, service.Name));
service.ProcessIdList.Add(process.Id);
serviceQuery.Update(service.Id, service);
processHandler.Start(process, (tokenSource) => prcHandler.Prepare(process.Id, serviceSettings, tokenSource.Token));
return(new HttpStatusCodeWithObjectResult(StatusCodes.Status202Accepted, process.ToProcessModel()));
}