public IActionResult Get(string id)
{
var service = serviceQuery.Get(id);
if (service == null)
{
return(new StatusCodeResult(StatusCodes.Status404NotFound));
}
if (service.Type != (int)ServiceTypeEnum.Search)
{
return(new HttpStatusCodeWithErrorResult(StatusCodes.Status400BadRequest, string.Format(ServiceResources.InvalidServiceTypeOnly_0_ServicesAreValidForThisRequest, "Search")));
}
SearchActivateSettings activateSettings = null;
SearchPrepareSettings prepareSettings = null;
var searchSettingsElastic = serviceQuery.GetSettings <SearchSettingsWrapperElastic>(service.Id);
if (searchSettingsElastic != null)
{
prepareSettings = searchSettingsElastic.ToSearchPrepareSettingsModel();
activateSettings = searchSettingsElastic.ToSearchActivateSettingsModel();
}
var respService = service.ToServiceModel <SearchService>();
respService.ActualProcessId = service.ProcessIdList.FirstOrDefault(pid => GlobalStore.Processes.IsExist(pid));
respService.ActivateSettings = activateSettings;
respService.PrepareSettings = prepareSettings;
return(new OkObjectResult(respService));
}
public void Prepare(string processId, SearchSettingsWrapperElastic settings, CancellationToken token)
{
try
{
var service = serviceQuery.Get(settings.ServiceId);
service.Status = (int)ServiceStatusEnum.Busy;
serviceQuery.Update(service.Id, service);
processHandler.Finished(processId, string.Format(ServiceResources.SuccessfullyPrepared_0_Service_1, ServiceTypeEnum.Classifier, service.Name));
service.Status = (int)ServiceStatusEnum.Prepared;
serviceQuery.Update(service.Id, service);
}
catch (Exception ex)
{
var service = serviceQuery.Get(settings.ServiceId);
service.Status = (int)ServiceStatusEnum.New;
serviceQuery.Update(service.Id, service);
if (ex.InnerException != null && ex.InnerException is OperationCanceledException)
{
processHandler.Cancelled(processId);
}
else
{
processHandler.Interrupted(processId, ex);
}
}
}
public IActionResult Get(string id)
{
var serviceElastic = serviceQuery.Get(id);
if (serviceElastic == null)
{
return(new StatusCodeResult(StatusCodes.Status404NotFound));
}
var service = serviceElastic.ToServiceModel <Service>();
service.ActualProcessId = serviceElastic.ProcessIdList.FirstOrDefault(pid => GlobalStore.Processes.IsExist(pid));
return(new OkObjectResult(service));
}
public IActionResult Get(string id)
{
var service = serviceQuery.Get(id);
if (service == null)
{
return(new StatusCodeResult(StatusCodes.Status404NotFound));
}
if (service.Type != (int)ServiceTypeEnum.Classifier)
{
return(new HttpStatusCodeWithErrorResult(StatusCodes.Status400BadRequest, string.Format(ServiceResources.InvalidServiceTypeOnly_0_ServicesAreValidForThisRequest, "Classifier")));
}
ClassifierActivateSettings activateSettings = null;
ClassifierPrepareSettings prepareSettings = null;
var classifierSettingsElastic = serviceQuery.GetSettings <ClassifierSettingsElastic>(service.Id);
if (classifierSettingsElastic != null)
{
if (service.Status == (int)ServiceStatusEnum.Prepared || service.Status == (int)ServiceStatusEnum.Active)
{
prepareSettings = new ClassifierPrepareSettings {
DataSetName = GlobalStore.DataSets.Get(classifierSettingsElastic.DataSetName).AliasName,
NGramList = classifierSettingsElastic.NGramList,
TagIdList = classifierSettingsElastic.Tags.Select(t => t.Id).ToList(),
CompressSettings = CompressHelper.ToCompressSettings(classifierSettingsElastic.CompressSettings),
CompressLevel = CompressHelper.ToCompressLevel(classifierSettingsElastic.CompressSettings)
};
if (service.Status == (int)ServiceStatusEnum.Active)
{
activateSettings = new ClassifierActivateSettings
{
TagIdList = classifierSettingsElastic.ActivatedTagIdList,
EmphasizedTagIdList = classifierSettingsElastic.EmphasizedTagIdList,
NGramList = classifierSettingsElastic.ActivatedNGramList
};
}
}
}
var respService = service.ToServiceModel <ClassifierService>();
respService.ActualProcessId = service.ProcessIdList.FirstOrDefault(pid => GlobalStore.Processes.IsExist(pid));
respService.ActivateSettings = activateSettings;
respService.PrepareSettings = prepareSettings;
return(new OkObjectResult(respService));
}
public void CancelBusyProcesses()
{
var busyProcesses = processQuery.GetAll(true);
foreach (var process in busyProcesses)
{
if (process.Type == (int)ProcessTypeEnum.PrcIndex)
{
var serviceId = process.AffectedObjectId?.ToString();
var service = serviceQuery.Get(serviceId);
if (service == null)
{
logger.LogError($"Cannot find Service {serviceId} of the interrupted process {process.Id}");
continue;
}
prcIndexServiceHandler.CleanPrcIndex(serviceId);
}
processHandler.Interrupted(process.Id, new SlambyException(Resources.ProcessResources.UnexpectedInterruptionError));
}
}
public void Prepare(string processId, ClassifierSettingsElastic settings, CancellationToken token)
{
var directoryPath = GetDirectoryPath(settings.ServiceId);
try
{
var service = serviceQuery.Get(settings.ServiceId);
service.Status = (int)ServiceStatusEnum.Busy;
serviceQuery.Update(service.Id, service);
IOHelper.SafeDeleteDictionary(directoryPath, true);
var globalStoreDataSet = GlobalStore.DataSets.Get(settings.DataSetName);
var dataSet = globalStoreDataSet.DataSet;
var allDicCount = settings.NGramList.Count * settings.Tags.Count;
var counter = 0;
var lockObject = new object();
Directory.CreateDirectory(directoryPath);
foreach (var nGram in settings.NGramList)
{
var subsetCreator = new SubsetCreator(dataSet.Name, new List <string> {
DocumentElastic.TextField
}, dataSet.InterpretedFields.Select(DocumentQuery.MapDocumentObjectName).ToList(), nGram, queryFactory, globalStoreDataSet.AttachmentFields);
var actualDirectory = string.Format("{0}/{1}", directoryPath, nGram);
Directory.CreateDirectory(actualDirectory);
Parallel.ForEach(settings.Tags, parallelService.ParallelOptions(), (tag, loopState) => {
token.ThrowIfCancellationRequested();
var subset = subsetCreator.CreateByTag(tag.Id, dataSet.TagField);
var dictionary = new TwisterAlgorithm(
subset, true, false,
settings.CompressSettings.CompressCategoryOccurence,
settings.CompressSettings.CompressDataSetOccurence,
(LogicalOperatorEnum)settings.CompressSettings.CompressOperator).GetDictionary();
var dicProtoBuf = new DictionaryProtoBuf
{
Id = tag.Id,
Dictionary = dictionary,
NGram = nGram
};
lock (lockObject)
{
dicProtoBuf.Serialize(string.Format("{0}/{1}", actualDirectory, dicProtoBuf.GetFileName()));
processHandler.Changed(processId, Math.Round(++counter / (double)allDicCount * 100, 2));
}
});
}
processHandler.Finished(processId, string.Format(ServiceResources.SuccessfullyPrepared_0_Service_1, ServiceTypeEnum.Classifier, service.Name));
service.Status = (int)ServiceStatusEnum.Prepared;
serviceQuery.Update(service.Id, service);
}
catch (Exception ex)
{
var service = serviceQuery.Get(settings.ServiceId);
service.Status = (int)ServiceStatusEnum.New;
serviceQuery.Update(service.Id, service);
IOHelper.SafeDeleteDictionary(directoryPath, true);
if (ex.InnerException != null && ex.InnerException is OperationCanceledException)
{
processHandler.Cancelled(processId);
}
else
{
processHandler.Interrupted(processId, ex);
}
}
}
public IActionResult Get(string id)
{
var service = serviceQuery.Get(id);
if (service == null)
{
return(new StatusCodeResult(StatusCodes.Status404NotFound));
}
if (service.Type != (int)ServiceTypeEnum.Prc)
{
return(new HttpStatusCodeWithErrorResult(StatusCodes.Status400BadRequest, string.Format(ServiceResources.InvalidServiceTypeOnly_0_ServicesAreValidForThisRequest, "Prc")));
}
PrcActivateSettings activateSettings = null;
PrcPrepareSettings prepareSettings = null;
PrcIndexSettings indexSettings = null;
var prcSettingsElastic = serviceQuery.GetSettings <PrcSettingsElastic>(service.Id);
if (prcSettingsElastic != null)
{
if (service.Status == (int)ServiceStatusEnum.Prepared || service.Status == (int)ServiceStatusEnum.Active)
{
prepareSettings = new PrcPrepareSettings
{
DataSetName = GlobalStore.DataSets.Get(prcSettingsElastic.DataSetName).AliasName,
TagIdList = prcSettingsElastic.Tags.Select(t => t.Id).ToList(),
CompressSettings = CompressHelper.ToCompressSettings(prcSettingsElastic.CompressSettings),
CompressLevel = CompressHelper.ToCompressLevel(prcSettingsElastic.CompressSettings)
};
if (service.Status == (int)ServiceStatusEnum.Active)
{
activateSettings = new PrcActivateSettings
{
FieldsForRecommendation = prcSettingsElastic.FieldsForRecommendation
};
if (prcSettingsElastic?.IndexSettings?.IndexDate != null)
{
indexSettings = new PrcIndexSettings()
{
Filter = new Filter()
{
Query = prcSettingsElastic.IndexSettings.FilterQuery,
TagIdList = prcSettingsElastic.IndexSettings.FilterTagIdList
}
};
}
}
}
}
var respService = service.ToServiceModel <PrcService>();
respService.ActualProcessId = service.ProcessIdList.FirstOrDefault(pid => GlobalStore.Processes.IsExist(pid));
respService.ActivateSettings = activateSettings;
respService.PrepareSettings = prepareSettings;
respService.IndexSettings = indexSettings;
return(new OkObjectResult(respService));
}
public void Prepare(string processId, PrcSettingsElastic settings, CancellationToken token)
{
var directoryPath = GetDirectoryPath(settings.ServiceId);
try
{
var service = serviceQuery.Get(settings.ServiceId);
service.Status = (int)ServiceStatusEnum.Busy;
serviceQuery.Update(service.Id, service);
IOHelper.SafeDeleteDictionary(directoryPath, true);
var globalStoreDataSet = GlobalStore.DataSets.Get(settings.DataSetName);
var dataSet = globalStoreDataSet.DataSet;
var progress = new Progress(settings.Tags.Count);
var subsetCreator = new SubsetCreator(dataSet.Name, new List <string> {
DocumentElastic.TextField
}, dataSet.InterpretedFields.Select(DocumentQuery.MapDocumentObjectName).ToList(), 1, queryFactory, globalStoreDataSet.AttachmentFields);
Directory.CreateDirectory(directoryPath);
var logPrefix = $"Prc Prepare {processId}";
logger.LogInformation($"{logPrefix} starts with ParallelLimit: {parallelService.ParallelLimit}, Tags Count: {settings.Tags.Count}");
var lockObject = new object();
Parallel.ForEach(settings.Tags, parallelService.ParallelOptions(), (tag, loopState) =>
{
token.ThrowIfCancellationRequested();
logger.LogInformation($"{logPrefix} preparing Tag: `{tag}`");
var subset = subsetCreator.CreateByTag(tag.Id, dataSet.TagField);
var algorithm = new TwisterAlgorithm(
subset, true, true,
settings.CompressSettings.CompressCategoryOccurence,
settings.CompressSettings.CompressDataSetOccurence,
(LogicalOperatorEnum)settings.CompressSettings.CompressOperator);
algorithm.InitTagDictionary();
var notNeededWords = subset.WordsWithOccurences.Keys.Except(
algorithm.TagDictionary
.Where(sd => sd.Value.PMI > 0)
.Select(sd => sd.Key)).ToList();
var td = algorithm.GetDictionary();
foreach (var word in notNeededWords)
{
subset.WordsWithOccurences.Remove(word);
}
lock (lockObject)
{
//dictionary serialization
var dicProtoBuf = new DictionaryProtoBuf
{
Id = tag.Id,
Dictionary = td,
NGram = 1
};
dicProtoBuf.Serialize(string.Format("{0}/{1}", directoryPath, dicProtoBuf.GetFileName()));
//subset serialization
var subsetProtoBuf = new SubsetProtoBuf
{
Id = tag.Id,
WordsWithOccurences = subset.WordsWithOccurences,
AllWordsOccurencesSumInTag = subset.AllWordsOccurencesSumInTag,
AllOccurencesSumInCorpus = subset.AllWordsOccurencesSumInCorpus
};
subsetProtoBuf.Serialize(string.Format("{0}/{1}", directoryPath, subsetProtoBuf.GetFileName()));
progress.Step();
processHandler.Changed(processId, progress.Percent.Round(2));
}
logger.LogInformation($"{logPrefix} prepared Tag: `{tag}`");
});
processHandler.Finished(processId, string.Format(ServiceResources.SuccessfullyPrepared_0_Service_1, ServiceTypeEnum.Prc, service.Name));
service.Status = (int)ServiceStatusEnum.Prepared;
serviceQuery.Update(service.Id, service);
}
catch (Exception ex)
{
var service = serviceQuery.Get(settings.ServiceId);
service.Status = (int)ServiceStatusEnum.New;
serviceQuery.Update(service.Id, service);
IOHelper.SafeDeleteDictionary(directoryPath, true);
if (ex.InnerException != null && ex.InnerException is OperationCanceledException)
{
processHandler.Cancelled(processId);
}
else
{
processHandler.Interrupted(processId, ex);
}
}
}
