public IActionResult Get(string id) { var service = serviceQuery.Get(id); if (service == null) { return(new StatusCodeResult(StatusCodes.Status404NotFound)); } if (service.Type != (int)ServiceTypeEnum.Search) { return(new HttpStatusCodeWithErrorResult(StatusCodes.Status400BadRequest, string.Format(ServiceResources.InvalidServiceTypeOnly_0_ServicesAreValidForThisRequest, "Search"))); } SearchActivateSettings activateSettings = null; SearchPrepareSettings prepareSettings = null; var searchSettingsElastic = serviceQuery.GetSettings <SearchSettingsWrapperElastic>(service.Id); if (searchSettingsElastic != null) { prepareSettings = searchSettingsElastic.ToSearchPrepareSettingsModel(); activateSettings = searchSettingsElastic.ToSearchActivateSettingsModel(); } var respService = service.ToServiceModel <SearchService>(); respService.ActualProcessId = service.ProcessIdList.FirstOrDefault(pid => GlobalStore.Processes.IsExist(pid)); respService.ActivateSettings = activateSettings; respService.PrepareSettings = prepareSettings; return(new OkObjectResult(respService)); }
public void Prepare(string processId, SearchSettingsWrapperElastic settings, CancellationToken token) { try { var service = serviceQuery.Get(settings.ServiceId); service.Status = (int)ServiceStatusEnum.Busy; serviceQuery.Update(service.Id, service); processHandler.Finished(processId, string.Format(ServiceResources.SuccessfullyPrepared_0_Service_1, ServiceTypeEnum.Classifier, service.Name)); service.Status = (int)ServiceStatusEnum.Prepared; serviceQuery.Update(service.Id, service); } catch (Exception ex) { var service = serviceQuery.Get(settings.ServiceId); service.Status = (int)ServiceStatusEnum.New; serviceQuery.Update(service.Id, service); if (ex.InnerException != null && ex.InnerException is OperationCanceledException) { processHandler.Cancelled(processId); } else { processHandler.Interrupted(processId, ex); } } }
public IActionResult Get(string id) { var serviceElastic = serviceQuery.Get(id); if (serviceElastic == null) { return(new StatusCodeResult(StatusCodes.Status404NotFound)); } var service = serviceElastic.ToServiceModel <Service>(); service.ActualProcessId = serviceElastic.ProcessIdList.FirstOrDefault(pid => GlobalStore.Processes.IsExist(pid)); return(new OkObjectResult(service)); }
public IActionResult Get(string id) { var service = serviceQuery.Get(id); if (service == null) { return(new StatusCodeResult(StatusCodes.Status404NotFound)); } if (service.Type != (int)ServiceTypeEnum.Classifier) { return(new HttpStatusCodeWithErrorResult(StatusCodes.Status400BadRequest, string.Format(ServiceResources.InvalidServiceTypeOnly_0_ServicesAreValidForThisRequest, "Classifier"))); } ClassifierActivateSettings activateSettings = null; ClassifierPrepareSettings prepareSettings = null; var classifierSettingsElastic = serviceQuery.GetSettings <ClassifierSettingsElastic>(service.Id); if (classifierSettingsElastic != null) { if (service.Status == (int)ServiceStatusEnum.Prepared || service.Status == (int)ServiceStatusEnum.Active) { prepareSettings = new ClassifierPrepareSettings { DataSetName = GlobalStore.DataSets.Get(classifierSettingsElastic.DataSetName).AliasName, NGramList = classifierSettingsElastic.NGramList, TagIdList = classifierSettingsElastic.Tags.Select(t => t.Id).ToList(), CompressSettings = CompressHelper.ToCompressSettings(classifierSettingsElastic.CompressSettings), CompressLevel = CompressHelper.ToCompressLevel(classifierSettingsElastic.CompressSettings) }; if (service.Status == (int)ServiceStatusEnum.Active) { activateSettings = new ClassifierActivateSettings { TagIdList = classifierSettingsElastic.ActivatedTagIdList, EmphasizedTagIdList = classifierSettingsElastic.EmphasizedTagIdList, NGramList = classifierSettingsElastic.ActivatedNGramList }; } } } var respService = service.ToServiceModel <ClassifierService>(); respService.ActualProcessId = service.ProcessIdList.FirstOrDefault(pid => GlobalStore.Processes.IsExist(pid)); respService.ActivateSettings = activateSettings; respService.PrepareSettings = prepareSettings; return(new OkObjectResult(respService)); }
public void CancelBusyProcesses() { var busyProcesses = processQuery.GetAll(true); foreach (var process in busyProcesses) { if (process.Type == (int)ProcessTypeEnum.PrcIndex) { var serviceId = process.AffectedObjectId?.ToString(); var service = serviceQuery.Get(serviceId); if (service == null) { logger.LogError($"Cannot find Service {serviceId} of the interrupted process {process.Id}"); continue; } prcIndexServiceHandler.CleanPrcIndex(serviceId); } processHandler.Interrupted(process.Id, new SlambyException(Resources.ProcessResources.UnexpectedInterruptionError)); } }
public void Prepare(string processId, ClassifierSettingsElastic settings, CancellationToken token) { var directoryPath = GetDirectoryPath(settings.ServiceId); try { var service = serviceQuery.Get(settings.ServiceId); service.Status = (int)ServiceStatusEnum.Busy; serviceQuery.Update(service.Id, service); IOHelper.SafeDeleteDictionary(directoryPath, true); var globalStoreDataSet = GlobalStore.DataSets.Get(settings.DataSetName); var dataSet = globalStoreDataSet.DataSet; var allDicCount = settings.NGramList.Count * settings.Tags.Count; var counter = 0; var lockObject = new object(); Directory.CreateDirectory(directoryPath); foreach (var nGram in settings.NGramList) { var subsetCreator = new SubsetCreator(dataSet.Name, new List <string> { DocumentElastic.TextField }, dataSet.InterpretedFields.Select(DocumentQuery.MapDocumentObjectName).ToList(), nGram, queryFactory, globalStoreDataSet.AttachmentFields); var actualDirectory = string.Format("{0}/{1}", directoryPath, nGram); Directory.CreateDirectory(actualDirectory); Parallel.ForEach(settings.Tags, parallelService.ParallelOptions(), (tag, loopState) => { token.ThrowIfCancellationRequested(); var subset = subsetCreator.CreateByTag(tag.Id, dataSet.TagField); var dictionary = new TwisterAlgorithm( subset, true, false, settings.CompressSettings.CompressCategoryOccurence, settings.CompressSettings.CompressDataSetOccurence, (LogicalOperatorEnum)settings.CompressSettings.CompressOperator).GetDictionary(); var dicProtoBuf = new DictionaryProtoBuf { Id = tag.Id, Dictionary = dictionary, NGram = nGram }; lock (lockObject) { dicProtoBuf.Serialize(string.Format("{0}/{1}", actualDirectory, dicProtoBuf.GetFileName())); processHandler.Changed(processId, Math.Round(++counter / (double)allDicCount * 100, 2)); } }); } processHandler.Finished(processId, string.Format(ServiceResources.SuccessfullyPrepared_0_Service_1, ServiceTypeEnum.Classifier, service.Name)); service.Status = (int)ServiceStatusEnum.Prepared; serviceQuery.Update(service.Id, service); } catch (Exception ex) { var service = serviceQuery.Get(settings.ServiceId); service.Status = (int)ServiceStatusEnum.New; serviceQuery.Update(service.Id, service); IOHelper.SafeDeleteDictionary(directoryPath, true); if (ex.InnerException != null && ex.InnerException is OperationCanceledException) { processHandler.Cancelled(processId); } else { processHandler.Interrupted(processId, ex); } } }
public IActionResult Get(string id) { var service = serviceQuery.Get(id); if (service == null) { return(new StatusCodeResult(StatusCodes.Status404NotFound)); } if (service.Type != (int)ServiceTypeEnum.Prc) { return(new HttpStatusCodeWithErrorResult(StatusCodes.Status400BadRequest, string.Format(ServiceResources.InvalidServiceTypeOnly_0_ServicesAreValidForThisRequest, "Prc"))); } PrcActivateSettings activateSettings = null; PrcPrepareSettings prepareSettings = null; PrcIndexSettings indexSettings = null; var prcSettingsElastic = serviceQuery.GetSettings <PrcSettingsElastic>(service.Id); if (prcSettingsElastic != null) { if (service.Status == (int)ServiceStatusEnum.Prepared || service.Status == (int)ServiceStatusEnum.Active) { prepareSettings = new PrcPrepareSettings { DataSetName = GlobalStore.DataSets.Get(prcSettingsElastic.DataSetName).AliasName, TagIdList = prcSettingsElastic.Tags.Select(t => t.Id).ToList(), CompressSettings = CompressHelper.ToCompressSettings(prcSettingsElastic.CompressSettings), CompressLevel = CompressHelper.ToCompressLevel(prcSettingsElastic.CompressSettings) }; if (service.Status == (int)ServiceStatusEnum.Active) { activateSettings = new PrcActivateSettings { FieldsForRecommendation = prcSettingsElastic.FieldsForRecommendation }; if (prcSettingsElastic?.IndexSettings?.IndexDate != null) { indexSettings = new PrcIndexSettings() { Filter = new Filter() { Query = prcSettingsElastic.IndexSettings.FilterQuery, TagIdList = prcSettingsElastic.IndexSettings.FilterTagIdList } }; } } } } var respService = service.ToServiceModel <PrcService>(); respService.ActualProcessId = service.ProcessIdList.FirstOrDefault(pid => GlobalStore.Processes.IsExist(pid)); respService.ActivateSettings = activateSettings; respService.PrepareSettings = prepareSettings; respService.IndexSettings = indexSettings; return(new OkObjectResult(respService)); }
public void Prepare(string processId, PrcSettingsElastic settings, CancellationToken token) { var directoryPath = GetDirectoryPath(settings.ServiceId); try { var service = serviceQuery.Get(settings.ServiceId); service.Status = (int)ServiceStatusEnum.Busy; serviceQuery.Update(service.Id, service); IOHelper.SafeDeleteDictionary(directoryPath, true); var globalStoreDataSet = GlobalStore.DataSets.Get(settings.DataSetName); var dataSet = globalStoreDataSet.DataSet; var progress = new Progress(settings.Tags.Count); var subsetCreator = new SubsetCreator(dataSet.Name, new List <string> { DocumentElastic.TextField }, dataSet.InterpretedFields.Select(DocumentQuery.MapDocumentObjectName).ToList(), 1, queryFactory, globalStoreDataSet.AttachmentFields); Directory.CreateDirectory(directoryPath); var logPrefix = $"Prc Prepare {processId}"; logger.LogInformation($"{logPrefix} starts with ParallelLimit: {parallelService.ParallelLimit}, Tags Count: {settings.Tags.Count}"); var lockObject = new object(); Parallel.ForEach(settings.Tags, parallelService.ParallelOptions(), (tag, loopState) => { token.ThrowIfCancellationRequested(); logger.LogInformation($"{logPrefix} preparing Tag: `{tag}`"); var subset = subsetCreator.CreateByTag(tag.Id, dataSet.TagField); var algorithm = new TwisterAlgorithm( subset, true, true, settings.CompressSettings.CompressCategoryOccurence, settings.CompressSettings.CompressDataSetOccurence, (LogicalOperatorEnum)settings.CompressSettings.CompressOperator); algorithm.InitTagDictionary(); var notNeededWords = subset.WordsWithOccurences.Keys.Except( algorithm.TagDictionary .Where(sd => sd.Value.PMI > 0) .Select(sd => sd.Key)).ToList(); var td = algorithm.GetDictionary(); foreach (var word in notNeededWords) { subset.WordsWithOccurences.Remove(word); } lock (lockObject) { //dictionary serialization var dicProtoBuf = new DictionaryProtoBuf { Id = tag.Id, Dictionary = td, NGram = 1 }; dicProtoBuf.Serialize(string.Format("{0}/{1}", directoryPath, dicProtoBuf.GetFileName())); //subset serialization var subsetProtoBuf = new SubsetProtoBuf { Id = tag.Id, WordsWithOccurences = subset.WordsWithOccurences, AllWordsOccurencesSumInTag = subset.AllWordsOccurencesSumInTag, AllOccurencesSumInCorpus = subset.AllWordsOccurencesSumInCorpus }; subsetProtoBuf.Serialize(string.Format("{0}/{1}", directoryPath, subsetProtoBuf.GetFileName())); progress.Step(); processHandler.Changed(processId, progress.Percent.Round(2)); } logger.LogInformation($"{logPrefix} prepared Tag: `{tag}`"); }); processHandler.Finished(processId, string.Format(ServiceResources.SuccessfullyPrepared_0_Service_1, ServiceTypeEnum.Prc, service.Name)); service.Status = (int)ServiceStatusEnum.Prepared; serviceQuery.Update(service.Id, service); } catch (Exception ex) { var service = serviceQuery.Get(settings.ServiceId); service.Status = (int)ServiceStatusEnum.New; serviceQuery.Update(service.Id, service); IOHelper.SafeDeleteDictionary(directoryPath, true); if (ex.InnerException != null && ex.InnerException is OperationCanceledException) { processHandler.Cancelled(processId); } else { processHandler.Interrupted(processId, ex); } } }