public ElasticsearchLarsDataService(
IElasticsearchCustomClient elasticsearchCustomClient,
IIndexSettings <IMaintainLarsIndex> larsSettings)
{
_elasticsearchCustomClient = elasticsearchCustomClient;
_larsSettings = larsSettings;
}
public IndexerService(IIndexSettings <T> indexSettings, IGenericIndexerHelper <T> indexerHelper, ILog log)
{
_indexSettings = indexSettings;
_indexerHelper = indexerHelper;
Log = log;
_name = IndexerNameLookup.GetIndexTypeName(typeof(T));
}
private static void SetArray <TArray, TItem>(IIndexSettings s, IDictionary <string, JProperty> settings, string key, Action <TArray> assign,
Action <TItem> assign2, JsonSerializer serializer = null
)
{
if (!settings.ContainsKey(key))
{
return;
}
var v = settings[key];
if (v.Value is JArray)
{
var value = serializer == null?v.Value.ToObject <TArray>() : v.Value.ToObject <TArray>(serializer);
assign(value);
s.Add(key, value);
}
else
{
var value = serializer == null?v.Value.ToObject <TItem>() : v.Value.ToObject <TItem>(serializer);
assign2(value);
s.Add(key, value);
}
settings.Remove(key);
}
internal static void RemoveReadOnlySettings (IIndexSettings settings)
{
if (settings == null) return;
foreach(var bad in ReadOnlySettings)
{
if (settings.ContainsKey(bad))
settings.Remove(bad);
}
}
public void Setup()
{
_mockHelper = new Mock <IGenericIndexerHelper <Indexer.Core.Models.Provider.Provider> >();
_mockSettings =
Mock.Of <IIndexSettings <Indexer.Core.Models.Provider.Provider> >(
x => x.IndexesAlias == "testproviderindexesalias");
_sut = new IndexerService <Indexer.Core.Models.Provider.Provider>(_mockSettings, _mockHelper.Object, Mock.Of <ILog>());
}
public AssessmentOrgsIndexer(
IIndexSettings <IMaintainAssessmentOrgsIndex> settings,
IMaintainAssessmentOrgsIndex assessmentOrgsIndexMaintainer,
IMetaDataHelper metaDataHelper,
ILog log)
{
_settings = settings;
_assessmentOrgsIndexMaintainer = assessmentOrgsIndexMaintainer;
_metaDataHelper = metaDataHelper;
_log = log;
}
public IndexerService(IIndexSettings <T> indexSettings,
IGenericIndexerHelper <T> indexerHelper,
IMonitoringService monitoringService,
ILog log)
{
_indexSettings = indexSettings;
_indexerHelper = indexerHelper;
_monitoringService = monitoringService;
_log = log;
_name = IndexTypeName;
}
public ApprenticeshipIndexer(
IIndexSettings <IMaintainApprenticeshipIndex> settings,
IMediator mediator,
IMaintainApprenticeshipIndex searchIndexMaintainer,
ILog log)
{
_settings = settings;
_mediator = mediator;
_searchIndexMaintainer = searchIndexMaintainer;
_log = log;
}
public LarsIndexer(
IIndexSettings <IMaintainLarsIndex> settings,
IMaintainLarsIndex searchIndexMaintainer,
IMetaDataHelper metaDataHelper,
ILog log)
{
_settings = settings;
_searchIndexMaintainer = searchIndexMaintainer;
_metaDataHelper = metaDataHelper;
_log = log;
}
private static void Set <T>(IIndexSettings s, IDictionary <string, JProperty> settings, string key, Action <T> assign, JsonSerializer serializer = null)
{
if (!settings.ContainsKey(key))
{
return;
}
var v = settings[key];
T value = serializer == null?v.Value.ToObject <T>() : v.Value.ToObject <T>(serializer);
assign(value);
s.Add(key, value);
settings.Remove(key);
}
public ProviderIndexer(
IIndexSettings <IMaintainProviderIndex> settings,
IMaintainProviderIndex indexMaintainer,
IGetRoatpProviders providerDataService,
IConsumeProviderEvents providerEventConsumer,
ILog log)
{
_settings = settings;
_providerDataService = providerDataService;
_providerEventConsumer = providerEventConsumer;
_indexMaintainer = indexMaintainer;
_log = log;
}
public ElasticsearchProviderIndexMaintainer(
IElasticsearchCustomClient elasticsearchClient,
IElasticsearchRoatpDocumentMapper elasticsearchRoatpDocumentMapper,
IIndexSettings <IMaintainProviderIndex> settings,
ILog log,
IElasticsearchConfiguration elasticsearchConfiguration)
: base(elasticsearchClient, elasticsearchRoatpDocumentMapper, log, "RoatpProvider")
{
_elasticsearchClient = elasticsearchClient;
_settings = settings;
_log = log;
_elasticsearchConfiguration = elasticsearchConfiguration;
}
public static async Task <ICreateIndexResponse> CreateIndexAsync(
this IElasticClient client,
string indexName,
IIndexSettings indexSettings = null,
MappingsDescriptor mappings = null)
{
if (string.IsNullOrEmpty(indexName))
{
throw new ArgumentNullException(nameof(indexName));
}
return(await CreateIndexAsync(client, indexSettings, mappings, indexName));
}
internal static void RemoveReadOnlySettings(IIndexSettings settings)
{
if (settings == null)
{
return;
}
foreach (var bad in ReadOnlySettings)
{
if (settings.ContainsKey(bad))
{
settings.Remove(bad);
}
}
}
public ProviderIndexer(
IIndexSettings <IMaintainProviderIndex> settings,
ICourseDirectoryProviderMapper courseDirectoryProviderMapper,
IUkrlpProviderMapper ukrlpProviderMapper,
IMaintainProviderIndex searchIndexMaintainer,
IProviderDataService providerDataService,
ILog log)
{
_settings = settings;
_providerDataService = providerDataService;
_courseDirectoryProviderMapper = courseDirectoryProviderMapper;
_ukrlpProviderMapper = ukrlpProviderMapper;
_searchIndexMaintainer = searchIndexMaintainer;
_log = log;
}
private static async Task <ICreateIndexResponse> CreateIndexAsync(
IElasticClient client,
IIndexSettings indexSettings,
MappingsDescriptor mappings,
string indexName)
{
indexSettings = indexSettings ?? new IndexSettings();
mappings = mappings ?? new MappingsDescriptor();
var response = await client.CreateIndexAsync(
indexName,
createDescriptor => createDescriptor
.Settings(settingDescriptor => settingDescriptor
.NumberOfShards(indexSettings.NumberOfShards)
.FileSystemStorageImplementation(
indexSettings.FileSystemStorageImplementation))
.Mappings(mappingsDescriptor => mappings));
return(response);
}
private static void SetKnownIndexSettings(JsonReader reader, JsonSerializer serializer, IIndexSettings s)
{
var settings = Flatten(JObject.Load(reader)).Properties().ToDictionary(kv => kv.Name);
Set <int?>(s, settings, NumberOfReplicas, v => s.NumberOfReplicas = v);
Set <AutoExpandReplicas>(s, settings, UpdatableIndexSettings.AutoExpandReplicas, v => s.AutoExpandReplicas = v);
Set <Time>(s, settings, RefreshInterval, v => s.RefreshInterval = v);
Set <bool?>(s, settings, BlocksReadOnly, v => s.BlocksReadOnly = v);
Set <bool?>(s, settings, BlocksRead, v => s.BlocksRead = v);
Set <bool?>(s, settings, BlocksWrite, v => s.BlocksWrite = v);
Set <bool?>(s, settings, BlocksMetadata, v => s.BlocksMetadata = v);
Set <int?>(s, settings, Priority, v => s.Priority = v);
Set <string>(s, settings, DefaultPipeline, v => s.DefaultPipeline = v);
Set <Union <int, RecoveryInitialShards> >(s, settings, UpdatableIndexSettings.RecoveryInitialShards,
v => s.RecoveryInitialShards = v, serializer);
Set <bool?>(s, settings, RequestsCacheEnable, v => s.RequestsCacheEnabled = v);
Set <int?>(s, settings, RoutingAllocationTotalShardsPerNode,
v => s.RoutingAllocationTotalShardsPerNode = v);
Set <Time>(s, settings, UnassignedNodeLeftDelayedTimeout,
v => s.UnassignedNodeLeftDelayedTimeout = v);
var t = s.Translog = new TranslogSettings();
Set <Time>(s, settings, TranslogSyncInterval, v => t.SyncInterval = v);
Set <TranslogDurability?>(s, settings, UpdatableIndexSettings.TranslogDurability, v => t.Durability = v);
var tf = s.Translog.Flush = new TranslogFlushSettings();
Set <string>(s, settings, TranslogFlushThresholdSize, v => tf.ThresholdSize = v);
Set <Time>(s, settings, TranslogFlushThresholdPeriod, v => tf.ThresholdPeriod = v);
s.Merge = new MergeSettings();
var p = s.Merge.Policy = new MergePolicySettings();
Set <int?>(s, settings, MergePolicyExpungeDeletesAllowed, v => p.ExpungeDeletesAllowed = v);
Set <string>(s, settings, MergePolicyFloorSegment, v => p.FloorSegment = v);
Set <int?>(s, settings, MergePolicyMaxMergeAtOnce, v => p.MaxMergeAtOnce = v);
Set <int?>(s, settings, MergePolicyMaxMergeAtOnceExplicit, v => p.MaxMergeAtOnceExplicit = v);
Set <string>(s, settings, MergePolicyMaxMergedSegment, v => p.MaxMergedSegment = v);
Set <int?>(s, settings, MergePolicySegmentsPerTier, v => p.SegmentsPerTier = v);
Set <double?>(s, settings, MergePolicyReclaimDeletesWeight, v => p.ReclaimDeletesWeight = v);
var ms = s.Merge.Scheduler = new MergeSchedulerSettings();
Set <int?>(s, settings, MergeSchedulerMaxThreadCount, v => ms.MaxThreadCount = v);
Set <bool?>(s, settings, MergeSchedulerAutoThrottle, v => ms.AutoThrottle = v);
var slowlog = s.SlowLog = new SlowLog();
var search = s.SlowLog.Search = new SlowLogSearch();
Set <LogLevel?>(s, settings, SlowlogSearchLevel, v => search.LogLevel = v);
var query = s.SlowLog.Search.Query = new SlowLogSearchQuery();
Set <Time>(s, settings, SlowlogSearchThresholdQueryWarn, v => query.ThresholdWarn = v);
Set <Time>(s, settings, SlowlogSearchThresholdQueryInfo, v => query.ThresholdInfo = v);
Set <Time>(s, settings, SlowlogSearchThresholdQueryDebug,
v => query.ThresholdDebug = v);
Set <Time>(s, settings, SlowlogSearchThresholdQueryTrace,
v => query.ThresholdTrace = v);
var fetch = s.SlowLog.Search.Fetch = new SlowLogSearchFetch();
Set <Time>(s, settings, SlowlogSearchThresholdFetchWarn, v => fetch.ThresholdWarn = v);
Set <Time>(s, settings, SlowlogSearchThresholdFetchInfo, v => fetch.ThresholdInfo = v);
Set <Time>(s, settings, SlowlogSearchThresholdFetchDebug,
v => fetch.ThresholdDebug = v);
Set <Time>(s, settings, SlowlogSearchThresholdFetchTrace,
v => fetch.ThresholdTrace = v);
var indexing = s.SlowLog.Indexing = new SlowLogIndexing();
Set <Time>(s, settings, SlowlogIndexingThresholdFetchWarn, v => indexing.ThresholdWarn = v);
Set <Time>(s, settings, SlowlogIndexingThresholdFetchInfo, v => indexing.ThresholdInfo = v);
Set <Time>(s, settings, SlowlogIndexingThresholdFetchDebug, v => indexing.ThresholdDebug = v);
Set <Time>(s, settings, SlowlogIndexingThresholdFetchTrace, v => indexing.ThresholdTrace = v);
Set <LogLevel?>(s, settings, SlowlogIndexingLevel, v => indexing.LogLevel = v);
Set <int?>(s, settings, SlowlogIndexingSource, v => indexing.Source = v);
Set <int?>(s, settings, NumberOfShards, v => s.NumberOfShards = v);
Set <int?>(s, settings, NumberOfRoutingShards, v => s.NumberOfRoutingShards = v);
Set <int?>(s, settings, RoutingPartitionSize, v => s.RoutingPartitionSize = v);
Set <FileSystemStorageImplementation?>(s, settings, StoreType, v => s.FileSystemStorageImplementation = v, serializer);
var sorting = s.Sorting = new SortingSettings();
SetArray <string[], string>(s, settings, IndexSortSettings.Fields, v => sorting.Fields = v, v => sorting.Fields = new[] { v });
SetArray <IndexSortOrder[], IndexSortOrder>(s, settings, Order, v => sorting.Order = v, v => sorting.Order = new[] { v });
SetArray <IndexSortMode[], IndexSortMode>(s, settings, Mode, v => sorting.Mode = v, v => sorting.Mode = new[] { v });
SetArray <IndexSortMissing[], IndexSortMissing>(s, settings, Missing, v => sorting.Missing = v, v => sorting.Missing = new[] { v });
var queries = s.Queries = new QueriesSettings();
var queriesCache = s.Queries.Cache = new QueriesCacheSettings();
Set <bool?>(s, settings, QueriesCacheEnabled, v => queriesCache.Enabled = v);
var softDeletes = s.SoftDeletes = new SoftDeleteSettings();
Set <bool?>(s, settings, SoftDeletesEnabled, v => softDeletes.Enabled = v);
var softDeletesRetention = s.SoftDeletes.Retention = new SoftDeleteRetentionSettings();
Set <long?>(s, settings, SoftDeletesEnabled, v => softDeletesRetention.Operations = v);
IDictionary <string, object> dict = s;
foreach (var kv in settings)
{
var setting = kv.Value;
if (kv.Key == UpdatableIndexSettings.Analysis || kv.Key == "index.analysis")
{
s.Analysis = setting.Value.Value <JObject>().ToObject <Analysis>(serializer);
}
if (kv.Key == Similarity || kv.Key == "index.similarity")
{
s.Similarity = setting.Value.Value <JObject>().ToObject <Similarities>(serializer);
}
else
{
dict?.Add(kv.Key, serializer.Deserialize(kv.Value.Value.CreateReader()));
}
}
}
public DefaultSeeder(IElasticClient client, IIndexSettings indexSettings)
{
Client = client;
IndexSettings = indexSettings ?? _defaultIndexSettings;
}
public DefaultSeeder(ElasticsearchNode node, IIndexSettings indexSettings)
{
this.Client = node.Client;
this.IndexSettings = indexSettings ?? _defaultIndexSettings;
}
private void SetKnownIndexSettings(JsonReader reader, JsonSerializer serializer, IIndexSettings s)
{
var settings = Flatten(JObject.Load(reader)).Properties().ToDictionary(kv => kv.Name);
Set <int?>(settings, UpdatableIndexSettings.NumberOfReplicas, v => s.NumberOfReplicas = v);
Set <string>(settings, UpdatableIndexSettings.AutoExpandReplicas, v => s.AutoExpandReplicas = v);
Set <TimeUnitExpression>(settings, UpdatableIndexSettings.RefreshInterval, v => s.RefreshInterval = v);
Set <bool?>(settings, UpdatableIndexSettings.BlocksReadOnly, v => s.BlocksReadOnly = v);
Set <bool?>(settings, UpdatableIndexSettings.BlocksRead, v => s.BlocksRead = v);
Set <bool?>(settings, UpdatableIndexSettings.BlocksWrite, v => s.BlocksWrite = v);
Set <bool?>(settings, UpdatableIndexSettings.BlocksMetadata, v => s.BlocksMetadata = v);
Set <int?>(settings, UpdatableIndexSettings.Priority, v => s.Priority = v);
Set <bool?>(settings, UpdatableIndexSettings.WarmersEnabled, v => s.WarmersEnabled = v);
Set <bool?>(settings, UpdatableIndexSettings.RequestCacheEnable, v => s.RequestCacheEnabled = v);
Set <Union <int, RecoveryInitialShards> >(settings, UpdatableIndexSettings.RecoveryInitialShards,
v => s.RecoveryInitialShards = v);
Set <int?>(settings, UpdatableIndexSettings.RoutingAllocationTotalShardsPerNode,
v => s.RoutingAllocationTotalShardsPerNode = v);
Set <TimeUnitExpression>(settings, UpdatableIndexSettings.UnassignedNodeLeftDelayedTimeout,
v => s.UnassignedNodeLeftDelayedTimeout = v);
var t = s.Translog = new TranslogSettings();
Set <TimeUnitExpression>(settings, UpdatableIndexSettings.TranslogSyncInterval, v => t.SyncInterval = v);
Set <TranslogDurability?>(settings, UpdatableIndexSettings.TranslogDurability, v => t.Durability = v);
Set <TranslogWriteMode?>(settings, UpdatableIndexSettings.TranslogFsType, v => t.FileSystemType = v);
var tf = s.Translog.Flush = new TranslogFlushSettings();
Set <string>(settings, UpdatableIndexSettings.TranslogFlushThresholdSize, v => tf.ThresholdSize = v);
Set <int?>(settings, UpdatableIndexSettings.TranslogFlushTreshHoldOps, v => tf.ThresholdOps = v);
Set <TimeUnitExpression>(settings, UpdatableIndexSettings.TranslogFlushThresholdPeriod, v => tf.ThresholdPeriod = v);
Set <TimeUnitExpression>(settings, UpdatableIndexSettings.TranslogInterval, v => tf.Interval = v);
s.Merge = new MergeSettings();
var p = s.Merge.Policy = new MergePolicySettings();
Set <int?>(settings, UpdatableIndexSettings.MergePolicyExpungeDeletesAllowed, v => p.ExpungeDeletesAllowed = v);
Set <string>(settings, UpdatableIndexSettings.MergePolicyFloorSegment, v => p.FloorSegment = v);
Set <int?>(settings, UpdatableIndexSettings.MergePolicyMaxMergeAtOnce, v => p.MaxMergeAtOnce = v);
Set <int?>(settings, UpdatableIndexSettings.MergePolicyMaxMergeAtOnceExplicit, v => p.MaxMergeAtOnceExplicit = v);
Set <string>(settings, UpdatableIndexSettings.MergePolicyMaxMergedSegment, v => p.MaxMergedSegment = v);
Set <int?>(settings, UpdatableIndexSettings.MergePolicySegmentsPerTier, v => p.SegmentsPerTier = v);
Set <double?>(settings, UpdatableIndexSettings.MergePolicyReclaimDeletesWeight, v => p.ReclaimDeletesWeight = v);
var ms = s.Merge.Scheduler = new MergeSchedulerSettings();
Set <int?>(settings, UpdatableIndexSettings.MergeSchedulerMaxThreadCount, v => ms.MaxThreadCount = v);
Set <bool?>(settings, UpdatableIndexSettings.MergeSchedulerAutoThrottle, v => ms.AutoThrottle = v);
var slowlog = s.SlowLog = new SlowLog();
var search = s.SlowLog.Search = new SlowLogSearch();
Set <LogLevel?>(settings, UpdatableIndexSettings.SlowlogSearchLevel, v => search.LogLevel = v);
var query = s.SlowLog.Search.Query = new SlowLogSearchQuery();
Set <TimeUnitExpression>(settings, UpdatableIndexSettings.SlowlogSearchThresholdQueryWarn, v => query.ThresholdWarn = v);
Set <TimeUnitExpression>(settings, UpdatableIndexSettings.SlowlogSearchThresholdQueryInfo, v => query.ThresholdInfo = v);
Set <TimeUnitExpression>(settings, UpdatableIndexSettings.SlowlogSearchThresholdQueryDebug,
v => query.ThresholdDebug = v);
Set <TimeUnitExpression>(settings, UpdatableIndexSettings.SlowlogSearchThresholdQueryTrace,
v => query.ThresholdTrace = v);
var fetch = s.SlowLog.Search.Fetch = new SlowLogSearchFetch();
Set <TimeUnitExpression>(settings, UpdatableIndexSettings.SlowlogSearchThresholdFetchWarn, v => fetch.ThresholdWarn = v);
Set <TimeUnitExpression>(settings, UpdatableIndexSettings.SlowlogSearchThresholdFetchInfo, v => fetch.ThresholdInfo = v);
Set <TimeUnitExpression>(settings, UpdatableIndexSettings.SlowlogSearchThresholdFetchDebug,
v => fetch.ThresholdDebug = v);
Set <TimeUnitExpression>(settings, UpdatableIndexSettings.SlowlogSearchThresholdFetchTrace,
v => fetch.ThresholdTrace = v);
var indexing = s.SlowLog.Indexing = new SlowLogIndexing();
Set <TimeUnitExpression>(settings, UpdatableIndexSettings.SlowlogIndexingThresholdFetchWarn,
v => indexing.ThresholdWarn = v);
Set <TimeUnitExpression>(settings, UpdatableIndexSettings.SlowlogIndexingThresholdFetchInfo,
v => indexing.ThresholdInfo = v);
Set <TimeUnitExpression>(settings, UpdatableIndexSettings.SlowlogIndexingThresholdFetchDebug,
v => indexing.ThresholdDebug = v);
Set <TimeUnitExpression>(settings, UpdatableIndexSettings.SlowlogIndexingThresholdFetchTrace,
v => indexing.ThresholdTrace = v);
Set <LogLevel?>(settings, UpdatableIndexSettings.SlowlogIndexingLevel, v => indexing.LogLevel = v);
Set <int?>(settings, UpdatableIndexSettings.SlowlogIndexingSource, v => indexing.Source = v);
Set <int?>(settings, "index.number_of_shards", v => s.NumberOfShards = v);
Set <FileSystemStorageImplementation?>(settings, "index.store.type", v => s.FileSystemStorageImplementation = v,
serializer);
IDictionary dict = s;
foreach (var kv in settings)
{
var setting = kv.Value;
if (kv.Key == "analysis" || kv.Key == "index.analysis")
{
s.Analysis = setting.Value.Value <JObject>().ToObject <Analysis>(serializer);
}
else
{
dict?.Add(kv.Key, serializer.Deserialize(kv.Value.Value.CreateReader()));
}
}
}
