public ReservationIndexRepository(IElasticLowLevelClient client, ReservationsApiEnvironment environment, IElasticSearchQueries elasticQueries, ILogger <ReservationIndexRepository> logger)
{
_client = client;
_environment = environment;
_elasticQueries = elasticQueries;
_logger = logger;
}
protected ServerErrorTestsBase()
{
var settings = TestClient.GetFixedReturnSettings(ResponseJson, 500);
this.LowLevelClient = new ElasticLowLevelClient(settings);
this.HighLevelClient = new ElasticClient(settings);
}
public static void AddElasticsearch(this IServiceCollection services, IConfiguration configuration)
{
var url = configuration["elasticsearch:url"];
var defaultIndex = configuration["elasticsearch:index"];
_updateSearchService = Boolean.Parse(configuration["elasticsearch:updateIndex"]);
_settings = new ConnectionSettings(new Uri(url))
.DefaultIndex(defaultIndex)
.DefaultFieldNameInferrer(s => s)
.DefaultMappingFor <ComplexStringIndex>(m => m
.IndexName("complexstring")
.TypeName("complexstring")
)
.DefaultMappingFor <ProjectIndex>(m => m
.IndexName("project")
.TypeName("project")
);
_elasticClient = new ElasticClient(_settings);
services.AddSingleton <IElasticClient>(_elasticClient);
var settingslow = new ConnectionConfiguration(new Uri(url))
.RequestTimeout(TimeSpan.FromMinutes(2));
_lowlevelClient = new ElasticLowLevelClient(settingslow);
}
public ElasticReader(
IConnectionContext context,
Field[] fields,
IElasticLowLevelClient client,
IRowFactory rowFactory,
ReadFrom readFrom
)
{
_context = context;
_fields = fields;
_fieldNames = fields.Select(f => _readFrom == ReadFrom.Input ? f.Name : f.Alias.ToLower()).ToArray();
_client = client;
_rowFactory = rowFactory;
_readFrom = readFrom;
_typeName = readFrom == ReadFrom.Input ? context.Entity.Name : context.Entity.Alias.ToLower();
_context.Entity.ReadSize = _context.Entity.ReadSize == 0 ? DefaultSize : _context.Entity.ReadSize;
if (_context.Entity.ReadSize > ElasticsearchDefaultSizeLimit)
{
_context.Warn("Elasticsearch's default size limit is 10000. {0} may be too high.", _context.Entity.ReadSize);
}
_version = ElasticVersionParser.ParseVersion(_context);
}
public LogAutoscalingEvent(IElasticLowLevelClient es, IAmazonAutoScaling asg, LogToElasticsearch logToEs) : base(request =>
(from asgEvent in Observable.Return(request)
let asgName = asgEvent.SelectToken("detail.AutoScalingGroupName")?.Value <string>()
where !string.IsNullOrEmpty(asgName)
let describeRequest = new DescribeAutoScalingGroupsRequest
{
AutoScalingGroupNames = new List <string> {
asgName
}
}
from describeResponse in asg.DescribeAutoScalingGroupsAsync(describeRequest)
from description in describeResponse.AutoScalingGroups
let tags = description.Tags
let app = tags.GetValue("repo") ?? tags.GetValue("Application") ?? tags.GetValue("app")
let env = tags.GetValue("env") ?? tags.GetValue("Environment")
let sha = tags.GetValue("sha")
let log = new
{
Name = asgEvent["detail-type"]?.Value <string>(),
RequestId = asgName,
Message = asgEvent
}
let time = asgEvent["time"].Value <DateTime>()
let alteredLog = new AlteredLog
{
Time = time,
Repo = app,
Env = env,
Sha = sha,
Log = JObject.FromObject(log, AlteredJson.DefaultJsonSerializer)
}
from response in logToEs.Execute(alteredLog)
select response).ToTask())
{
}
protected override void InitializeTarget()
{
base.InitializeTarget();
var uri = ConnectionStringName.GetConnectionString() ?? Uri;
var nodes = uri.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries).Select(url => new Uri(url));
var connectionPool = new StaticConnectionPool(nodes);
var config = new ConnectionConfiguration(connectionPool);
if (RequireAuth)
{
config.BasicAuthentication(Username, Password);
}
if (ElasticsearchSerializer != null)
{
config = new ConnectionConfiguration(connectionPool, _ => ElasticsearchSerializer);
}
_client = new ElasticLowLevelClient(config);
if (!string.IsNullOrEmpty(ExcludedProperties))
{
_excludedProperties = ExcludedProperties.Split(new [] { ',' }, StringSplitOptions.RemoveEmptyEntries).ToList();
}
}
/// <summary>
/// This will rebuild an entire index alias behind the scenes with no downtime. It creates a new index
/// and populates it then uses Elasticsearch's hot-swapping technique to bring it online.
/// </summary>
public async static Task RebuildIndexWithHotSwapAsync(
this IElasticLowLevelClient client,
string alias,
JObject indexMapping,
Func <Task <IEnumerable <BulkIndexingDoc> > > reader,
CancellationToken ctx = default(CancellationToken))
{
var deletableIndices = JArray.Parse((await client.CatAliasesAsync <StringResponse>(alias, new CatAliasesRequestParameters {
Format = "json"
}, ctx)).Body)
.Select(x => new
{
alias = x.Value <string>("alias"),
index = x.Value <string>("index")
})
.ToList();
var index = GenerateUniqueIndexNameForAlias(alias);
await client.IndicesCreateAsync <StringResponse>(index, PostData.String(indexMapping?.ToString()), ctx : ctx);
while (!ctx.IsCancellationRequested)
{
// TODO: If an exception is thrown, delete the half-created index
var docs = await reader();
if (ctx.IsCancellationRequested || !docs.Any())
{
break;
}
var body = docs.SelectMany(doc => doc.ToLines().Select(x => x.ToString(Formatting.None)));
var bulkResponse = await client.BulkAsync <StringResponse>(index, PostData.MultiJson(body));
ThrowOnPartialBulkSuccess(bulkResponse);
}
if (ctx.IsCancellationRequested)
{
return;
}
var actions = deletableIndices.Select(idx => (object)new
{
remove = new { idx.index, idx.alias }
}).ToList();
actions.Add(new { add = new { index, alias } });
// This is the hot-swap. The actions in the list are performed atomically
await client.IndicesUpdateAliasesForAllAsync <StringResponse>(PostData.String(JObject.FromObject(new
{
actions
}).ToString()), ctx : ctx);
if (deletableIndices.Any())
{
await client.IndicesDeleteAsync <StringResponse>(string.Join(",", deletableIndices.Select(x => x.index)), ctx : ctx);
}
}
public ElClient(string elasticURL)
{
var uri = new Uri(elasticURL);
var settings = new ConnectionConfiguration(uri);
Client = new ElasticLowLevelClient(settings);
}
protected ServerErrorTestsBase()
{
var settings = FixedResponseClient.CreateConnectionSettings(ResponseJson, 500);
this.LowLevelClient = new ElasticLowLevelClient(settings);
this.HighLevelClient = new ElasticClient(settings);
}
private async Task CreateIndexIfNeededAsync(IElasticLowLevelClient client)
{
if (!_seederConfiguration.CreateIndexIfNotExists)
{
return;
}
StringResponse result = await client.Indices.ExistsAsync <StringResponse>(_elasticsearchConfiguration.IndexName);
if (result.HttpStatusCode == (int)HttpStatusCode.NotFound)
{
string json = await File.ReadAllTextAsync(_seederConfiguration.CreateIndexRequestBodyContentFilePath);
StringResponse response = await client.Indices.CreateAsync <StringResponse>(
_elasticsearchConfiguration.IndexName,
PostData.String(json));
if (!response.Success)
{
const string messageConst = "Unable to create index. Response from server: ";
ServerError error;
var message = response.TryGetServerError(out error)
? $"{messageConst}{error.Error.Reason}"
: $"{messageConst}{response.Body}";
throw new SystemException(message);
}
Log.Logger.Information($"Index '{_elasticsearchConfiguration.IndexName}' created");
}
else if (!result.Success)
{
Log.Logger.Warning($"Unable to check if index exists. Server response: [{result.HttpStatusCode}] {result.Body}");
}
}
/// <summary>
/// Constructor with ElasticsearchProcessorSettings.
/// </summary>
/// <param name="settings">Settings.</param>
public ElasticsearchProcessor(ElasticsearchProcessorSettings settings)
{
ConnectionConfiguration esSettings = new ConnectionConfiguration(settings.Address)
.RequestTimeout(TimeSpan.FromSeconds(settings.TimeOutInSeconds));
Client = new ElasticLowLevelClient(esSettings);
}
private void EnsureConnectionOpen()
{
if (_client == null)
{
var uri = ConnectionStringName.GetConnectionString() ?? Uri;
var nodes = uri.Render(new LogEventInfo()).Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries).Select(url => new Uri(url));
var connectionPool = new StaticConnectionPool(nodes);
var config =
new ConnectionSettings(connectionPool, sourceSerializer: (builtin, settings) => new JsonNetSerializer(
builtin, settings,
() => new JsonSerializerSettings
{
NullValueHandling = NullValueHandling.Include,
ReferenceLoopHandling = ReferenceLoopHandling.Ignore
},
resolver => resolver.NamingStrategy = new SnakeCaseNamingStrategy()
));
if (RequireAuth)
{
config.BasicAuthentication(Username, Password);
}
if (DisableAutomaticProxyDetection)
{
config.DisableAutomaticProxyDetection();
}
_client = new ElasticLowLevelClient(config);
}
}
public static void AddElasticSearch(this IServiceCollection services, IConfiguration config)
{
var url = config["elasticsearch:url"];
var defaultIndex = config["elasticsearch:index"];
IsElasticUsed = bool.Parse(config["elasticsearch:updateIndex"]);
_settings = new ConnectionSettings(new Uri(url))
.DefaultFieldNameInferrer(s => s)
.DefaultMappingFor <UserIndex>(m => m
.IndexName("user")
.TypeName("user")
);
_settings.EnableDebugMode();
if (!string.IsNullOrWhiteSpace(defaultIndex))
{
_settings.DefaultIndex(defaultIndex);
}
Client = new ElasticClient(_settings);
services.AddSingleton <IElasticClient>(Client);
services.AddTransient <ISearchService>(a => new SearchService(Client));
var settingslow = new ConnectionConfiguration(new Uri(url))
.RequestTimeout(TimeSpan.FromMinutes(2));
_lowlevelClient = new ElasticLowLevelClient(settingslow);
}
public ElasticPartialUpdater(OutputContext context, Configuration.Field[] fields, IElasticLowLevelClient client)
{
_context = context;
_fields = fields;
_client = client;
_index = context.Connection.Index;
_type = context.Entity.Alias.ToLower();
}
/// <summary>
///
/// </summary>
/// <param name="elasticLowLevelClient"></param>
/// <param name="cleanPayload"></param>
/// <param name="elasticOpType"></param>
public ElasticsearchLogClient(IElasticLowLevelClient elasticLowLevelClient,
Func <string, long?, string, string> cleanPayload,
ElasticOpType elasticOpType)
{
_elasticLowLevelClient = elasticLowLevelClient;
_cleanPayload = cleanPayload;
_elasticOpType = elasticOpType;
}
public ElasticAnalyticsService(IElasticLowLevelClient elasticClient,
IDateTimeProvider dateTimeProvider,
IWebHostEnvironment environment,
ILogger <ElasticAnalyticsService> logger)
{
_elasticClient = elasticClient;
_dateTimeProvider = dateTimeProvider;
_environment = environment;
_logger = logger;
}
public ElasticsearchSession(IElasticLowLevelClient elasticClient,
IUniqueNameResolver uniqueNameResolver,
IIdGenerator documentsIdGenerator,
IEsfStateInputValidator validator)
{
_elasticClient = elasticClient;
_indexName = uniqueNameResolver.GetUniqueName();
_typeName = uniqueNameResolver.GetUniqueName();
_documentsIdGenerator = documentsIdGenerator;
_validator = validator;
}
/// <summary>
/// Initializes a new instance of the <see cref="SearchService{T}"/> class.
/// </summary>
/// <param name="logger">The log handler for the controller.</param>
/// <param name="serviceConfig">The service config settings <see cref="SearchServiceConfig"/></param>
/// <param name="client">Interface to high level ElasticSearch client.</param>
/// <param name="lowLevelClient">Interface to low level ElasticSearch client.</param>
public SearchService(
ILogger logger,
SearchServiceConfig serviceConfig,
IElasticClient client,
IElasticLowLevelClient lowLevelClient)
{
_logger = logger;
_serviceConfig = serviceConfig;
_client = client;
_lowLevelClient = lowLevelClient;
}
public ElasticQueryReader(
InputContext context,
IElasticLowLevelClient client,
IRowFactory rowFactory
)
{
_context = context;
_client = client;
_rowFactory = rowFactory;
_fields = context.InputFields.ToDictionary(k => k.Name, v => v);
}
public ElasticsearchSessionFactory(
IElasticLowLevelClient elasticClient,
IUniqueNameResolver uniqueNameResolver,
IIdGenerator idGenerator,
IEsfStateInputValidator validator)
{
_elasticClient = elasticClient;
_uniqueNameResolver = uniqueNameResolver;
_idGenerator = idGenerator;
_validator = validator;
}
public ElasticClient(ITransport <IConnectionSettingsValues> transport)
{
transport.ThrowIfNull(nameof(transport));
transport.Settings.ThrowIfNull(nameof(transport.Settings));
transport.Settings.Serializer.ThrowIfNull(nameof(transport.Settings.Serializer));
transport.Settings.Inferrer.ThrowIfNull(nameof(transport.Settings.Inferrer));
this.Transport = transport;
this.LowLevel = new ElasticLowLevelClient(this.Transport);
this.LowLevelDispatch = new LowLevelDispatch(this.LowLevel);
}
/// <summary>
/// This performs a bulk command against the index pointed at by an alias. This is to avoid the potential for
/// a hotswap to change the underlying index from under us while we're reindexing. The index mapping is only
/// used if the aliased index does not yet exist.
/// </summary>
public async static Task BulkTargetingAliasAsync(
this IElasticLowLevelClient client,
string alias,
JObject indexMappingIfNotExists,
Func <Task <IEnumerable <BulkIndexingDoc> > > reader,
CancellationToken ctx = default(CancellationToken))
{
var targetIndices = JArray.Parse((await client.CatAliasesAsync <StringResponse>(alias, new CatAliasesRequestParameters {
Format = "json"
}, ctx)).Body)
.Select(x => x.Value <string>("index"))
.ToList();
string index;
if (targetIndices.Count == 0)
{
index = GenerateUniqueIndexNameForAlias(alias);
await client.IndicesCreateAsync <StringResponse>(index, PostData.String(indexMappingIfNotExists?.ToString()), ctx : ctx);
await client.IndicesUpdateAliasesForAllAsync <StringResponse>(PostData.String(JObject.FromObject(new
{
actions = new[] { new { add = new { index, alias } } }
}).ToString()), ctx : ctx);
}
else if (targetIndices.Count > 1)
{
throw new ArgumentException(
$"{nameof(BulkTargetingAliasAsync)} can only be used against an alias targeting a single index. The `{alias}` alias covers {targetIndices.Count} indices",
nameof(alias));
}
else
{
index = targetIndices.First();
}
while (!ctx.IsCancellationRequested)
{
var docs = await reader();
if (ctx.IsCancellationRequested || !docs.Any())
{
break;
}
var body = docs.SelectMany(doc => doc.ToLines().Select(x => x.ToString(Formatting.None)));
var bulkResponse = await client.BulkAsync <StringResponse>(index, PostData.MultiJson(body));
ThrowOnPartialBulkSuccess(bulkResponse);
}
}
public ElasticWriter(OutputContext context, IElasticLowLevelClient client)
{
_context = context;
_client = client;
_prefix = "{\"index\": {\"_index\": \"" + context.Connection.Index + "\", \"_type\": \"" + context.Entity.Alias.ToLower() + "\", \"_id\": \"";
_fields = context.OutputFields.Select(f => new AliasField {
Alias = f.Alias.ToLower(), Field = f
}).ToArray();
_settings = new JsonSerializerSettings {
Formatting = Formatting.None,
ContractResolver = new DefaultContractResolver()
};
}
public LogToElasticsearch(IElasticLowLevelClient es) : base(async(logBatch) =>
{
var multiJson = logBatch.ToEsMultiJson();
//let _ = Write(multiJson)
var bulkRequest = PostData.String(multiJson);
var bulkRequestParameters = new BulkRequestParameters
{
// todo
};
var bulkResponse = await es.BulkAsync <StringResponse>(bulkRequest, bulkRequestParameters);
return(bulkResponse);
})
{
}
/// <summary>
/// Initializes the current instance.
/// </summary>
protected override void InitializeTarget()
{
base.InitializeTarget();
var uri = Uri;
var nodes = uri.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries).Select(url => new Uri(url));
var connectionPool = new StaticConnectionPool(nodes);
var config = new ConnectionConfiguration(connectionPool);
config.DisableAutomaticProxyDetection();
_client = new ElasticLowLevelClient(config);
}
public StoreToElasticsearch(IElasticLowLevelClient es, string indexName = "terraform") : base(async(request) =>
{
var logBatch = JObject.FromObject(request.Data, AlteredJson.DefaultJsonSerializer);
var multiJson = logBatch.ToEsMultiJson(indexName);
//let _ = Write(multiJson)
var bulkRequest = PostData.String(multiJson);
var bulkRequestParameters = new BulkRequestParameters
{
// todo
};
var bulkResponse = await es.BulkAsync <StringResponse>(bulkRequest, bulkRequestParameters);
return(bulkResponse);
})
{
}
public ElasticOutputController(
OutputContext context,
IAction initializer,
IVersionDetector inputVersionDetector,
IVersionDetector outputVersionDetector,
IElasticLowLevelClient client
) : base(
context,
initializer,
inputVersionDetector,
outputVersionDetector
)
{
_client = client;
_stopWatch = new Stopwatch();
}
public ElasticReader(
IConnectionContext context,
Field[] fields,
IElasticLowLevelClient client,
IRowFactory rowFactory,
ReadFrom readFrom
)
{
_context = context;
_fields = fields;
_fieldNames = fields.Select(f => _readFrom == ReadFrom.Input ? f.Name : f.Alias.ToLower()).ToArray();
_client = client;
_rowFactory = rowFactory;
_readFrom = readFrom;
_typeName = readFrom == ReadFrom.Input ? context.Entity.Name : context.Entity.Alias.ToLower();
}
public ElasticOutputController(
OutputContext context,
IAction initializer,
IInputProvider inputProvider,
IOutputProvider outputProvider,
IElasticLowLevelClient client
) : base(
context,
initializer,
inputProvider,
outputProvider
)
{
_client = client;
_stopWatch = new Stopwatch();
}
private async Task IndexProductAsync(ProductForIndex productForIndex, IElasticLowLevelClient client)
{
string insertData = JsonConvert.SerializeObject(productForIndex);
Log.Logger.Information($"Inserting product: [{productForIndex.SearchResultData.Id}]' {productForIndex.SearchResultData.Name}'");
StringResponse response = await client.IndexAsync <StringResponse>(
_elasticsearchConfiguration.IndexName,
productForIndex.SearchResultData.Id.ToString(),
PostData.String(insertData));
if (response.Success)
{
Log.Logger.Information($"Product [{productForIndex.SearchResultData.Id}]' {productForIndex.SearchResultData.Name}' successfuly indexed");
}
else
{
Log.Logger.Error($"Unable to index product (ID: {productForIndex.SearchResultData.Id}): {response.Body}");
}
}
public LowLevelDispatch(IElasticLowLevelClient rawElasticClient)
{
this._lowLevel = rawElasticClient;
}
