private static List <Func <object> > CatCommands(ElasticClient elastic)
{
return(new List <Func <object> >
{
() => elastic.CatAliases(new CatAliasesRequest()),
() => elastic.CatAllocation(new CatAllocationRequest()),
() => elastic.CatCount(new CatCountRequest()),
() => elastic.CatFielddata(new CatFielddataRequest()),
() => elastic.CatHealth(new CatHealthRequest()),
() => elastic.CatHelp(new CatHelpRequest()),
() => elastic.CatIndices(new CatIndicesRequest()),
() => elastic.CatMaster(new CatMasterRequest()),
() => elastic.CatNodeAttributes(new CatNodeAttributesRequest()),
() => elastic.CatNodes(new CatNodesRequest()),
() => elastic.CatPendingTasks(new CatPendingTasksRequest()),
() => elastic.CatPlugins(new CatPluginsRequest()),
() => elastic.CatRecovery(new CatRecoveryRequest()),
() => elastic.CatRepositories(new CatRepositoriesRequest()),
() => elastic.CatSegments(new CatSegmentsRequest()),
() => elastic.CatShards(new CatShardsRequest()),
// CatSnapshots is failing with a JSON deserialization error.
// It might be a bug in the client or an incompatibility between client
// and server versions.
// () => elastic.CatSnapshots(new CatSnapshotsRequest()),
() => elastic.CatTasks(new CatTasksRequest()),
() => elastic.CatTemplates(new CatTemplatesRequest()),
() => elastic.CatThreadPool(new CatThreadPoolRequest()),
});
}
private string GetIndexName(IDatabaseInformation databaseInformation)
{
if (!databaseInformation.DatabaseName.Contains("{"))
{
indexName = databaseInformation.DatabaseName;
}
else
{
var dbInformationList = databaseInformation.DatabaseName.Split('{');
var indexNumber = Convert.ToInt32(dbInformationList[1].Replace("{", string.Empty).Replace("}", string.Empty).Trim());
var indexRecord = client.CatIndices().Records
.Where(r => r.Index.Contains(dbInformationList[0]))
.OrderByDescending(r => Convert.ToInt32(r.Index.Split('_').LastOrDefault()))
.Skip(indexNumber)
.FirstOrDefault();
if (indexRecord == null)
{
throw new Exception($"Cannot gotton index name!({databaseInformation.DatabaseName})");
}
indexName = indexRecord.Index;
}
return(indexName);
}
private static List <Func <object> > CatCommands(ElasticClient elastic)
{
return(new List <Func <object> >
{
() => elastic.CatAliases(new CatAliasesRequest()),
() => elastic.CatAllocation(new CatAllocationRequest()),
() => elastic.CatCount(new CatCountRequest()),
() => elastic.CatFielddata(new CatFielddataRequest()),
() => elastic.CatHealth(new CatHealthRequest()),
() => elastic.CatHelp(new CatHelpRequest()),
() => elastic.CatIndices(new CatIndicesRequest()),
() => elastic.CatMaster(new CatMasterRequest()),
() => elastic.CatNodeAttributes(new CatNodeAttributesRequest()),
() => elastic.CatNodes(new CatNodesRequest()),
() => elastic.CatPendingTasks(new CatPendingTasksRequest()),
() => elastic.CatPlugins(new CatPluginsRequest()),
() => elastic.CatRecovery(new CatRecoveryRequest()),
() => elastic.CatRepositories(new CatRepositoriesRequest()),
() => elastic.CatSegments(new CatSegmentsRequest()),
() => elastic.CatShards(new CatShardsRequest()),
() => elastic.CatSnapshots(new CatSnapshotsRequest()),
() => elastic.CatTasks(new CatTasksRequest()),
() => elastic.CatTemplates(new CatTemplatesRequest()),
() => elastic.CatThreadPool(new CatThreadPoolRequest()),
});
}
public List <IndexStats> GetAllIndices(string prefix)
{
var allIndices = _client.CatIndices(x => x.AllIndices());
var myIndices = allIndices.Records.Where(x => x.Index.StartsWith(prefix)).Select(x => new IndexStats {
Name = x.Index, Count = x.DocsCount
}).ToList();
return(myIndices);
}
public ElasticsearchResponse GetElasticHealth(IEnumerable <Uri> uris, IEnumerable <string> requiredAliases, string environment)
{
try
{
using (var connectionSettings = new ConnectionSettings(new StaticConnectionPool(uris)))
{
var elasticClient = new ElasticClient(connectionSettings);
var catAliases = elasticClient
.CatAliases()
.Records
.Where(m => m.Alias.StartsWith(environment)).ToList();
var rawAliases = catAliases.Select(al => al.Alias.Replace(environment, string.Empty));
var aliases = new List <ElasticsearchAlias>();
foreach (var catAlias in catAliases)
{
var index = elasticClient.CatIndices(m => m.Index(catAlias.Index));
aliases.Add(new ElasticsearchAlias
{
AliasName = catAlias.Alias,
IndexName = catAlias.Index,
Status = index.Records.FirstOrDefault()?.Status,
DocumentCount = index.Records.FirstOrDefault()?.DocsCount,
Health = index.Records.FirstOrDefault()?.Health
});
}
return(new ElasticsearchResponse
{
RequiredAliasesExist = requiredAliases.All(al => rawAliases.Contains(al)),
ElasticsearchAliases = aliases
});
}
}
catch (Exception exception)
{
_logger.Warn(exception, "Not able to get aliases");
return(new ElasticsearchResponse {
ElasticsearchAliases = new List <ElasticsearchAlias>(), Exception = exception
});
}
}
public Dictionary <string, TeamData> GetData()
{
//Obtain all indices and convert respoonse into List of index records...
var catIndicesResponse = client.CatIndices();
var listIndicesRecord = catIndicesResponse.Records.ToList();
IndexRecordParser indexParser = new IndexRecordParser();
//Parse each index record into our dictionary
foreach (CatIndicesRecord indexRecord in listIndicesRecord)
{
teamData = indexParser.ParseIndexRecordIntoDict(teamData, indexRecord);
}
//TODO: add json serialize and adding file to s3 bucket "datalens-leaderboards"
return(teamData);
}
public static void Main()
{
var uri = new Uri("http://log02.corp.monsenso.com:9200");
var settings = new ConnectionSettings(uri);
settings.DisableDirectStreaming();
var client = new ElasticClient(settings);
var passwords = 0;
var authHeaders = 0;
foreach (var idx in client.CatIndices().Records)
{
var idxName = idx.Index;
// var idxName = "logstash-2018.02.09";
System.Console.WriteLine(idxName);
var results =
client.Search <dynamic>(s => s
.Index(idxName)
.Size(2000)
.Type("logevent")
.Query(q =>
(q.Exists(e => e.Field("fields.RequestForm.password")) &&
!q.Match(m => m.Field("fields.RequestForm.password").Query("*** Redacted ***"))
)
||
(q.Exists(e => e.Field("fields.RequestHeaders.Authorization")) &&
!q.Match(m => m.Field("fields.RequestHeaders.Authorization").Query("*** Redacted ***"))
)
)
);
// System.Console.WriteLine(results.DebugInformation);
// return;
System.Console.WriteLine(results.Total);
foreach (var hit in results.Hits)
{
var path = (DocumentPath <dynamic>)hit.Id;
path.Type("logevent");
var doc = client.Get <dynamic>(path, x => x.Index(idxName));
if (doc.Source == null)
{
System.Console.WriteLine("null!!!");
continue;
}
// var jsonString = JsonConvert.SerializeObject(
// doc,
// new JsonSerializerSettings
// {
// Converters = new List<JsonConverter> { new StringEnumConverter() },
// ContractResolver = new ElasticContractResolver(settings, new List<Func<Type, JsonConverter>>())
// });
// System.Console.WriteLine(jsonString);
bool update = false;
if (doc.Source.fields.RequestForm?.password != null &&
doc.Source.fields.RequestForm?.password != "*** Redacted ***")
{
update = true;
passwords++;
doc.Source.fields.RequestForm.password = "******";
}
if (doc.Source.fields.RequestHeaders?.Authorization != null &&
doc.Source.fields.RequestHeaders?.Authorization != "*** Redacted ***")
{
update = true;
authHeaders++;
doc.Source.fields.RequestHeaders.Authorization = "*** Redacted ***";
}
if (update)
{
System.Console.WriteLine(hit.Id);
client.Update <dynamic>(path, u => u.Doc(doc.Source).Index(idxName));
}
}
}
System.Console.WriteLine("Passwords removed: " + passwords);
System.Console.WriteLine("Authorization headers removed: " + authHeaders);
}
public IActionResult Index()
{
var indexesNames = _client.CatIndices().Records.Select(x => x.Index);
return(View(indexesNames));
}
public ICatResponse <CatIndicesRecord> IndexList()
{
var request = new CatIndicesRequest();
return(client.CatIndices(request));
}
public void Restore(string backupFile, string targetUrl)
{
if (!string.IsNullOrWhiteSpace(backupFile))
{
if (!File.Exists(backupFile))
{
Logger.Error("No data will be restored, file not found: {path}", backupFile);
return;
}
var node = new Uri(targetUrl);
var settings = new ConnectionSettings(node);
var client = new ElasticClient(settings);
var al = client.CatIndices();
if (al.Records.Any(x => string.Equals(x.Index, "data", StringComparison.InvariantCultureIgnoreCase)))
{
Logger.Information("Deleting index {index}", "data");
var deleteResponse = client.DeleteIndex(new DeleteIndexRequest("data"));
if (!deleteResponse.IsValid)
{
Logger.Error(deleteResponse.OriginalException, "Delete index failed: {@error}", deleteResponse.ServerError.Error);
}
}
Logger.Information("Creating index {index}", "data");
var createResponse = client.CreateIndex("data",
d => d.Settings(
s => s
.NumberOfShards(1)
.Analysis(a => a
.TokenFilters(x => x.EdgeNGram("autocomplete_filter", n => n.MinGram(1).MaxGram(20)))
.Analyzers(an => an
.Custom("autocomplete", c => c
.Tokenizer("standard")
.Filters("lowercase", "autocomplete_filter"))))));
Logger.Information("Mapping autocomplete analyzer for {property}", "full_address_line");
var mapResponse = client.Map <dynamic>(x => x.Properties(p => p.String(s => s.Name("full_address_line").Analyzer("autocomplete"))));
if (!mapResponse.IsValid)
{
Logger.Error(mapResponse.OriginalException, "Map failed: {@error}", mapResponse.ServerError.Error);
}
if (!createResponse.IsValid)
{
Logger.Error(createResponse.OriginalException, "Create index failed: {@error}", createResponse.ServerError.Error);
}
using (Logger.BeginTimedOperation("Bulk data indexing"))
using (var file = MemoryMappedFile.CreateFromFile(backupFile, FileMode.Open))
using (var view = file.CreateViewStream())
using (var reader = new StreamReader(view))
{
var total = 0;
var tasks = new List <Task>();
var buffer = new List <string>(NumItemsInBulkGroup);
while (!reader.EndOfStream)
{
var line = reader.ReadLine();
buffer.Add(line);
if (buffer.Count < NumItemsInBulkGroup)
{
continue;
}
total += buffer.Count;
var capture = buffer;
buffer = new List <string>();
tasks.Add(Task.Factory.StartNew(() => Push(client, capture)));
}
if (buffer.Count > 0)
{
total += buffer.Count;
tasks.Add(Task.Factory.StartNew(() => Push(client, buffer)));
}
Task.WaitAll(tasks.ToArray());
Logger.Information("Total {count} records has been indexed", total);
}
}
}