protected async Task SetupOnce()
{
Elasticsearch = await new Elasticsearch().Ready();
ElasticClient = new ElasticClient(new ConnectionSettings(Elasticsearch.Url));
AppSettingsOptionsMock = new Mock <IOptions <AppSettings> >();
AppSettingsOptionsMock.Setup(o => o.Value).Returns(new AppSettings
{
AdminUser = "******"
});
EFContext = EFContextFactory.CreateInMemoryEFContext();
MapperMock = new Mock <IMapper>();
LoggerMock = new Mock <ILogger>();
CostSearchBuilderMock = new Mock <ICostSearchBuilder>();
ElasticSearchIndexService = new ElasticSearchIndexService(
new[]
{
new Lazy <ICostSearchBuilder, PluginMetadata>(() => CostSearchBuilderMock.Object, new PluginMetadata {
BuType = BuType.Pg
})
},
LoggerMock.Object,
EFContext,
MapperMock.Object,
ElasticClient,
AppSettingsOptionsMock.Object
);
await OneTimeSetup();
await CreateIndexes();
}
private string GetIndexs(Elasticsearch input)
{
var timeRange = input.Query.Bool.Must.FirstOrDefault(
x => x.Range?.Timestamp?.Gt != null && x.Range?.Timestamp?.Lt != null
).Range.Timestamp;
if (timeRange == null ||
(timeRange.Gt - timeRange.Lt).Value.TotalDays > 10)
{
return($"logstash-.*");
}
var start = timeRange.Gt.Value.DateTime;
var end = timeRange.Lt.Value.DateTime;
var index = string.Empty;
var tempDate = start.Date;
do
{
index += $"logstash-{tempDate:yyyy.MM.dd},";
tempDate = tempDate.AddDays(1);
}while (tempDate <= end.Date && tempDate <= DateTime.Today);
return(index.TrimEnd(','));
}
public async Task Can_log_output()
{
var logged = false;
using (var elasticsearch = new Elasticsearch(c => c.EnableLogging().LogTo(message => logged = true)))
{
await elasticsearch.Ready();
////Assert
Assert.That(logged);
}
}
public void Has_specific_version()
{
using (var elasticsearch = new Elasticsearch(i => i.SetPort(4444).EnableLogging().SetElasticsearchStartTimeout(60)).ReadySync())
{
////Arrange
var client = new ElasticClient(new ConnectionSettings(elasticsearch.Url));
////Act
var version = client.RootNodeInfo().Version.Number;
////Assert
Assert.That(version, Is.EqualTo("7.1.0"));
}
}
public void Can_start()
{
using (var elasticsearch = new Elasticsearch())
{
////Arrange
var client = new ElasticClient(new ConnectionSettings(elasticsearch.Url));
////Act
var result = client.Ping();
////Assert
Assert.That(result.IsValid);
}
}
public void Can_start()
{
using (var elasticsearch = new Elasticsearch())
{
////Arrange
var client = new ElasticClient(new ConnectionSettings(elasticsearch.Url));
////Act
var result = client.Ping();
////Assert
Assert.That(result.IsValid);
}
}
public void Can_start_with_persistent()
{
using (var elasticsearch = new Elasticsearch(c => c.RootFolder(@"c:\\temp\\es")))
{
////Arrange
var client = new ElasticClient(new ConnectionSettings(elasticsearch.Url));
////Act
var result = client.Ping();
////Assert
Assert.That(result.IsValid);
}
}
public void Can_start_sync()
{
using (var elasticsearch = new Elasticsearch(i => i.SetPort(4444).EnableLogging().SetElasticsearchStartTimeout(60)).ReadySync())
{
////Arrange
var client = new ElasticClient(new ConnectionSettings(elasticsearch.Url));
////Act
var result = client.Ping();
////Assert
Assert.That(result.IsValid);
}
}
private async Task <string> ToSearchAsync(Elasticsearch input)
{
var index = GetIndexs(input);
var request = input.ToJson();
var url = $"{_elasticsearchPath}/{index}/logevent/_search";
_log.LogInformation("Send {request} To {url}", new object[] { request, url });
var response = await $"{_elasticsearchPath}/{index}/logevent/_search"
.WithHeader("Content-Type", "application/json")
.PostStringAsync(input.ToJson());
return(await response.Content.ReadAsStringAsync());
}
public void Can_change_configuration()
{
using (var elasticsearch = new Elasticsearch(c => c.Port(444).EnableLogging()))
{
////Arrange
var client = new ElasticClient(new ConnectionSettings(elasticsearch.Url));
////Act
var result = client.Ping();
////Assert
Assert.That(result.IsValid);
Assert.That(elasticsearch.Url.Port, Is.EqualTo(444));
}
}
public void Can_change_configuration()
{
using (var elasticsearch = new Elasticsearch(c => c.Port(444).EnableLogging()))
{
////Arrange
var client = new ElasticClient(new ConnectionSettings(elasticsearch.Url));
////Act
var result = client.Ping();
////Assert
Assert.That(result.IsValid);
Assert.That(elasticsearch.Url.Port, Is.EqualTo(444));
}
}
static void Main(string[] args)
{
Elasticsearch objSearch = new Elasticsearch();
objSearch.AddNewIndex(new City()
{
ID = 1, Name = "delhi", State = "delhi", Country = "India", Population = "9.879 million"
});
objSearch.AddNewIndex(new City()
{
ID = 2, Name = "mumbai", State = "Maharashtra", Country = "India", Population = "11.98 million"
});
var result = objSearch.GetResult();
}
public void Can_insert_data()
{
using (var elasticsearch = new Elasticsearch())
{
////Arrange
var client = new ElasticClient(new ConnectionSettings(elasticsearch.Url));
////Act
client.Index(new { id = "tester"}, i => i.Index("test-index"));
////Assert
var result = client.Get<dynamic>("tester", "test-index");
Assert.That(result, Is.Not.Null);
}
}
public void Can_insert_data()
{
using (var elasticsearch = new Elasticsearch())
{
////Arrange
var client = new ElasticClient(new ConnectionSettings(elasticsearch.Url));
////Act
client.Index(new { id = "tester" }, i => i.Index("test-index").Type("test-type"));
////Assert
var result = client.Get <dynamic>("tester", "test-index", "test-type");
Assert.That(result, Is.Not.Null);
Assert.That(result.Found);
}
}
public async Task Init()
{
_costSearchBuilderMock = new Mock <ICostSearchBuilder>();
_loggerMock = new Mock <ILogger>();
_mapperMock = new Mock <IMapper>();
_appSettingsMock = new Mock <IOptions <AppSettings> >();
_efContext = EFContextFactory.CreateInMemoryEFContext();
_pluginMetadataServices = new[]
{
new Lazy <ICostSearchBuilder, PluginMetadata>(() => _costSearchBuilderMock.Object, new PluginMetadata {
BuType = BuType.Pg
})
};
_elasticsearch = new Elasticsearch();
await _elasticsearch.Ready();
_elasticClient = new ElasticClient(new ConnectionSettings(_elasticsearch.Url));
_elasticSearchIndexService = new ElasticSearchIndexService(
_pluginMetadataServices, _loggerMock.Object, _efContext, _mapperMock.Object, _elasticClient, _appSettingsMock.Object);
}
public void Can_install_plugin()
{
string pluginName = "mobz/elasticsearch-head";
using (var elasticsearch = new Elasticsearch(c => c.AddPlugin(new Configuration.Plugin(pluginName))))
{
////Arrange
var client = new ElasticClient(new ConnectionSettings(elasticsearch.Url));
////Act
var result = client.CatPlugins();
int pluginCount = 0;
foreach (CatPluginsRecord plugin in result.Records)
{
pluginCount++;
}
////Assert
Assert.That(result.IsValid);
Assert.AreEqual(1, pluginCount);
}
}
public void Can_install_plugin_url()
{
string pluginName = "test_plugin_135076"; // random numbers to make sure the name doesn't conflict with publicly available plugins
string pluginUrl = "file:///" + Directory.GetCurrentDirectory() + "/TestFiles/test_plugin_135076.zip";
using (var elasticsearch = new Elasticsearch(c => c.AddPlugin(new Configuration.Plugin(pluginName, pluginUrl))))
{
////Arrange
var client = new ElasticClient(new ConnectionSettings(elasticsearch.Url));
////Act
var result = client.CatPlugins();
int pluginCount = 0;
foreach (CatPluginsRecord plugin in result.Records)
{
pluginCount++;
}
////Assert
Assert.That(result.IsValid);
Assert.AreEqual(1, pluginCount);
}
}
private Elasticsearch GetElasticsearchByInput(LogFiltrateInputDto input)
{
var size = (input.PageIndex - 1) * input.PageSize;
if (size < 0)
{
size = 0;
}
var search = new Elasticsearch
{
Query = new ElasticsearchQuery
{
Bool = new ElasticsearchBool
{
Must = new List <ElasticsearchMust>()
}
},
Size = input.PageSize,
From = size
};
DateTime?start = null, end = null;
if (input.Times?.Length > 1)
{
start = input.Times[0].DateTime;
end = input.Times[1].DateTime;
}
else if (input.TimeSelect > 0)
{
start = DateTime.Now.AddMinutes(-input.TimeSelect);
end = DateTime.Now;
}
if (start != null && end != null)
{
search.Query.Bool.Must.Add(new ElasticsearchMust
{
Range = new ElasticsearchRange
{
Timestamp = new ElasticsearchTimestamp
{
Gt = start.Value,
Lt = end.Value
}
}
});
}
else
{
throw new Exception("plase select time range");
}
if (!string.IsNullOrWhiteSpace(input.Lv))
{
search.Query.Bool.Must.Add(new ElasticsearchMust
{
Term = new ElasticsearchTerm
{
LevelKeyword = input.Lv
}
});
}
if (!string.IsNullOrWhiteSpace(input.RequestId))
{
search.Query.Bool.Must.Add(new ElasticsearchMust
{
Term = new ElasticsearchTerm
{
FieldsRequestIdKeyword = input.RequestId
}
});
}
return(search);
}
public async Task <(ElasticsearchBucket[] messageTemplate, ElasticsearchBucket[] context, ElasticsearchBucket[] requestPath)> GetAggregation(Elasticsearch input)
{
var count = 0;
Again:
var resultStr = await ToSearchAsync(input);
var result = JsonConvert.DeserializeObject <ElasticsearchAggregations>(resultStr);
if (result.Shards != null && result.Shards.Failed > 0)
{
foreach (var item in result.Shards.Failures)
{
var res = await $"{_elasticsearchPath}/{item.Index}/_mapping/logevent"
.WithHeader("Content-Type", "application/json")
.PostJsonAsync(new
{
properties = input.Aggs.ToDictionary(
x =>
{
var arr = x.Value.Terms.Field.Split('.');
if (arr.LastOrDefault() == "keyword")
{
arr = arr.Take(arr.Length - 1).ToArray();
}
return(string.Join(".", arr));
}, x => new
{
type = "text",
fielddata = true
}
)
});
_log.LogWarning("Set Es Mapping {Index} {Response}", new object[]
{
item.Index,
await res.Content.ReadAsStringAsync()
});
if (count < 3) // 重新刷数据
{
count++;
goto Again;
}
}
}
var requestPaths = result.Aggregations["aggRequestPath"]
.Buckets
.Select(x =>
{
if (x.Key.StartsWith("/api/"))
{
x.Key = x.Key.Replace("/api/", ""); // 移除统一前缀
}
return(x);
})
.ToArray();
return(
null, // result.Aggregations["aggMessageTemplate"].Buckets, 暂不查询
result.Aggregations["aggSourceContext"].Buckets,
requestPaths
);
}
public async Task <string> GetSearch(Elasticsearch input)
{
return(await ToSearchAsync(input));
}
public async Task Can_insert_data_with_persistent()
{
var nodeName = @"data_with_persistent_" + DateTime.Now.Millisecond;
var clusterName = @"cluster_data_with_persistent_" + DateTime.Now.Millisecond;;
const string FOLDER = @"c:\\temp\es004";
const string INDEX = "test-index";
const string TYPE = "test-type";
const int PORT = 9205;
var id = Guid.NewGuid().ToString();
if (Directory.Exists(FOLDER))
{
Directory.Delete(FOLDER, true);
}
using (var elasticsearch = new Elasticsearch(c => c
.SetNodeName(nodeName)
.SetClusterName(clusterName)
.Port(PORT)
.RootFolder(FOLDER)
.OverwriteRootFolder()
))
{
////Arrange
var client = new ElasticClient(new ConnectionSettings(elasticsearch.Url));
////Act
client.Index(new { id }, i => i.Index(INDEX).Type(TYPE));
////Assert
var result = client.Get <dynamic>(id, INDEX, TYPE);
Assert.That(result, Is.Not.Null);
Assert.That(result.Found);
}
Console.WriteLine("Shutting down the 1st instance");
var count = 1;
while (count > 0)
{
Console.Write(".");
count = Process.GetProcessesByName("java").Length;
await Task.Delay(500);
}
Console.WriteLine();
using (var elasticsearch = new Elasticsearch(c => c
.Port(PORT)
.RootFolder(FOLDER)
.SetNodeName(nodeName)
.SetClusterName(clusterName)
))
{
Console.WriteLine("2nd instance started");
////Arrange
var client = new ElasticClient(new ConnectionSettings(elasticsearch.Url));
////Assert
var result = client.Get <dynamic>(id, INDEX, TYPE);
Assert.That(result, Is.Not.Null);
Assert.That(result.Found);
}
}
