public DataTable GetDriverElasticQuery(String w_clause)
{
ElasticClient es = new ElasticClient();
es = EsClient();
QueryContainer ContQry = new QueryContainer();
MatchAllQuery Qry1 = new MatchAllQuery();
Qry1.Name = "Qry1";
ContQry = Qry1;
SearchRequest esserach = new SearchRequest();
esserach.Query = ContQry;
//
// SearchDescriptor<Device> esdesct = new SearchDescriptor<Device>();
// var esresponse = new SearchResponse<Driver>();
var esresponse = es.Search <Device>(esserach);
DataTable dt = new DataTable();
if (esresponse.Documents.Count > 0)
{
dt = ListToDataTable <Device>(esresponse.Documents.ToList());
}
return(dt);
}
public virtual SearchDescriptor <T> GetQueryFor <U>(Q.Query <U> query) where U : class, new()
{
//var filterContainer = new FilterContainer();
//if ( query.Facets!= null && query.Facets.Count > 0)
//{
// filterContainer = query.Facets.Aggregate(filterContainer,
// (current, facet) => current & Filter<T>.Term(facet.Key, facet.Value));
//}
QueryContainer queryContainer = new MatchAllQuery();
if (query.Criterion != null && query.Criterion.Count > 0)
{
queryContainer = query.Criterion.Aggregate(queryContainer, (current, field) => current & new TermQuery
{
Field = field.Key,
Value = field.Value
});
}
var searchDescriptor = new SearchDescriptor <T>()
.Query(q => queryContainer)
.Size(query.PageSize)
.From(query.PageIndex);
return(searchDescriptor);
}
/// <summary>
/// Find the timestamp of the most recent record for each device.
/// </summary>
/// <returns>The high water mark.</returns>
/// <param name="indexName">IndexName</param>
/// <param name="query">Term Query to select records</param>
public static T getHighWaterMark <T>(ElasticClient client, string indexName, QueryContainer query) where T : class
{
if (query == null)
{
query = new MatchAllQuery();
}
SearchRequest req = new SearchRequest(indexName)
{
From = 0,
Size = 1,
Query = query,
Sort = new List <ISort> {
new SortField {
Field = "timestamp", Order = SortOrder.Descending
}
}
};
var searchResponse = client.Search <T>(req);
if (searchResponse.Documents.Count() > 1)
{
throw new ArgumentOutOfRangeException("More than one record returned from high-watermark");
}
var mostRecent = searchResponse.Documents.FirstOrDefault();
return(mostRecent);
}
public Tuple <string, List <BoardGame> > QueryData_Buckets(string query)
{
_elasticClient.Flush("boardgames");
var queryContainer = new QueryContainer();
queryContainer &= new MatchAllQuery();
var aggregation = new TermsAggregation("category_aggregation")
{
Field = Infer.Field <BoardGame>(p => p.Category),
};
var searchRequest = new SearchRequest("boardgames", typeof(BoardGame))
{
Query = queryContainer,
Aggregations = aggregation
};
var searchResponse = _elasticClient.Search <BoardGame>(searchRequest);
var categories = searchResponse.Aggregations.Terms("category_aggregation").Buckets;
return(new Tuple <string, List <BoardGame> >(
searchResponse.DebugInformation,
searchResponse.Documents.ToList()
));
}
public Tuple <string, List <BoardGame> > QueryData_MinPlayTime(string query)
{
_elasticClient.Flush("boardgames");
var queryContainer = new QueryContainer();
queryContainer &= new MatchAllQuery();
var aggregation = new MinAggregation("min_playing_time_aggregation",
Infer.Field <BoardGame>(p => p.PlayingTime));
var searchRequest = new SearchRequest("boardgames", typeof(BoardGame))
{
Query = queryContainer,
Aggregations = aggregation
};
var searchResponse = _elasticClient.Search <BoardGame>(searchRequest);
var minPlayingTime = searchResponse.Aggregations.Min("min_playing_time_aggregation");
return(new Tuple <string, List <BoardGame> >(
searchResponse.DebugInformation,
searchResponse.Documents.ToList()
));
}
protected ConcurrentDictionary <string, T> GetDicObjectScroll <T>(string _DefaultIndex, QueryContainer query, SourceFilter so) where T : class
{
try
{
ConcurrentDictionary <string, T> bag = new ConcurrentDictionary <string, T>();
string scroll_timeout = "5m";
int scroll_pageize = 2000;
if (query == null)
{
query = new MatchAllQuery();
}
if (so == null)
{
so = new SourceFilter()
{
}
}
;
try
{
var searchResponse = client.Search <T>(sd => sd.Source(s => so).Index(_DefaultIndex).From(0).Take(scroll_pageize).Query(q => query).Scroll(scroll_timeout));
while (true)
{
if (!searchResponse.IsValid || string.IsNullOrEmpty(searchResponse.ScrollId))
{
break;
}
if (!searchResponse.Documents.Any())
{
break;
}
var tmp = searchResponse.Hits;
foreach (var item in tmp)
{
bag.TryAdd(item.Id, item.Source);
}
searchResponse = client.Scroll <T>(scroll_timeout, searchResponse.ScrollId);
}
client.ClearScroll(new ClearScrollRequest(searchResponse.ScrollId));
}
catch (Exception)
{
}
finally
{
}
return(bag);
}
catch (Exception ex)
{
}
return(null);
}
public ISearchResponse <Book> SearchBookWithAggregation(string criteria = "", string searchField = "", List <string> refinements = null, int count = 10)
{
QueryContainer _query;
if (String.IsNullOrEmpty(criteria))
{
_query = new MatchAllQuery();
}
else
{
if (String.IsNullOrEmpty(searchField))
{
searchField = "_all";
_query = new MatchQuery()
{
Field = searchField,
Query = criteria
};
}
else
{
_query = new TermQuery()
{
Field = searchField,
Value = criteria
};
}
}
SearchRequest searchRequest = new SearchRequest
{
From = 0,
Size = count,
Query = _query
};
if (refinements != null && refinements.Count > 0)
{
var _aggregations = new Dictionary <string, IAggregationContainer>();
foreach (var field in refinements)
{
_aggregations.Add(field, new AggregationContainer
{
Terms = new TermsAggregator
{
Field = field
}
});
}
searchRequest.Aggregations = _aggregations;
}
return(esClient.Search <Book>(searchRequest));
}
public void Export_Returns_Valid_Json()
{
var query = new MatchAllQuery();
var expected = JsonConvert.SerializeObject(new
{
match_all = (string)null
}, Formatting.None);
Assert.Equal(expected, query.Export().ToString(Formatting.None));
}
public async Task <ConcurrentBag <IHit <T> > > BackupAndScroll <T>(string _default_index, QueryContainer query, SourceFilter so, string scroll_timeout = "5m", int scroll_pageize = 2000) where T : class
{
if (query == null)
{
query = new MatchAllQuery();
}
if (so == null)
{
so = new SourceFilter()
{
}
}
;
ConcurrentBag <IHit <T> > bag = new ConcurrentBag <IHit <T> >();
try
{
var searchResponse = await client.SearchAsync <T>(sd => sd.Source(s => so).Index(_default_index).From(0).Take(scroll_pageize).Query(q => query).Scroll(scroll_timeout));
List <Task> lst_task = new List <Task>();
while (true)
{
if (!searchResponse.IsValid || string.IsNullOrEmpty(searchResponse.ScrollId))
{
break;
}
if (!searchResponse.Documents.Any())
{
break;
}
var tmp = searchResponse.Hits;
foreach (var item in tmp)
{
bag.Add(item);
}
searchResponse = await client.ScrollAsync <T>(scroll_timeout, searchResponse.ScrollId);
}
await client.ClearScrollAsync(new ClearScrollRequest(searchResponse.ScrollId));
await Task.WhenAll(lst_task);
}
catch (Exception)
{
}
finally
{
}
return(bag);
}
public PutWarmerRequestTests()
{
QueryContainer query = new MatchAllQuery();
var request = new PutWarmerRequest("warmer_1")
{
Indices = new IndexNameMarker[] { "my-index" },
SearchDescriptor = new SearchRequest { Query = query }
};
var response = this._client.PutWarmer(request);
this._status = response.ConnectionStatus;
}
public PutWarmerRequestTests()
{
QueryContainer query = new MatchAllQuery();
var request = new PutWarmerRequest("warmer_1")
{
Indices = new IndexNameMarker[] { "my-index" },
SearchDescriptor = new SearchRequest {
Query = query
}
};
var response = this._client.PutWarmer(request);
this._status = response.ConnectionStatus;
}
/// <summary>
/// Scans all documents. Add this if we want field selectors:
/// .Query(q => q.Term(f => f.Property1, "1")
/// && q.Term(k => k.Property2, "0"
///
/// </summary>
/// <param name="esClient">Es client.</param>
/// <param name="index">Index.</param>
/// <param name="modifier">Modifier.</param>
/// <typeparam name="T">The 1st type parameter.</typeparam>
public static void ScanAllDocs <T>(this ElasticClient esClient, string index, string sortField, Action <T, string> modifier, QueryContainer query) where T : class
{
const string scrollTTLMins = "10m";
const int scrollPageSize = 500;
if (query == null)
{
query = new MatchAllQuery();
}
var req = new SearchRequest(index)
{
From = 0,
Size = scrollPageSize,
Query = query,
Sort = new List <ISort> {
new FieldSort {
Field = sortField, Order = SortOrder.Ascending
}
},
Scroll = scrollTTLMins
};
ISearchResponse <T> scanResults = esClient.Search <T>(req);
if (scanResults.Hits.Any())
{
foreach (var hit in scanResults.Hits)
{
modifier(hit.Source, hit.Id);
}
}
var results = esClient.Scroll <T>(scrollTTLMins, scanResults.ScrollId);
while (results.Hits.Any())
{
foreach (var hit in results.Hits)
{
modifier(hit.Source, hit.Id);
}
results = esClient.Scroll <T>(scrollTTLMins, results.ScrollId);
}
}
/// <summary>
/// Find the timestamp of the most recent record for each device.
/// </summary>
/// <returns>The high water mark.</returns>
/// <param name="indexName">IndexName</param>
/// <param name="query">Term Query to select records</param>
public static T getHighWaterMark <T>(this ElasticClient client, string indexName, QueryContainer query) where T : class
{
if (query == null)
{
query = new MatchAllQuery();
}
var req = new SearchRequest(indexName)
{
From = 0,
Size = 1,
Query = query,
Sort = new List <ISort> {
new FieldSort {
Field = "timestamp", Order = SortOrder.Descending
}
}
};
var searchResponse = client.Search <T>(req);
if (searchResponse.IsValid)
{
if (searchResponse.Documents.Count() > 1)
{
throw new ArgumentOutOfRangeException("More than one record returned from high-watermark");
}
var mostRecent = searchResponse.Documents.FirstOrDefault();
return(mostRecent);
}
if (searchResponse.OriginalException != null)
{
Utils.Log("Exception querying ES for watermark: {0}", searchResponse.OriginalException.Message);
}
else
{
Utils.Log("Error querying ES for watermark: {0}", searchResponse);
}
return(null);
}
/// <summary>
/// Find the timestamp of the most recent record for each device.
/// </summary>
/// <returns>The high water mark.</returns>
/// <param name="indexName">IndexName</param>
/// <param name="query">Term Query to select records</param>
public static T getHighWaterMark <T>(ElasticClient client, string indexName, QueryContainer query) where T : class
{
if (query == null)
{
query = new MatchAllQuery();
}
var typeQuery = new TypeQuery
{
Name = "named_query",
Boost = 1.1,
Value = typeof(T).Name
};
var req = new SearchRequest(indexName)
{
From = 0,
Size = 1,
Query = query && typeQuery,
Sort = new List <ISort> {
new SortField {
Field = "timestamp", Order = SortOrder.Descending
}
}
};
var searchResponse = client.Search <T>(req);
if (searchResponse.IsValid)
{
if (searchResponse.Documents.Count() > 1)
{
throw new ArgumentOutOfRangeException("More than one record returned from high-watermark");
}
var mostRecent = searchResponse.Documents.FirstOrDefault();
return(mostRecent);
}
Utils.Log("Exception querying ES: {0}", searchResponse.OriginalException.Message);
throw new ApplicationException("Invalid ES connection.");
}
public static long GetCount <T>(ElasticClient client, String indexName, String mappingType, string key)
where T : class
{
QueryContainer query = new MatchAllQuery() && new TermQuery()
{
Value = key
};
var searchRequest = new SearchRequest
{
Indices = new IndexNameMarker[] { indexName },
Types = new TypeNameMarker[] { mappingType },
Query = query,
SearchType = SearchType.Count
};
return(client.Search <T>(searchRequest).Total);
//return client.Count(s => s
// .Index(indexName)
// .Type(mappingType)
// .Query(q => q.Match(
// m => m.Query(key)))).Count;
}
/// <summary>
/// 构建查询条件
/// </summary>
/// <param name="fields"></param>
/// <returns>{item1:must,item2:should}</returns>
public ESConditionModel BuildQueryContainer(List <ESField> fields, string analyzer)
{
List <QueryContainer> must = new List <QueryContainer>(),
should = new List <QueryContainer>(),
mustNot = new List <QueryContainer>();
if (fields != null && fields.Count > 0)
{
foreach (var item in fields)
{
QueryContainer container = null;
//1.数据类型辨析（暂时不做实现）
if (item.DataType == ESDataType.GeoPoint)
{
//switch (item.QueryType)
//{
// case ESQueryType.Geo_Distance:
// container = new GeoDistanceQuery()
// {
// Location = new GeoLocation(0.1, 0.2),
// Distance =new Distance(),
// };
// break;
// case ESQueryType.Geo_Polygon:
// container = new GeoPolygonQuery(){
// Points=,
// };
// break;
// case ESQueryType.Geo_Bounding_Box:
// container = new GeoBoundingBoxQuery()
// {
// BoundingBox = new BoundingBox()
// {
// BottomRight =,
// TopLeft =,
// }
// };
// break;
//}
}
else
{
switch (item.QueryType)
{
case ESQueryType.Match:
container = new MatchQuery()
{
Analyzer = analyzer,
Field = item.Key,
Query = item.Value.Item1.ToString(),
};
break;
case ESQueryType.All:
container = new MatchAllQuery()
{
};
break;
case ESQueryType.Match_Phrase:
container = new MatchPhraseQuery()
{
Field = item.Key,
Analyzer = analyzer,
Query = item.Value.Item1.ToString()
};
break;
case ESQueryType.Match_Phrase_Prefix:
container = new MatchPhrasePrefixQuery()
{
Field = item.Key,
Analyzer = analyzer,
Query = item.Value.ToString()
};
break;
case ESQueryType.Term:
container = new TermQuery()
{
Field = item.Key,
Value = item.Value.Item1
};
break;
case ESQueryType.Range:
switch (item.DataType)
{
case ESDataType.Text:
break;
case ESDataType.Number:
container = new NumericRangeQuery()
{
Field = item.Key,
GreaterThanOrEqualTo = Convert.ToDouble(item.Value.Item1),
LessThanOrEqualTo = Convert.ToDouble(item.Value.Item2),
};
break;
case ESDataType.Date:
container = new DateRangeQuery()
{
Field = item.Key,
GreaterThanOrEqualTo = DateMath.FromString(item.Value.Item1.ToString()),
LessThanOrEqualTo = DateMath.FromString(item.Value.Item2.ToString())
};
break;
default:
break;
}
break;
}
}
//2.条件类型解析
if (container != null)
{
switch (item.ConditionType)
{
case ESConditionType.Must:
must.Add(container);
break;
case ESConditionType.Should:
should.Add(container);
break;
case ESConditionType.MustNot:
mustNot.Add(container);
break;
default:
break;
}
}
}
}
return(new ESConditionModel()
{
Must = must,
Should = should,
MustNot = mustNot
});
}
//public ConcurrentBag<T> GetObjectScroll<T>(QueryContainer qc, SourceFilter so, List<ISort> lSort = null) where T : class
//{
// return GetObjectScroll<T>(_DefaultIndex, qc, so, lSort);
//}
protected ConcurrentBag <T> GetObjectScroll <T>(string _DefaultIndex, QueryContainer query, SourceFilter so, List <ISort> lSort = null, string routing = "", int scroll_pageize = 2000) where T : class
{
try
{
string scroll_timeout = "5m";
//int scroll_pageize = 2000;
var seenSlices = new ConcurrentBag <int>();
ConcurrentBag <T> bag = new ConcurrentBag <T>();
if (query == null)
{
query = new MatchAllQuery();
}
if (so == null)
{
so = new SourceFilter()
{
}
}
;
try
{
var searchResponse = client.Search <T>(sd => sd.Source(s => so).Index(_DefaultIndex).From(0).Take(scroll_pageize).Query(q => query).Scroll(scroll_timeout));
while (true)
{
if (!searchResponse.IsValid || string.IsNullOrEmpty(searchResponse.ScrollId))
{
break;
}
if (!searchResponse.Documents.Any())
{
break;
}
var tmp = searchResponse.Hits;
Parallel.ForEach(tmp, new ParallelOptions {
MaxDegreeOfParallelism = 4
}, item =>
{
var doc = HitToDocument(item);
bag.Add(doc);
});
/*
* foreach (var item in tmp)
* {
* var doc = HitToDocument(item);
* bag.Add(doc);
* }
*/
searchResponse = client.Scroll <T>(scroll_timeout, searchResponse.ScrollId);
}
client.ClearScroll(new ClearScrollRequest(searchResponse.ScrollId));
}
catch (Exception)
{
}
finally
{
}
return(bag);
}
catch
{
}
return(null);
}
/// <summary>
/// Search on index
/// </summary>
public QueryResult <T> Execute()
{
string indexName = (new T()).IndexAlias;
QueryContainer criteriaQuery = null;
var sortingQuery = new List <ISort>();
if (searchInfo.Fields.Count > 0)
{
criteriaQuery = Helper.BuildQuery(searchInfo);
//create sorting
foreach (var sort in searchInfo.Sort)
{
if (sort is EsGeoSort <T> )
{
var gsort = (EsGeoSort <T>)sort;
sortingQuery.Add(new GeoDistanceSort {
Field = gsort.Field, Unit = DistanceUnit.Meters, DistanceType = GeoDistanceType.Plane, Points = gsort.Points, Order = gsort.Ascending ? SortOrder.Ascending : SortOrder.Descending
});
}
else
{
sortingQuery.Add(new FieldSort {
Field = sort.Field, Order = sort.Ascending ? SortOrder.Ascending : SortOrder.Descending
});
}
}
}
else
{
criteriaQuery = new MatchAllQuery();
}
var searchRequest = new SearchRequest <T>(indexName)
{
Query = criteriaQuery,
From = Start,
Size = MaxResults
};
//Set source columns
if (searchInfo.Source.Count > 0)
{
var sourceFilter = new SourceFilter();
var bfields = searchInfo.Source[0];
Fields fields = bfields;
for (int i = 1; i < searchInfo.Source.Count; i++)
{
fields = fields.And(searchInfo.Source[i]);
}
sourceFilter.Includes = fields;
searchRequest.Source = sourceFilter;
}
//Sorting
if (sortingQuery.Count > 0)
{
searchRequest.Sort = sortingQuery;
}
//Aggregations
if (searchInfo.Aggregations.Count() > 0)
{
searchRequest.Aggregations = searchInfo.Aggregations;
}
if (searchInfo.termDictionary.Count > 0)
{
if (searchRequest.Aggregations == null)
{
searchRequest.Aggregations = new AggregationDictionary();
}
foreach (var term in searchInfo.termDictionary)
{
searchRequest.Aggregations.Add(term.Key, term.Value);
}
}
//Send request
var searchResponse = Manager.EsClient.Search <T>(searchRequest);
if (!searchResponse.IsValid)
{
throw new Exception("Low Level Fail Call.", searchResponse.OriginalException);
}
//handle result
_result = new QueryResult <T>
{
Documents = searchResponse.Documents.AsEnumerable(),
Scores = searchResponse.Hits.Select(s => s.Score),
TotalResuts = searchResponse.Total
};
//get aggregations if any
if (searchInfo.termDictionary != null)
{
foreach (var termInfo in searchInfo.termDictionary)
{
var termKey = termInfo.Key;
var termVal = termInfo.Value;
var terms = searchResponse.Aggregations.Terms(termKey);
if (terms != null)
{
var bucketDictionary = new Dictionary <string, AggregatorMetrics>();
foreach (var bucket in terms.Buckets)
{
var keyBucket = bucket.Key;
var agMetric = new AggregatorMetrics();
foreach (var funcKey in bucket.Keys)
{
if (funcKey == string.Format("{0}_sum", termKey))
{
var sum = bucket.Sum(funcKey);
agMetric.Sum = (sum == null) ? null : sum.Value;
}
if (funcKey == string.Format("{0}_avg", termKey))
{
var val = bucket.Average(funcKey);
agMetric.Average = (val == null) ? null : val.Value;
}
if (funcKey == string.Format("{0}_min", termKey))
{
var val = bucket.Min(funcKey);
agMetric.Min = (val == null) ? null : val.Value;
}
if (funcKey == string.Format("{0}_max", termKey))
{
var val = bucket.Max(funcKey);
agMetric.Max = (val == null) ? null : val.Value;
}
}
var cnt = bucket.DocCount;
agMetric.Count = cnt;
bucketDictionary.Add(bucket.Key, agMetric);
}
_result.Aggregations.Grouped.Add(termKey, bucketDictionary);
}
}
}
return(_result);
}
public static List <Picture> GetPicturesOfComputer(
string computerId, string text, string[] tags, DateTime startDate, DateTime endDate, int start = 0, int rows = 10)
{
QueryContainer textContainer;
QueryContainer tagsContainer;
if (string.IsNullOrEmpty(text))
{
textContainer = new MatchAllQuery();
}
else
{
textContainer = new MatchQuery()
{
Field = "Text", Query = text
}
};
if ((tags == null) || (tags.Length > 0))
{
tagsContainer = new MatchAllQuery();
}
else
{
tagsContainer = new QueryContainer(new MatchQuery()
{
Field = "Tags", Query = tags[0]
});
for (int i = 1; i < tags.Length; i++)
{
tagsContainer = tagsContainer && new MatchQuery()
{
Field = "Tags", Query = tags[i]
};
}
}
QueryContainer query = new FilteredQuery()
{
Filter = new FilterContainer(
new RangeFilter()
{
Field = "Date",
GreaterThanOrEqualTo = startDate.ToString("yyyy-MM-ddTHH:mm:ss"),
LowerThanOrEqualTo = endDate.ToString("yyyy-MM-ddTHH:mm:ss")
}),
Query = new QueryContainer(
new MatchQuery()
{
Field = "ComputerId",
Query = computerId
})
};
QueryContainer finalQuery = query && textContainer && tagsContainer;
ISearchResponse <Picture> response =
_elasticClient.Search <Picture>(s => s
.Type(PICTURE_OBJECT_TYPE)
.From(start)
.Size(rows)
.SortAscending("Date")
.Query(finalQuery));
return(response.Documents.ToList());
}
public virtual QueryContainer GetDefaultQuery()
{
QueryContainer queryContainer = new MatchAllQuery();
return(queryContainer);
}
public static MatchAllQuery BuildMatchAllQuery()
{
MatchAllQuery.Builder builder = MatchAllQuery.CreateBuilder();
return(builder.Build());
}
