private static AggregationDescriptor <FlowLog> GetAggregationDescriptor(AggregationDescriptor <FlowLog> descriptor, List <AggregationParameter> parameters)
{
if (parameters != null && parameters.Any())
{
foreach (var parameter in parameters)
{
if (parameter.SourceAggExpression != null)
{
descriptor.Filters(parameter.AggregationIdentifier, f => f.Filters(filter =>
{
FilterContainer container = new FilterContainer();
container &= filter.Query(t => t.Terms(parameter.SourceAggExpression.Field, parameter.SourceAggExpression.Values));
if (parameter.DestAggExpression != null)
{
container &= filter.Query(t => t.Terms(parameter.DestAggExpression.Field, parameter.DestAggExpression.Values));
}
return(container);
}
).Aggregations(ag1 => ag1.Terms(actionAggKey, t1 => t1.Field(f1 => f1.action).
Aggregations(ag2 => ag2.Terms(protocolAggKey, t2 => t2.Field(f2 => f2.protocol).
Aggregations(ag3 => ag3.Terms(dstPortAggKey, t3 => t3.Field(f3 => f3.dstport).
Aggregations(a => a.Sum(sumOfBytesAggKey, byField => byField.Field(fieldName => fieldName.bytes))
.Sum(sumOfPacketsAggKey, byField => byField.Field(fieldName => fieldName.packets)))))))))
);
}
}
}
return(descriptor);
}
private AggregationDescriptor <PersistentEvent> BuildAggregations(AggregationDescriptor <PersistentEvent> aggregation, IEnumerable <FieldAggregation> fields)
{
foreach (var field in fields)
{
switch (field.Type)
{
case FieldAggregationType.Average:
aggregation.Average(field.Key, a => field.DefaultValueScript == null ? a.Field(field.Field) : a.Script(field.DefaultValueScript));
break;
case FieldAggregationType.Distinct:
aggregation.Cardinality(field.Key, a => (field.DefaultValueScript == null ? a.Field(field.Field) : a.Script(field.DefaultValueScript)).PrecisionThreshold(100));
break;
case FieldAggregationType.Sum:
aggregation.Sum(field.Key, a => field.DefaultValueScript == null ? a.Field(field.Field) : a.Script(field.DefaultValueScript));
break;
case FieldAggregationType.Min:
aggregation.Min(field.Key, a => field.DefaultValueScript == null ? a.Field(field.Field) : a.Script(field.DefaultValueScript));
break;
case FieldAggregationType.Max:
aggregation.Max(field.Key, a => field.DefaultValueScript == null ? a.Field(field.Field) : a.Script(field.DefaultValueScript));
break;
case FieldAggregationType.Last:
// TODO: Populate with the last value.
break;
case FieldAggregationType.Term:
var termField = field as TermFieldAggregation;
if (termField == null)
{
throw new InvalidOperationException("term aggregation must be of type TermFieldAggregation");
}
aggregation.Terms(field.Key, t => {
var tad = t.Field(field.Field);
if (!String.IsNullOrEmpty(termField.ExcludePattern))
{
tad.Exclude(termField.ExcludePattern);
}
if (!String.IsNullOrEmpty(termField.IncludePattern))
{
tad.Include(termField.IncludePattern);
}
return(tad);
});
break;
default:
throw new InvalidOperationException($"Unknown FieldAggregation type: {field.Type}");
}
}
return(aggregation);
}
public static AggregationDescriptor <T> GroupBy <T>(this AggregationDescriptor <T> innerAggregation, IEnumerable <String> keys) where T : class
{
var reversedAndLowered = keys.Select(x => x.FirstCharacterToLower()).Reverse().ToList();
var aggregations = reversedAndLowered.Aggregate(innerAggregation, (s, i) => s.GroupBy(i));
return(aggregations);
}
public static AggregationDescriptor <T> GroupBy <T>(this AggregationDescriptor <T> innerAggregation, String key) where T : class
{
AggregationDescriptor <T> v = new AggregationDescriptor <T>();
v.Terms(key, tr =>
{
TermsAggregationDescriptor <T> trmAggDescriptor = new TermsAggregationDescriptor <T>();
trmAggDescriptor.Field(key);
trmAggDescriptor.Size(int.MaxValue);
return(trmAggDescriptor.Aggregations(x => innerAggregation));
});
return(v);
}
public static AggregationDescriptor <T> GroupBy <T>(this AggregationDescriptor <T> innerAggregation, Expression <Func <T, Object> > fieldGetter) where T : class
{
AggregationDescriptor <T> v = new AggregationDescriptor <T>();
var fieldName = fieldGetter.GetAggName(AggType.GroupBy);
v.Terms(fieldName, tr =>
{
TermsAggregationDescriptor <T> trmAggDescriptor = new TermsAggregationDescriptor <T>();
trmAggDescriptor.Field(fieldGetter);
trmAggDescriptor.Size(int.MaxValue);
return(trmAggDescriptor.Aggregations(x => innerAggregation));
});
return(v);
}
public static AggregationDescriptor <T> IntoHistogram <T>(this AggregationDescriptor <T> innerAggregation,
Expression <Func <T, Object> > fieldGetter, int interval) where T : class
{
AggregationDescriptor <T> v = new AggregationDescriptor <T>();
var fieldName = GetName(fieldGetter);
v.Histogram(fieldName, dr =>
{
HistogramAggregationDescriptor <T> dateAggDesc = new HistogramAggregationDescriptor <T>();
dateAggDesc.Field(fieldGetter).Interval(interval);
return(dateAggDesc.Aggregations(x => innerAggregation));
});
return(v);
}
public static AggregationDescriptor <T> SortedTopHits <T>(this AggregationDescriptor <T> agg, int size, Expression <Func <T, object> > fieldSort, SortType sorttype, params Expression <Func <T, object> >[] fieldGetter) where T : class
{
var aggName = sorttype + fieldSort.GetAggName(AggType.TopHits);
var sortFieldDescriptor = new SortFieldDescriptor <T>();
sortFieldDescriptor = sortFieldDescriptor.OnField(fieldSort);
if (sorttype == SortType.Ascending)
{
sortFieldDescriptor = sortFieldDescriptor.Ascending();
}
else
{
sortFieldDescriptor = sortFieldDescriptor.Descending();
}
return(agg.TopHits(aggName, x => x.Size(size).Source(i => i.Include(fieldGetter)).Sort(s => sortFieldDescriptor)));
}
public static AggregationDescriptor <T> SumBy <T>(this AggregationDescriptor <T> agg, Expression <Func <T, object> > fieldGetter, Expression <Func <T, bool> > filterRule = null) where T : class
{
var aggName = fieldGetter.GetAggName(AggType.Sum);
if (filterRule == null)
{
return(agg.Sum(aggName, x => x.Field(fieldGetter)));
}
var filterName = filterRule.GenerateFilterName();
agg.Filter(filterName,
f =>
f.Filter(fd => filterRule.Body.GenerateFilterDescription <T>())
.Aggregations(innerAgg => innerAgg.Sum(aggName, field => field.Field(fieldGetter))));
return(agg);
}
private AggregationDescriptor <T> GetAggregationDescriptor <T>(object query) where T : class
{
var facetQuery = query as IFacetQuery;
if (facetQuery == null || facetQuery.FacetFields.Count == 0)
{
return(null);
}
var descriptor = new AggregationDescriptor <T>();
foreach (var t in facetQuery.FacetFields)
{
descriptor = descriptor.Terms(t.Field, s => s.Field(t.Field).Size(t.Size ?? 100));
}
return(descriptor);
}
//private static Dictionary<string, string> GetSubnets()
//{
// subnets = new Dictionary<string, string>();
// subnets.Add("subnet1", "172.31.10.115,172.31.24.161,66.155.40.249");
// subnets.Add("subnet2", "203.178.148.19,114.43.14.109,178.71.248.224");
// subnets.Add("subnet3", "198.55.111.50,52.25.149.105,58.218.213.208");
// subnets.Add("subnet4", "199.203.59.121,111.123.180.44,172.31.18.125");
// return subnets;
//}
#endregion
#region GetAggregationDescriptor
private static AggregationDescriptor <FlowLog> GetAggregationDescriptor(AggregationDescriptor <FlowLog> descriptor, Dictionary <string, string> subnets)
{
if (subnets.Any())
{
var ipList = new List <string>();
foreach (var subnet in subnets)
{
var subnetIPs = subnet.Value.Split(ipSplitChar);
ipList.AddRange(subnetIPs);
descriptor.Filters(subnet.Key, f => f.Filters(t => t.Terms(field => field.srcaddr, subnetIPs))
.Aggregations(a => GetNestedAggregationDescriptor(a, subnets, subnet.Key)));
}
if (ipList.Any())
{
descriptor.Filters(anonymousSubnet, f => f.Filters(t => t.Not(n => n.Terms(field => field.srcaddr, ipList)))
.Aggregations(a => GetNestedAggregationDescriptor(a, subnets, anonymousSubnet)));
}
}
return(descriptor);
}
private static AggregationDescriptor <FlowLog> GetNestedAggregationDescriptor(AggregationDescriptor <FlowLog> descriptor, Dictionary <string, string> subnets, string sourceSubnetKey)
{
if (subnets.Any())
{
var ipList = new List <string>();
foreach (var subnet in subnets)
{
if (subnet.Key != sourceSubnetKey) //Not sure if we need to filter this
{
var subnetIPs = subnet.Value.Split(ipSplitChar);
ipList.AddRange(subnetIPs);
descriptor.Filters(subnet.Key, f => f.Filters(t => t.Terms(field => field.dstaddr, subnetIPs))
.Aggregations(a => a.Sum(sumOfBytesAggKey, byField => byField.Field(fieldName => fieldName.bytes))
.Sum(sumOfPacketsAggKey, byField => byField.Field(fieldName => fieldName.packets))));
}
}
if (ipList.Any())
{
descriptor.Filters(anonymousSubnet, f => f.Filters(t => t.Not(n => n.Terms(field => field.dstaddr, ipList)))
.Aggregations(a => a.Sum(sumOfBytesAggKey, byField => byField.Field(fieldName => fieldName.bytes))
.Sum(sumOfPacketsAggKey, byField => byField.Field(fieldName => fieldName.packets))));
}
}
return(descriptor);
}
public static AggregationDescriptor <T> DateHistogram <T>(this AggregationDescriptor <T> agg,
Expression <Func <T, Object> > fieldGetter, DateInterval dateInterval) where T : class
{
return(agg.DateHistogram(GetName(fieldGetter), x => x.Field(fieldGetter).Interval(dateInterval)));
}
public static AggregationDescriptor <T> PercentilesBy <T>(this AggregationDescriptor <T> agg, Expression <Func <T, object> > fieldGetter) where T : class
{
var aggName = fieldGetter.GetAggName(AggType.Percentile);
return(agg.Percentiles(aggName, x => x.Field(fieldGetter)));
}
public static AggregationDescriptor <T> DistinctBy <T>(this AggregationDescriptor <T> agg, Expression <Func <T, object> > fieldGetter) where T : class
{
var aggName = fieldGetter.GetAggName(AggType.Distinct);
return(agg.Terms(aggName, x => x.Field(fieldGetter).Size(int.MaxValue)));
}
public static AggregationDescriptor <T> TopHits <T>(this AggregationDescriptor <T> agg, int size, params Expression <Func <T, object> >[] fieldGetter) where T : class
{
var aggName = AggType.TopHits.ToString();
return(agg.TopHits(aggName, x => x.Size(size).Source(i => i.Include(fieldGetter))));
}
private AggregationDescriptor<PersistentEvent> BuildAggregations(AggregationDescriptor<PersistentEvent> aggregation, IEnumerable<FieldAggregation> fields) {
foreach (var field in fields) {
switch (field.Type) {
case FieldAggregationType.Average:
aggregation.Average(field.Key, a => field.DefaultValueScript == null ? a.Field(field.Field) : a.Script(field.DefaultValueScript));
break;
case FieldAggregationType.Distinct:
aggregation.Cardinality(field.Key, a => (field.DefaultValueScript == null ? a.Field(field.Field) : a.Script(field.DefaultValueScript)).PrecisionThreshold(100));
break;
case FieldAggregationType.Sum:
aggregation.Sum(field.Key, a => field.DefaultValueScript == null ? a.Field(field.Field) : a.Script(field.DefaultValueScript));
break;
case FieldAggregationType.Min:
aggregation.Min(field.Key, a => field.DefaultValueScript == null ? a.Field(field.Field) : a.Script(field.DefaultValueScript));
break;
case FieldAggregationType.Max:
aggregation.Max(field.Key, a => field.DefaultValueScript == null ? a.Field(field.Field) : a.Script(field.DefaultValueScript));
break;
case FieldAggregationType.Last:
// TODO: Populate with the last value.
break;
case FieldAggregationType.Term:
var termField = field as TermFieldAggregation;
if (termField == null)
throw new InvalidOperationException("term aggregation must be of type TermFieldAggregation");
aggregation.Terms(field.Key, t => {
var tad = t.Field(field.Field);
if (!String.IsNullOrEmpty(termField.ExcludePattern))
tad.Exclude(termField.ExcludePattern);
if (!String.IsNullOrEmpty(termField.IncludePattern))
tad.Include(termField.IncludePattern);
return tad;
});
break;
default:
throw new InvalidOperationException($"Unknown FieldAggregation type: {field.Type}");
}
}
return aggregation;
}