internal float Accrete(HistogramHeatPoint point, AggregationMethod aggregatioMethod, float percentile = 0f)
{
switch (aggregatioMethod)
{
case AggregationMethod.Increment:
return(point.histogram.Count);
case AggregationMethod.Cumulative:
return(point.histogram.Sum(x => (float)Convert.ToDecimal(x)));
case AggregationMethod.Min:
point.histogram.Sort();
return(point.histogram[0]);
case AggregationMethod.Max:
point.histogram.Sort();
return(point.histogram[point.histogram.Count - 1]);
case AggregationMethod.First:
return(point.first);
case AggregationMethod.Last:
return(point.last);
case AggregationMethod.Average:
return(point.histogram.Sum(x => (float)Convert.ToDecimal(x)) / point.histogram.Count);
case AggregationMethod.Percentile:
point.histogram.Sort();
int percentileIdx = (int)((percentile / 100f) * point.histogram.Count);
int finalIdx = (percentileIdx >= point.histogram.Count - 1) ? point.histogram.Count - 1 : percentileIdx;
return(point.histogram[finalIdx]);
}
return(0);
}
public void TestAggregateFunction()
{
AggregationMethodFactory aggFactory = ValidatedNodeToTest.Factory;
AggregationMethod agg = aggFactory.Make(SupportExprNodeFactory.MethodResService, -1, 1, 1);
Assert.AreEqual(typeof(double?), agg.ValueType);
Assert.IsNull(agg.Value);
agg.Enter(82);
Assert.AreEqual(0D, agg.Value);
agg.Enter(78);
Assert.AreEqual(2D, agg.Value);
agg.Enter(70);
double result = agg.Value.AsDouble();
Assert.AreEqual("4.4444", result.ToString().Substring(0, 6));
agg.Enter(58);
Assert.AreEqual(8D, agg.Value);
agg.Enter(42);
Assert.AreEqual(12.8D, agg.Value);
agg.Leave(82);
Assert.AreEqual(12D, agg.Value);
agg.Leave(58);
result = agg.Value.AsDouble();
Assert.AreEqual("14.2222", result.ToString().Substring(0, 7));
}
public static AggregationMethod MakeDistinctAggregator(AggregationMethod aggregationMethod, bool hasFilter)
{
if (hasFilter)
{
return(new AggregatorDistinctValueFilter(aggregationMethod));
}
return(new AggregatorDistinctValue(aggregationMethod));
}
private static MethodCallExpression AggCallExpression(AggregationMethod aggMethod, ParameterExpression sourceParameter, LambdaExpression lambda)
{
String methodName;
switch (aggMethod)
{
case AggregationMethod.Average:
methodName = nameof(Enumerable.Average);
break;
case AggregationMethod.CountDistinct:
return(CountDistinctExpression(sourceParameter, lambda));
case AggregationMethod.Max:
methodName = nameof(Enumerable.Max);
break;
case AggregationMethod.Min:
methodName = nameof(Enumerable.Min);
break;
case AggregationMethod.Sum:
methodName = nameof(Enumerable.Sum);
break;
default:
throw new NotSupportedException();
}
MethodInfo closeMethod;
MethodInfo openMethod = OeMethodInfoHelper.GetAggMethodInfo(methodName, lambda.ReturnType);
if (openMethod == null)
{
Func <IEnumerable <Object>, Func <Object, Object>, Object> aggFunc;
switch (aggMethod)
{
case AggregationMethod.Max:
aggFunc = Enumerable.Max <Object, Object>;
break;
case AggregationMethod.Min:
aggFunc = Enumerable.Min <Object, Object>;
break;
default:
throw new InvalidOperationException($"Enumerable.{methodName} not found");;
}
openMethod = aggFunc.GetMethodInfo().GetGenericMethodDefinition();
closeMethod = openMethod.MakeGenericMethod(lambda.Parameters[0].Type, lambda.ReturnType);
}
else
{
closeMethod = openMethod.MakeGenericMethod(lambda.Parameters[0].Type);
}
return(Expression.Call(closeMethod, sourceParameter, lambda));
}
/// <summary>
/// Create an AggregateExpressionToken.
/// </summary>
/// <param name="expression">The aggregate expression.</param>
/// <param name="withVerb">The aggregation method.</param>
/// <param name="alias">The alias for this query token.</param>
public AggregateExpressionToken(QueryToken expression, AggregationMethod withVerb, string alias)
{
ExceptionUtils.CheckArgumentNotNull(expression, "expression");
ExceptionUtils.CheckArgumentNotNull(alias, "alias");
this.expression = expression;
this.method = withVerb;
this.alias = alias;
}
public AggregationMethod[] NewAggregatorsInternal(AggregationMethodFactory[] prototypes, int agentInstanceId)
{
AggregationMethod[] row = new AggregationMethod[prototypes.Length];
for (int i = 0; i < prototypes.Length; i++)
{
row[i] = prototypes[i].Make(this, agentInstanceId, -1, i);
}
return(row);
}
public AggregationMethod Make()
{
AggregationMethod method = AggregationFunctionFactory.NewAggregator();
if (!Parent.IsDistinct)
{
return(method);
}
return(AggregationMethodFactoryUtil.MakeDistinctAggregator(method, false));
}
public AggregationMethod Make(MethodResolutionService methodResolutionService, int agentInstanceId, int groupId, int aggregationId)
{
AggregationMethod method = _aggregationFunctionFactory.NewAggregator();
if (!_parent.IsDistinct)
{
return(method);
}
return(methodResolutionService.MakeDistinctAggregator(agentInstanceId, groupId, aggregationId, method, _aggregatedValueType, false));
}
public static AggregationMethod[] NewAggregators(AggregationMethodFactory[] prototypes)
{
var row = new AggregationMethod[prototypes.Length];
for (int i = 0; i < prototypes.Length; i++)
{
row[i] = prototypes[i].Make();
}
return(row);
}
public AggregationMethod Make()
{
AggregationMethod method = MakeMinMaxAggregator(Parent.MinMaxTypeEnum, Type, HasDataWindows, Parent.HasFilter);
if (!Parent.IsDistinct)
{
return(method);
}
return(AggregationMethodFactoryUtil.MakeDistinctAggregator(method, Parent.HasFilter));
}
public AggregationMethod Make()
{
AggregationMethod method = MakeSumAggregator(InputValueType, Parent.HasFilter);
if (!Parent.IsDistinct)
{
return(method);
}
return(AggregationMethodFactoryUtil.MakeDistinctAggregator(method, Parent.HasFilter));
}
public AggregationMethod Make()
{
AggregationMethod method = MakeCountAggregator(IgnoreNulls, Parent.HasFilter);
if (!Parent.IsDistinct)
{
return(method);
}
return(AggregationMethodFactoryUtil.MakeDistinctAggregator(method, Parent.HasFilter));
}
public AggregationMethod Make(MethodResolutionService methodResolutionService, int agentInstanceId, int groupId, int aggregationId)
{
AggregationMethod method = methodResolutionService.MakeAvedevAggregator(agentInstanceId, groupId, aggregationId, _parent.HasFilter);
if (!_parent.IsDistinct)
{
return(method);
}
return(methodResolutionService.MakeDistinctAggregator(agentInstanceId, groupId, aggregationId, method, _aggregatedValueType, _parent.HasFilter));
}
public AggregationMethod Make(MethodResolutionService methodResolutionService, int agentInstanceId, int groupId, int aggregationId)
{
AggregationMethod method = methodResolutionService.MakeNthAggregator(agentInstanceId, groupId, aggregationId, _childType, _size + 1);
if (!_parent.IsDistinct)
{
return(method);
}
return(methodResolutionService.MakeDistinctAggregator(agentInstanceId, groupId, aggregationId, method, _childType, false));
}
public AggregationMethod Make()
{
AggregationMethod method = MakeStddevAggregator(Parent.HasFilter);
if (!Parent.IsDistinct)
{
return(method);
}
return(AggregationMethodFactoryUtil.MakeDistinctAggregator(method, Parent.HasFilter));
}
public AggregationMethod Make()
{
AggregationMethod method = MakeAvgAggregator(ChildType, Parent.HasFilter, OptionalMathContext);
if (!Parent.IsDistinct)
{
return(method);
}
return(AggregationMethodFactoryUtil.MakeDistinctAggregator(method, Parent.HasFilter));
}
/// <summary>
/// Create a AggregateExpression.
/// </summary>
/// <param name="expression">The aggregation expression.</param>
/// <param name="method">The <see cref="AggregationMethod"/>.</param>
/// <param name="alias">The aggregation alias.</param>
/// <param name="typeReference">The <see cref="IEdmTypeReference"/> of this aggregate expression.</param>
public AggregateExpression(SingleValueNode expression, AggregationMethod method, string alias, IEdmTypeReference typeReference)
{
ExceptionUtils.CheckArgumentNotNull(expression, "expression");
ExceptionUtils.CheckArgumentNotNull(alias, "alias");
ExceptionUtils.CheckArgumentNotNull(typeReference, "typeReference");
this.expression = expression;
this.method = method;
this.alias = alias;
this.typeReference = typeReference;
}
/// <summary>
/// Create a AggregateExpression.
/// </summary>
/// <param name="expression">The aggregation expression.</param>
/// <param name="method">The <see cref="AggregationMethod"/>.</param>
/// <param name="alias">The aggregation alias.</param>
/// <param name="typeReference">The <see cref="IEdmTypeReference"/> of this aggregate expression.</param>
public AggregateExpression(SingleValueNode expression, AggregationMethod method, string alias, IEdmTypeReference typeReference)
{
ExceptionUtils.CheckArgumentNotNull(expression, "expression");
ExceptionUtils.CheckArgumentNotNull(alias, "alias");
ExceptionUtils.CheckArgumentNotNull(typeReference, "typeReference");
this.expression = expression;
this.method = method;
this.alias = alias;
this.typeReference = typeReference;
}
public Expression CallEnumerableFirstOrLast(Expression allProjectedRows, AggregationMethod aggregationMethod, Type elementType)
{
// Enumerable.FirstOrDefault<TRowDataType>(allProjectedRows)
var methodName = aggregationMethod == AggregationMethod.first ? "FirstOrDefault" : "LastOrDefault";
var firstOrLastMethodInfo = typeof(Enumerable)
.GetMethods(BindingFlags.Public | BindingFlags.Static)
.Single(mi => mi.Name == methodName && mi.GetParameters().Length == 1)
.MakeGenericMethod(elementType);
return(Expression.Call(firstOrLastMethodInfo, allProjectedRows));
}
/// <summary>
/// Create a PropertyAggregateExpression.
/// </summary>
/// <param name="expression">The aggregation expression.</param>
/// <param name="method">The <see cref="AggregationMethod"/>.</param>
/// <param name="alias">The aggregation alias.</param>
/// <param name="typeReference">The <see cref="IEdmTypeReference"/> of this aggregate expression.</param>
public AggregateExpression(SingleValueNode expression, AggregationMethod method, string alias, IEdmTypeReference typeReference)
: base(AggregateExpressionKind.PropertyAggregate, alias)
{
ExceptionUtils.CheckArgumentNotNull(expression, "expression");
ExceptionUtils.CheckArgumentNotNull(alias, "alias");
this.expression = expression;
this.method = method;
//// TypeReference is null for dynamic properties
this.typeReference = typeReference;
}
public void CanAggregate2(AggregationMethod method, double nanLimit, int[] data, byte[] status, double expected)
{
//// Arrange
//var command = new AggregationService(1, NullLogger.Instance, LoggerFactory.Create(_ => { }));
//var kernelSize = data.Length;
//var config = new AggregationUnit() { Method = method, Argument = "360" };
//// Act
//var actual = command.ApplyAggregationFunction(config, kernelSize, data, status, nanLimit, NullLogger.Instance);
//// Assert
//Assert.Equal(expected, actual[0]);
}
private string getTransformation(AggregationMethod m)
{
switch (m)
{
case AggregationMethod.Average: return("average");
case AggregationMethod.CountDistinct: return("countdistinct");
case AggregationMethod.Sum: return("sum");
case AggregationMethod.Min: return("min");
default: return("max");
}
}
private static MethodCallExpression AggCallExpression(AggregationMethod aggMethod, ParameterExpression sourceParameter, LambdaExpression lambda)
{
String methodName;
switch (aggMethod)
{
case AggregationMethod.Average:
methodName = nameof(Enumerable.Average);
break;
case AggregationMethod.CountDistinct:
return(CountDistinctExpression(sourceParameter, lambda));
case AggregationMethod.Max:
methodName = nameof(Enumerable.Max);
break;
case AggregationMethod.Min:
methodName = nameof(Enumerable.Min);
break;
case AggregationMethod.Sum:
methodName = nameof(Enumerable.Sum);
break;
case AggregationMethod.VirtualPropertyCount:
return(CountExpression(sourceParameter));
default:
throw new NotSupportedException();
}
MethodInfo closeMethod;
MethodInfo openMethod = OeMethodInfoHelper.GetAggMethodInfo(methodName, lambda.ReturnType);
if (openMethod.GetGenericArguments().Length == 1)
{
closeMethod = openMethod.MakeGenericMethod(lambda.Parameters[0].Type);
}
else
{
closeMethod = openMethod.MakeGenericMethod(lambda.Parameters[0].Type, lambda.ReturnType);
}
return(Expression.Call(closeMethod, sourceParameter, lambda));
}
public CellValueCalculator FirstOrLast(AggregationMethod aggregationMethod, FormulaExpressionsCompiler compiler)
{
var allRowsCalculator = All(compiler);
var key = $"{aggregationMethod}:{CalculateExpression}";
return(new CellValueCalculator(
ResultType,
GetPrecalculatedAggregatedValue(ResultType, compiler.Parameters, key),
allRowsCalculator.AggregatedValuesCalculators.Insert(
0,
KeyValuePair.Create(
key,
compiler.CallEnumerableFirstOrLast(
allRowsCalculator.CalculateExpression,
aggregationMethod,
ResultType))),
canBeAggregated: true));
}
/// <summary>
/// Converts a <see cref="AggregationMethod"/> to a string.
/// </summary>
/// <param name="method">The aggregation method to convert.</param>
/// <returns>The converted string.</returns>
public static string ToString(AggregationMethod method)
{
var ret = "";
switch (method)
{
case AggregationMethod.Average:
ret = "avg";
break;
case AggregationMethod.Sum:
ret = "sum";
break;
case AggregationMethod.EndOfPeriod:
ret = "eop";
break;
}
return(ret);
}
public AggregationServiceMatchRecognize MakeService(AgentInstanceContext agentInstanceContext)
{
var aggregatorsEachStream = new AggregationMethod[_factoryEachStream.Length][];
int count = 0;
for (int stream = 0; stream < _factoryEachStream.Length; stream++)
{
AggregationMethodFactory[] thatStream = _factoryEachStream[stream];
if (thatStream != null)
{
aggregatorsEachStream[stream] = new AggregationMethod[thatStream.Length];
for (int aggId = 0; aggId < thatStream.Length; aggId++)
{
aggregatorsEachStream[stream][aggId] =
_factoryEachStream[stream][aggId].Make(
agentInstanceContext.StatementContext.MethodResolutionService,
agentInstanceContext.AgentInstanceId, 0, aggId);
count++;
}
}
}
var aggregatorsAll = new AggregationMethod[count];
count = 0;
for (int stream = 0; stream < _factoryEachStream.Length; stream++)
{
if (_factoryEachStream[stream] != null)
{
for (int aggId = 0; aggId < _factoryEachStream[stream].Length; aggId++)
{
aggregatorsAll[count] = aggregatorsEachStream[stream][aggId];
count++;
}
}
}
return(new AggregationServiceMatchRecognizeImpl(_evaluatorsEachStream, aggregatorsEachStream, aggregatorsAll));
}
/// <summary>
/// Get the observations or data values for an economic data series. Corresponds to http://api.stlouisfed.org/fred/series/observations
/// </summary>
/// <param name="seriesId">The id for a series.</param>
/// <param name="fileType">The type of file to send.</param>
/// <param name="filename">The where to save the file.</param>
/// <param name="observationStart">The start of the observation period.</param>
/// <param name="observationEnd">The end of the observation period.</param>
/// <param name="realtimeStart">The start of the real-time period.</param>
/// <param name="realtimeEnd">The end of the real-time period.</param>
/// <param name="vintageDates">A comma separated string of YYYY-MM-DD formatted dates in history (e.g. 2000-01-01,2005-02-24).
/// Vintage dates are used to download data as it existed on these specified dates in history. Vintage dates can be
/// specified instead of a real-time period using realtime_start and realtime_end.
///
/// Sometimes it may be useful to enter a vintage date that is not a date when the data values were revised.
/// For instance you may want to know the latest available revisions on 2001-09-11 (World Trade Center and
/// Pentagon attacks) or as of a Federal Open Market Committee (FOMC) meeting date. Entering a vintage date is
/// also useful to compare series on different releases with different release dates.</param>
/// <param name="limit">The maximum number of results to return. An integer between 1 and 100000, optional, default: 100000</param>
/// <param name="offset">non-negative integer, optional, default: 0</param>
/// <param name="order">Sort results is ascending or descending observation_date order.</param>
/// <param name="transformation">Type of data value transformation.</param>
/// <param name="frequency">An optional parameter that indicates a lower frequency to aggregate values to.
/// The FRED frequency aggregation feature converts higher frequency data series into lower frequency data series
/// (e.g. converts a monthly data series into an annual data series). In FRED, the highest frequency data is daily,
/// and the lowest frequency data is annual. There are 3 aggregation methods available- average, sum, and end of period.</param>
/// <param name="method">A key that indicates the aggregation method used for frequency aggregation.</param>
/// <param name="outputType">Output type:
/// 1. Observations by Real-Time Period
/// 2. Observations by Vintage Date, All Observations
/// 3. Observations by Vintage Date, New and Revised Observations Only
/// 4. Observations, Initial Release Only
/// </param>
/// <returns>Observations or data values for an economic data series.</returns>
public void GetSeriesObservationsFile(string seriesId, FileType fileType, string filename,
DateTime observationStart,
DateTime observationEnd,
DateTime realtimeStart,
DateTime realtimeEnd,
IEnumerable<DateTime> vintageDates, int limit = 100000,
int offset = 0, SortOrder order = SortOrder.Ascending,
Transformation transformation = Transformation.None,
Frequency frequency = Frequency.None,
AggregationMethod method = AggregationMethod.Average,
OutputType outputType = OutputType.RealTime)
{
var extension = GetExtension(filename);
if (fileType != FileType.Xml && fileType != FileType.Json && !extension.Equals(".zip", StringComparison.OrdinalIgnoreCase))
{
filename += ".zip";
}
GetSeriesObservationsStreamAsync(seriesId, fileType, new FileStream(filename, FileMode.Create), observationStart, observationEnd,
realtimeStart, realtimeEnd, vintageDates, limit, offset, order, transformation, frequency, method, outputType).RunSynchronously();
}
/// <summary>
/// Get the observations or data values for an economic data series. Corresponds to http://api.stlouisfed.org/fred/series/observations
/// </summary>
/// <param name="seriesId">The id for a series.</param>
/// <param name="fileType">The type of file to send.</param>
/// <param name="filename">The where to save the file.</param>
/// <param name="observationStart">The start of the observation period.</param>
/// <param name="observationEnd">The end of the observation period.</param>
/// <param name="realtimeStart">The start of the real-time period.</param>
/// <param name="realtimeEnd">The end of the real-time period.</param>
/// <param name="vintageDates">A comma separated string of YYYY-MM-DD formatted dates in history (e.g. 2000-01-01,2005-02-24).
/// Vintage dates are used to download data as it existed on these specified dates in history. Vintage dates can be
/// specified instead of a real-time period using realtime_start and realtime_end.
///
/// Sometimes it may be useful to enter a vintage date that is not a date when the data values were revised.
/// For instance you may want to know the latest available revisions on 2001-09-11 (World Trade Center and
/// Pentagon attacks) or as of a Federal Open Market Committee (FOMC) meeting date. Entering a vintage date is
/// also useful to compare series on different releases with different release dates.</param>
/// <param name="limit">The maximum number of results to return. An integer between 1 and 100000, optional, default: 100000</param>
/// <param name="offset">non-negative integer, optional, default: 0</param>
/// <param name="order">Sort results is ascending or descending observation_date order.</param>
/// <param name="transformation">Type of data value transformation.</param>
/// <param name="frequency">An optional parameter that indicates a lower frequency to aggregate values to.
/// The FRED frequency aggregation feature converts higher frequency data series into lower frequency data series
/// (e.g. converts a monthly data series into an annual data series). In FRED, the highest frequency data is daily,
/// and the lowest frequency data is annual. There are 3 aggregation methods available- average, sum, and end of period.</param>
/// <param name="method">A key that indicates the aggregation method used for frequency aggregation.</param>
/// <param name="outputType">Output type:
/// 1. Observations by Real-Time Period
/// 2. Observations by Vintage Date, All Observations
/// 3. Observations by Vintage Date, New and Revised Observations Only
/// 4. Observations, Initial Release Only
/// </param>
/// <returns>Observations or data values for an economic data series.</returns>
public void GetSeriesObservationsFile(string seriesId, FileType fileType, string filename,
DateTime observationStart,
DateTime observationEnd,
DateTime realtimeStart,
DateTime realtimeEnd,
IEnumerable <DateTime> vintageDates, int limit = 100000,
int offset = 0, SortOrder order = SortOrder.Ascending,
Transformation transformation = Transformation.None,
Frequency frequency = Frequency.None,
AggregationMethod method = AggregationMethod.Average,
OutputType outputType = OutputType.RealTime)
{
var extension = GetExtension(filename);
if (fileType != FileType.Xml && fileType != FileType.Json && !extension.Equals(".zip", StringComparison.OrdinalIgnoreCase))
{
filename += ".zip";
}
GetSeriesObservationsStreamAsync(seriesId, fileType, new FileStream(filename, FileMode.Create), observationStart, observationEnd,
realtimeStart, realtimeEnd, vintageDates, limit, offset, order, transformation, frequency, method, outputType).RunSynchronously();
}
private double[] ApplyAggregationFunction(AggregationMethod method,
string argument,
int kernelSize,
double[] data,
double nanLimit,
ILogger logger)
{
var targetDatasetLength = data.Length / kernelSize;
var result = new double[targetDatasetLength];
switch (method)
{
case AggregationMethod.Mean:
Parallel.For(0, targetDatasetLength, x =>
{
var chunkData = this.GetNaNFreeData(data, x, kernelSize);
var isHighQuality = (chunkData.Length / (double)kernelSize) >= nanLimit;
if (isHighQuality)
{
result[x] = ArrayStatistics.Mean(chunkData);
}
else
{
result[x] = double.NaN;
}
});
break;
case AggregationMethod.MeanPolar:
double[] sin = new double[targetDatasetLength];
double[] cos = new double[targetDatasetLength];
double limit;
if (argument.Contains("*PI"))
{
limit = Double.Parse(argument.Replace("*PI", "")) * Math.PI;
}
else
{
limit = Double.Parse(argument);
}
var factor = 2 * Math.PI / limit;
Parallel.For(0, targetDatasetLength, x =>
{
var chunkData = this.GetNaNFreeData(data, x, kernelSize);
var length = chunkData.Length;
var isHighQuality = (length / (double)kernelSize) >= nanLimit;
if (isHighQuality)
{
for (int i = 0; i < chunkData.Length; i++)
{
sin[x] += Math.Sin(chunkData[i] * factor);
cos[x] += Math.Cos(chunkData[i] * factor);
}
result[x] = Math.Atan2(sin[x], cos[x]) / factor;
if (result[x] < 0)
{
result[x] += limit;
}
}
else
{
result[x] = double.NaN;
}
});
break;
case AggregationMethod.Min:
Parallel.For(0, targetDatasetLength, x =>
{
var chunkData = this.GetNaNFreeData(data, x, kernelSize);
var isHighQuality = (chunkData.Length / (double)kernelSize) >= nanLimit;
if (isHighQuality)
{
result[x] = ArrayStatistics.Minimum(chunkData);
}
else
{
result[x] = double.NaN;
}
});
break;
case AggregationMethod.Max:
Parallel.For(0, targetDatasetLength, x =>
{
var chunkData = this.GetNaNFreeData(data, x, kernelSize);
var isHighQuality = (chunkData.Length / (double)kernelSize) >= nanLimit;
if (isHighQuality)
{
result[x] = ArrayStatistics.Maximum(chunkData);
}
else
{
result[x] = double.NaN;
}
});
break;
case AggregationMethod.Std:
Parallel.For(0, targetDatasetLength, x =>
{
var chunkData = this.GetNaNFreeData(data, x, kernelSize);
var isHighQuality = (chunkData.Length / (double)kernelSize) >= nanLimit;
if (isHighQuality)
{
result[x] = ArrayStatistics.StandardDeviation(chunkData);
}
else
{
result[x] = double.NaN;
}
});
break;
case AggregationMethod.Rms:
Parallel.For(0, targetDatasetLength, x =>
{
var chunkData = this.GetNaNFreeData(data, x, kernelSize);
var isHighQuality = (chunkData.Length / (double)kernelSize) >= nanLimit;
if (isHighQuality)
{
result[x] = ArrayStatistics.RootMeanSquare(chunkData);
}
else
{
result[x] = double.NaN;
}
});
break;
case AggregationMethod.SampleAndHold:
Parallel.For(0, targetDatasetLength, x =>
{
var chunkData = this.GetNaNFreeData(data, x, kernelSize);
var isHighQuality = (chunkData.Length / (double)kernelSize) >= nanLimit;
if (isHighQuality)
{
result[x] = chunkData.First();
}
else
{
result[x] = double.NaN;
}
});
break;
case AggregationMethod.Sum:
Parallel.For(0, targetDatasetLength, x =>
{
var chunkData = this.GetNaNFreeData(data, x, kernelSize);
var isHighQuality = (chunkData.Length / (double)kernelSize) >= nanLimit;
if (isHighQuality)
{
result[x] = Vector <double> .Build.Dense(chunkData).Sum();
}
else
{
result[x] = double.NaN;
}
});
break;
default:
logger.LogWarning($"The aggregation method '{method}' is not known. Skipping period.");
break;
}
return(result);
}
private double[] ApplyAggregationFunction <T>(AggregationMethod method,
string argument,
int kernelSize,
T[] data,
byte[] status,
double nanLimit,
ILogger logger) where T : unmanaged
{
var targetDatasetLength = data.Length / kernelSize;
var result = new double[targetDatasetLength];
switch (method)
{
case AggregationMethod.MinBitwise:
T[] bitField_and = new T[targetDatasetLength];
Parallel.For(0, targetDatasetLength, x =>
{
var chunkData = this.GetNaNFreeData(data, status, x, kernelSize);
var length = chunkData.Length;
var isHighQuality = (length / (double)kernelSize) >= nanLimit;
if (isHighQuality)
{
for (int i = 0; i < length; i++)
{
if (i == 0)
{
bitField_and[x] = GenericBitOr <T> .BitOr(bitField_and[x], chunkData[i]);
}
else
{
bitField_and[x] = GenericBitAnd <T> .BitAnd(bitField_and[x], chunkData[i]);
}
}
result[x] = Convert.ToDouble(bitField_and[x]);
}
else
{
result[x] = double.NaN;
}
});
break;
case AggregationMethod.MaxBitwise:
T[] bitField_or = new T[targetDatasetLength];
Parallel.For(0, targetDatasetLength, x =>
{
var chunkData = this.GetNaNFreeData(data, status, x, kernelSize);
var length = chunkData.Length;
var isHighQuality = (length / (double)kernelSize) >= nanLimit;
if (isHighQuality)
{
for (int i = 0; i < length; i++)
{
bitField_or[x] = GenericBitOr <T> .BitOr(bitField_or[x], chunkData[i]);
}
result[x] = Convert.ToDouble(bitField_or[x]);
}
else
{
result[x] = double.NaN;
}
});
break;
default:
logger.LogWarning($"The aggregation method '{method}' is not known. Skipping period.");
break;
}
return(result);
}
private static void VerifyAggregateExpressionToken(string expectedEndPathIdentifier, AggregationMethod expectedVerb, string expectedAlias, AggregateExpressionToken actual)
{
actual.Expression.Should().NotBeNull();
var expression = actual.Expression as EndPathToken;
expression.Should().NotBeNull();
expression.Identifier.Should().Be(expectedEndPathIdentifier);
actual.Method.Should().Be(expectedVerb);
actual.Alias.Should().Be(expectedAlias);
}
/// <summary>
/// Get the observations or data values for an economic data series. Corresponds to http://api.stlouisfed.org/fred/series/observations
/// </summary>
/// <param name="seriesId">The id for a series.</param>
/// <param name="fileType">The type of file to send.</param>
/// <param name="filename">The where to save the file.</param>
/// <param name="observationStart">The start of the observation period.</param>
/// <param name="observationEnd">The end of the observation period.</param>
/// <param name="realtimeStart">The start of the real-time period.</param>
/// <param name="realtimeEnd">The end of the real-time period.</param>
/// <param name="vintageDates">A comma separated string of YYYY-MM-DD formatted dates in history (e.g. 2000-01-01,2005-02-24).
/// Vintage dates are used to download data as it existed on these specified dates in history. Vintage dates can be
/// specified instead of a real-time period using realtime_start and realtime_end.
///
/// Sometimes it may be useful to enter a vintage date that is not a date when the data values were revised.
/// For instance you may want to know the latest available revisions on 2001-09-11 (World Trade Center and
/// Pentagon attacks) or as of a Federal Open Market Committee (FOMC) meeting date. Entering a vintage date is
/// also useful to compare series on different releases with different release dates.</param>
/// <param name="limit">The maximum number of results to return. An integer between 1 and 100000, optional, default: 100000</param>
/// <param name="offset">non-negative integer, optional, default: 0</param>
/// <param name="order">Sort results is ascending or descending observation_date order.</param>
/// <param name="transformation">Type of data value transformation.</param>
/// <param name="frequency">An optional parameter that indicates a lower frequency to aggregate values to.
/// The FRED frequency aggregation feature converts higher frequency data series into lower frequency data series
/// (e.g. converts a monthly data series into an annual data series). In FRED, the highest frequency data is daily,
/// and the lowest frequency data is annual. There are 3 aggregation methods available- average, sum, and end of period.</param>
/// <param name="method">A key that indicates the aggregation method used for frequency aggregation.</param>
/// <param name="outputType">Output type:
/// 1. Observations by Real-Time Period
/// 2. Observations by Vintage Date, All Observations
/// 3. Observations by Vintage Date, New and Revised Observations Only
/// 4. Observations, Initial Release Only
/// </param>
/// <returns>Observations or data values for an economic data series.</returns>
public async Task GetSeriesObservationsFileAsync(string seriesId, FileType fileType, string filename,
DateTime observationStart,
DateTime observationEnd,
DateTime realtimeStart,
DateTime realtimeEnd,
IEnumerable<DateTime> vintageDates, int limit = 100000,
int offset = 0, SortOrder order = SortOrder.Ascending,
Transformation transformation = Transformation.None,
Frequency frequency = Frequency.None,
AggregationMethod method = AggregationMethod.Average,
OutputType outputType = OutputType.RealTime)
{
var vintageString = vintageDates.Aggregate(string.Empty,
(current, vintageDate) =>
current + (vintageDate.ToFredDateString() + ","));
if (vintageString.Length > 0)
{
vintageString = vintageString.Substring(0, vintageString.Length - 1);
}
var realtimeStartStr = (vintageString.Length == 0) ? realtimeStart.ToFredDateString() : string.Empty;
var realtimeEndStr = (vintageString.Length == 0) ? realtimeEnd.ToFredDateString() : string.Empty;
var url = String.Format(Urls.SeriesObservations, _key, seriesId, realtimeStartStr,
realtimeEndStr, limit, offset, Extensions.ToString(order),
observationStart.ToFredDateString(),
observationEnd.ToFredDateString(),
Extensions.ToString(transformation), Extensions.ToString(frequency),
Extensions.ToString(method), Extensions.ToString(outputType),
Extensions.ToString(fileType), vintageString);
try
{
if (fileType != FileType.Xml && !Path.GetExtension(filename).Equals(".zip", StringComparison.OrdinalIgnoreCase))
{
filename += ".zip";
}
await _downloader.DownloadFileAsync(url, filename);
}
catch (WebException exp)
{
var response = (HttpWebResponse)exp.Response;
var buffer = new byte[response.ContentLength];
response.GetResponseStream().Read(buffer, 0, buffer.Length);
var xml = Encoding.UTF8.GetString(buffer);
var start = xml.IndexOf("message=", StringComparison.OrdinalIgnoreCase);
if (start < 0)
{
throw;
}
start += 9;
var end = xml.LastIndexOf("\"", StringComparison.OrdinalIgnoreCase);
var message = xml.Substring(start, end - start);
throw new FredExecption(message, exp);
}
}
public void AAggNoAccessEnterLeave(bool enter, int index, AggregationMethod aggregationMethod)
{
}
/// <summary>
/// Get the observations or data values for an economic data series. Corresponds to http://api.stlouisfed.org/fred/series/observations
/// </summary>
/// <param name="seriesId">The id for a series.</param>
/// <param name="observationStart">The start of the observation period.</param>
/// <param name="observationEnd">The end of the observation period.</param>
/// <param name="realtimeStart">The start of the real-time period.</param>
/// <param name="realtimeEnd">The end of the real-time period.</param>
/// <param name="vintageDates">A comma separated string of YYYY-MM-DD formatted dates in history (e.g. 2000-01-01,2005-02-24).
/// Vintage dates are used to download data as it existed on these specified dates in history. Vintage dates can be
/// specified instead of a real-time period using realtime_start and realtime_end.
///
/// Sometimes it may be useful to enter a vintage date that is not a date when the data values were revised.
/// For instance you may want to know the latest available revisions on 2001-09-11 (World Trade Center and
/// Pentagon attacks) or as of a Federal Open Market Committee (FOMC) meeting date. Entering a vintage date is
/// also useful to compare series on different releases with different release dates.</param>
/// <param name="limit">The maximum number of results to return. An integer between 1 and 100000, optional, default: 100000</param>
/// <param name="offset">non-negative integer, optional, default: 0</param>
/// <param name="order">Sort results is ascending or descending observation_date order.</param>
/// <param name="transformation">Type of data value transformation.</param>
/// <param name="frequency">An optional parameter that indicates a lower frequency to aggregate values to.
/// The FRED frequency aggregation feature converts higher frequency data series into lower frequency data series
/// (e.g. converts a monthly data series into an annual data series). In FRED, the highest frequency data is daily,
/// and the lowest frequency data is annual. There are 3 aggregation methods available- average, sum, and end of period.</param>
/// <param name="method">A key that indicates the aggregation method used for frequency aggregation.</param>
/// <param name="outputType">Output type:
/// 1. Observations by Real-Time Period
/// 2. Observations by Vintage Date, All Observations
/// 3. Observations by Vintage Date, New and Revised Observations Only
/// 4. Observations, Initial Release Only
/// </param>
/// <returns>Observations or data values for an economic data series.</returns>
public async Task<IEnumerable<Observation>> GetSeriesObservationsAsync(string seriesId, DateTime observationStart,
DateTime observationEnd, DateTime realtimeStart,
DateTime realtimeEnd,
IEnumerable<DateTime> vintageDates, int limit = 100000,
int offset = 0, SortOrder order = SortOrder.Ascending,
Transformation transformation = Transformation.None,
Frequency frequency = Frequency.None,
AggregationMethod method = AggregationMethod.Average,
OutputType outputType = OutputType.RealTime)
{
var vintageString = vintageDates.Aggregate(string.Empty,
(current, vintageDate) =>
current + (vintageDate.ToFredDateString() + ","));
if (vintageString.Length > 0)
{
vintageString = vintageString.Substring(0, vintageString.Length - 1);
}
var realtimeStartStr = (vintageString.Length == 0) ? realtimeStart.ToFredDateString() : string.Empty;
var realtimeEndStr = (vintageString.Length == 0) ? realtimeEnd.ToFredDateString() : string.Empty;
var url = String.Format(Urls.SeriesObservations, _key, seriesId, realtimeStartStr,
realtimeEndStr, limit, offset, Extensions.ToString(order),
observationStart.ToFredDateString(),
observationEnd.ToFredDateString(),
Extensions.ToString(transformation), Extensions.ToString(frequency),
Extensions.ToString(method), Extensions.ToString(outputType), "xml", vintageString);
var root = await GetRootAsync(url);
return root.Elements("observation").Select(
element =>
{
var valElm = element.Attribute("value");
double? value = null;
if (valElm != null && !string.IsNullOrWhiteSpace(valElm.Value))
{
double dOut;
var success = double.TryParse(valElm.Value, out dOut);
if (success)
{
value = dOut;
}
}
return new Observation
{
RealtimeStart = element.Attribute("realtime_start").Value.ToFredDate(),
RealtimeEnd = element.Attribute("realtime_end").Value.ToFredDate(),
Date = element.Attribute("date").Value.ToFredDate(),
Value = value
};
}
).ToList();
}
/// <summary>
/// Get the observations or data values for an economic data series. Corresponds to http://api.stlouisfed.org/fred/series/observations
/// </summary>
/// <param name="seriesId">The id for a series.</param>
/// <param name="observationStart">The start of the observation period.</param>
/// <param name="observationEnd">The end of the observation period.</param>
/// <param name="vintageDates">A comma separated string of YYYY-MM-DD formatted dates in history (e.g. 2000-01-01,2005-02-24).
/// Vintage dates are used to download data as it existed on these specified dates in history. Vintage dates can be
/// specified instead of a real-time period using realtime_start and realtime_end.
///
/// Sometimes it may be useful to enter a vintage date that is not a date when the data values were revised.
/// For instance you may want to know the latest available revisions on 2001-09-11 (World Trade Center and
/// Pentagon attacks) or as of a Federal Open Market Committee (FOMC) meeting date. Entering a vintage date is
/// also useful to compare series on different releases with different release dates.</param>
/// <param name="limit">The maximum number of results to return. An integer between 1 and 100000, optional, default: 100000</param>
/// <param name="offset">non-negative integer, optional, default: 0</param>
/// <param name="order">Sort results is ascending or descending observation_date order.</param>
/// <param name="transformation">Type of data value transformation.</param>
/// <param name="frequency">An optional parameter that indicates a lower frequency to aggregate values to.
/// The FRED frequency aggregation feature converts higher frequency data series into lower frequency data series
/// (e.g. converts a monthly data series into an annual data series). In FRED, the highest frequency data is daily,
/// and the lowest frequency data is annual. There are 3 aggregation methods available- average, sum, and end of period.</param>
/// <param name="method">A key that indicates the aggregation method used for frequency aggregation.</param>
/// <param name="outputType">Output type:
/// 1. Observations by Real-Time Period
/// 2. Observations by Vintage Date, All Observations
/// 3. Observations by Vintage Date, New and Revised Observations Only
/// 4. Observations, Initial Release Only
/// </param>
/// <returns>Observations or data values for an economic data series.</returns>
public Task<IEnumerable<Observation>> GetSeriesObservationsAsync(string seriesId, DateTime observationStart,
DateTime observationEnd,
IEnumerable<DateTime> vintageDates,
int limit = 100000,
int offset = 0, SortOrder order = SortOrder.Ascending,
Transformation transformation = Transformation.None,
Frequency frequency = Frequency.None,
AggregationMethod method = AggregationMethod.Average,
OutputType outputType = OutputType.RealTime)
{
var now = CstTime();
return GetSeriesObservationsAsync(seriesId, observationStart, observationEnd, now, now, vintageDates, limit, offset, order, transformation, frequency, method, outputType);
}
/// <summary>
/// Get the observations or data values for an economic data series. Corresponds to http://api.stlouisfed.org/fred/series/observations
/// </summary>
/// <param name="seriesId">The id for a series.</param>
/// <param name="observationStart">The start of the observation period.</param>
/// <param name="observationEnd">The end of the observation period.</param>
/// <param name="realtimeStart">The start of the real-time period.</param>
/// <param name="realtimeEnd">The end of the real-time period.</param>
/// <param name="limit">The maximum number of results to return. An integer between 1 and 100000, optional, default: 100000</param>
/// <param name="offset">non-negative integer, optional, default: 0</param>
/// <param name="order">Sort results is ascending or descending observation_date order.</param>
/// <param name="transformation">Type of data value transformation.</param>
/// <param name="frequency">An optional parameter that indicates a lower frequency to aggregate values to.
/// The FRED frequency aggregation feature converts higher frequency data series into lower frequency data series
/// (e.g. converts a monthly data series into an annual data series). In FRED, the highest frequency data is daily,
/// and the lowest frequency data is annual. There are 3 aggregation methods available- average, sum, and end of period.</param>
/// <param name="method">A key that indicates the aggregation method used for frequency aggregation.</param>
/// <param name="outputType">Output type:
/// 1. Observations by Real-Time Period
/// 2. Observations by Vintage Date, All Observations
/// 3. Observations by Vintage Date, New and Revised Observations Only
/// 4. Observations, Initial Release Only
/// </param>
/// <returns>Observations or data values for an economic data series.</returns>
public Task<IEnumerable<Observation>> GetSeriesObservationsAsync(string seriesId, DateTime observationStart,
DateTime observationEnd, DateTime realtimeStart,
DateTime realtimeEnd,
int limit = 100000,
int offset = 0, SortOrder order = SortOrder.Ascending,
Transformation transformation = Transformation.None,
Frequency frequency = Frequency.None,
AggregationMethod method = AggregationMethod.Average,
OutputType outputType = OutputType.RealTime)
{
return GetSeriesObservationsAsync(seriesId, observationStart, observationEnd, realtimeStart, realtimeEnd, Enumerable.Empty<DateTime>(), limit, offset, order, transformation, frequency, method, outputType);
}
/// <summary>
/// Get the observations or data values for an economic data series. Corresponds to http://api.stlouisfed.org/fred/series/observations
/// </summary>
/// <param name="seriesId">The id for a series.</param>
/// <param name="limit">The maximum number of results to return. An integer between 1 and 100000, optional, default: 100000</param>
/// <param name="offset">non-negative integer, optional, default: 0</param>
/// <param name="order">Sort results is ascending or descending observation_date order.</param>
/// <param name="transformation">Type of data value transformation.</param>
/// <param name="frequency">An optional parameter that indicates a lower frequency to aggregate values to.
/// The FRED frequency aggregation feature converts higher frequency data series into lower frequency data series
/// (e.g. converts a monthly data series into an annual data series). In FRED, the highest frequency data is daily,
/// and the lowest frequency data is annual. There are 3 aggregation methods available- average, sum, and end of period.</param>
/// <param name="method">A key that indicates the aggregation method used for frequency aggregation.</param>
/// <param name="outputType">Output type:
/// 1. Observations by Real-Time Period
/// 2. Observations by Vintage Date, All Observations
/// 3. Observations by Vintage Date, New and Revised Observations Only
/// 4. Observations, Initial Release Only
/// </param>
/// <returns>Observations or data values for an economic data series.</returns>
public Task<IEnumerable<Observation>> GetSeriesObservationsAsync(string seriesId,
int limit = 100000,
int offset = 0, SortOrder order = SortOrder.Ascending,
Transformation transformation = Transformation.None,
Frequency frequency = Frequency.None,
AggregationMethod method = AggregationMethod.Average,
OutputType outputType = OutputType.RealTime)
{
var now = CstTime();
return GetSeriesObservationsAsync(seriesId, new DateTime(1776, 07, 04), new DateTime(9999, 12, 31), now, now, Enumerable.Empty<DateTime>(), limit, offset, order, transformation, frequency, method, outputType);
}
/// <summary>
/// Converts a <see cref="AggregationMethod"/> to a string.
/// </summary>
/// <param name="method">The aggregation method to convert.</param>
/// <returns>The converted string.</returns>
public static string ToString(AggregationMethod method)
{
var ret = "";
switch (method)
{
case AggregationMethod.Average:
ret = "avg";
break;
case AggregationMethod.Sum:
ret = "sum";
break;
case AggregationMethod.EndOfPeriod:
ret = "eop";
break;
}
return ret;
}
private IEdmTypeReference CreateAggregateExpressionTypeReference(SingleValueNode expression, AggregationMethod withVerb)
{
var expressionType = expression.TypeReference;
if (expressionType == null && aggregateExpressionsCache != null)
{
var openProperty = expression as SingleValueOpenPropertyAccessNode;
if (openProperty != null)
{
expressionType = GetTypeReferenceByPropertyName(openProperty.Name);
}
}
switch (withVerb)
{
case AggregationMethod.Average:
var expressionPrimitiveKind = expressionType.PrimitiveKind();
switch (expressionPrimitiveKind)
{
case EdmPrimitiveTypeKind.Int32:
case EdmPrimitiveTypeKind.Int64:
case EdmPrimitiveTypeKind.Double:
return EdmCoreModel.Instance.GetPrimitive(EdmPrimitiveTypeKind.Double, expressionType.IsNullable);
case EdmPrimitiveTypeKind.Decimal:
return EdmCoreModel.Instance.GetPrimitive(EdmPrimitiveTypeKind.Decimal, expressionType.IsNullable);
default:
throw new ODataException(
ODataErrorStrings.ApplyBinder_AggregateExpressionIncompatibleTypeForMethod(expression,
expressionPrimitiveKind));
}
case AggregationMethod.CountDistinct:
return EdmCoreModel.Instance.GetPrimitive(EdmPrimitiveTypeKind.Int64, false);
case AggregationMethod.Max:
case AggregationMethod.Min:
case AggregationMethod.Sum:
return expressionType;
default:
throw new ODataException(ODataErrorStrings.ApplyBinder_UnsupportedAggregateMethod(withVerb));
}
}