/// <summary>
/// Get left and right data padding
/// </summary>
/// <param name="metadata">Model metadata</param>
/// <returns>Left padding, right padding</returns>
public static DataPadding GetSeriesPadding(BlackFoxMetadata metadata)
{
int leftInput = int.MinValue;
int rightInput = 0;
int leftОutput = int.MinValue;
int rightOutput = 0;
for (int i = 0; i < metadata.InputShifts.Length; i++)
{
leftInput = Math.Max(leftInput, -metadata.InputShifts[i] + metadata.InputWindows[i]);
rightInput = Math.Max(rightInput, metadata.InputShifts[i]);
}
for (int i = 0; i < metadata.OutputShifts.Length; i++)
{
leftОutput = Math.Max(leftОutput, -metadata.OutputShifts[i] + metadata.OutputWindows[i]);
rightOutput = Math.Max(rightOutput, metadata.OutputShifts[i]);
}
int left_padding = Math.Max(Math.Max(0, leftInput - 1), leftОutput - 1);
left_padding = metadata.OutputSampleStep * (int)Math.Ceiling((decimal)left_padding / metadata.OutputSampleStep);
int right_padding = Math.Max(Math.Max(rightInput, rightOutput), 0);
return(new DataPadding(left_padding, right_padding));
}
private static List <double> GetOutputRow(double[][] data, BlackFoxMetadata metadata, int i, int outputCount, int inputOffset = 0)
{
List <double> yRow = new List <double>();
for (int j = 0; j < outputCount; j++)
{
int c = inputOffset + j;
int shift = metadata.OutputShifts[j];
int window = metadata.OutputWindows[j];
int offset = i + 1 + shift;
double[] d = GetColumnData(data, offset - window, offset, c);
yRow.AddRange(d);
}
return(yRow);
}
/// <summary>
/// Packing input and output data for series
/// </summary>
/// <param name="data"></param>
/// <param name="metadata"></param>
/// <returns>Packed data</returns>
public static PackedData PackDataForSeries(double[][] data, BlackFoxMetadata metadata)
{
int inputCount = metadata.InputShifts.Length;
int outputCount = metadata.OutputShifts.Length;
int sample_step = metadata.OutputSampleStep;
DataPadding padding = GetSeriesPadding(metadata);
List <List <double> > xData = new List <List <double> >();
List <List <double> > yData = new List <List <double> >();
int rows_count = data.Length - padding.Right;
for (int i = padding.Left; i < rows_count; i += sample_step)
{
List <double> xRow = GetInputRow(data, metadata, i, inputCount);
xData.Add(xRow);
List <double> yRow = GetOutputRow(data, metadata, i, outputCount, inputCount);
yData.Add(yRow);
}
return(new PackedData(xData, yData));
}
/// <summary>
/// Packing output data for series
/// </summary>
/// <param name="output_data">Data as numpy array</param>
/// <param name="metadata">Model metadata</param>
/// <returns>Packed data</returns>
public static List <List <double> > PackOutputDataForSeries(double[][] output_data, BlackFoxMetadata metadata)
{
int outputCount = metadata.OutputShifts.Length;
int sampleStep = metadata.OutputSampleStep;
DataPadding padding = GetSeriesPadding(metadata);
List <List <double> > yData = new List <List <double> >();
int rowsCount = output_data.Length - padding.Right;
for (int i = padding.Left; i < rowsCount; i += sampleStep)
{
List <double> yRow = GetOutputRow(output_data, metadata, i, outputCount, 0);
yData.Add(yRow);
}
return(yData);
}
/// <summary>
/// Packing input data for series
/// </summary>
/// <param name="inputData">Data as numpy array</param>
/// <param name="metadata">Model metadata</param>
/// <returns>Packed data</returns>
public static List <List <double> > PackInputDataForSeries(IEnumerable <IEnumerable <double> > inputData, BlackFoxMetadata metadata)
{
DataPadding padding = GetSeriesPadding(metadata);
List <List <double> > xData = new List <List <double> >();;
int rowsCount = inputData.Count() - padding.Right;
for (int i = padding.Left; i < rowsCount; i += metadata.OutputSampleStep)
{
List <double> xRow = GetInputRow(inputData, metadata, i, metadata.InputShifts.Length);
xData.Add(xRow);
}
return(xData);
}
private static List <double> GetInputRow(IEnumerable <IEnumerable <double> > data, BlackFoxMetadata metadata, int i, int input_count)
{
List <double> xRow = new List <double>();
for (int j = 0; j < input_count; j++)
{
int shift = metadata.InputShifts[j];
int window = metadata.InputWindows[j];
int step = metadata.InputSteps[j];
string aggregationType = metadata.InputAggregationTypes[j];
int offset = i + 1 + shift;
double[] d = GetColumnData(data, offset - window, offset, j);
int n = d.Length;
List <double> newD = new List <double>();
if (aggregationType == "sum")
{
for (int s = (n) % step; s < n - 1; s += step)
{
newD.Add(GetColumnData(d, s, s + step).Sum());
}
}
else if (aggregationType == "avg")
{
for (int s = (n) % step; s < n - 1; s += step)
{
newD.Add(GetColumnData(d, s, s + step).Average());
}
}
else
{
for (int s = (n - 1) % step; s < n; s += step)
{
newD.Add(d[s]);
}
}
xRow.AddRange(newD);
}
return(xRow);
}
/// <summary>
/// Remove insignificant columns from data, if model has feature selection
/// </summary>
/// <param name="data"></param>
/// <param name="metadata">Model metadata</param>
/// <returns>New data</returns>
public static IEnumerable <IEnumerable <double> > RemoveNotUsedInputs(IEnumerable <IEnumerable <double> > data, BlackFoxMetadata metadata)
{
if (metadata.IsInputUsed != null)
{
List <List <double> > newData = new List <List <double> >();
int j = 0;
foreach (var row in data)
{
j = 0;
List <double> newRow = new List <double>();
foreach (var d in row)
{
if (metadata.IsInputUsed[j])
{
newRow.Add(d);
}
j++;
}
newData.Add(newRow);
}
return(newData);
}
else
{
return(data);
}
}
/// <summary>
/// Scale data from normalized values to real values.Use after prediction.
/// </summary>
/// <param name="outputData">Output data</param>
/// <param name="metadata">Model metadata</param>
/// <param name="ignoreIntegratedScaler">If False(default), only scale data if model does not contain integrated scaler</param>
/// <returns>Scaled data</returns>
public static IEnumerable <IEnumerable <double> > ScaleOutputData(IEnumerable <IEnumerable <double> > outputData, BlackFoxMetadata metadata, bool ignoreIntegratedScaler = false)
{
if (ignoreIntegratedScaler || !metadata.IsScalerIntegrated)
{
return(ScaleDataFromConfig(
outputData,
metadata.ScalerConfig.Output,
metadata.ScalerName
));
}
else
{
return(outputData);
}
}
/// <summary>
/// Prepare the input for prediction with the following steps
/// 1. removing insignificant columns
/// 2. packing data for series
/// 3. scaling(normalizing) values
/// </summary>
/// <param name="inputData">Input data</param>
/// <param name="metadata">Model metadata</param>
/// <returns>Prepared values</returns>
public static IEnumerable <IEnumerable <double> > PrepareInputData(IEnumerable <IEnumerable <double> > inputData, BlackFoxMetadata metadata)
{
var usedInputs = Input.RemoveNotUsedInputs(inputData, metadata);
if (metadata.HasRolling)
{
usedInputs = Series.PackInputDataForSeries(usedInputs, metadata);
}
return(Scaler.ScaleInputData(usedInputs, metadata));
}