/// <summary> /// Get left and right data padding /// </summary> /// <param name="metadata">Model metadata</param> /// <returns>Left padding, right padding</returns> public static DataPadding GetSeriesPadding(BlackFoxMetadata metadata) { int leftInput = int.MinValue; int rightInput = 0; int leftОutput = int.MinValue; int rightOutput = 0; for (int i = 0; i < metadata.InputShifts.Length; i++) { leftInput = Math.Max(leftInput, -metadata.InputShifts[i] + metadata.InputWindows[i]); rightInput = Math.Max(rightInput, metadata.InputShifts[i]); } for (int i = 0; i < metadata.OutputShifts.Length; i++) { leftОutput = Math.Max(leftОutput, -metadata.OutputShifts[i] + metadata.OutputWindows[i]); rightOutput = Math.Max(rightOutput, metadata.OutputShifts[i]); } int left_padding = Math.Max(Math.Max(0, leftInput - 1), leftОutput - 1); left_padding = metadata.OutputSampleStep * (int)Math.Ceiling((decimal)left_padding / metadata.OutputSampleStep); int right_padding = Math.Max(Math.Max(rightInput, rightOutput), 0); return(new DataPadding(left_padding, right_padding)); }
private static List <double> GetOutputRow(double[][] data, BlackFoxMetadata metadata, int i, int outputCount, int inputOffset = 0) { List <double> yRow = new List <double>(); for (int j = 0; j < outputCount; j++) { int c = inputOffset + j; int shift = metadata.OutputShifts[j]; int window = metadata.OutputWindows[j]; int offset = i + 1 + shift; double[] d = GetColumnData(data, offset - window, offset, c); yRow.AddRange(d); } return(yRow); }
/// <summary> /// Packing input and output data for series /// </summary> /// <param name="data"></param> /// <param name="metadata"></param> /// <returns>Packed data</returns> public static PackedData PackDataForSeries(double[][] data, BlackFoxMetadata metadata) { int inputCount = metadata.InputShifts.Length; int outputCount = metadata.OutputShifts.Length; int sample_step = metadata.OutputSampleStep; DataPadding padding = GetSeriesPadding(metadata); List <List <double> > xData = new List <List <double> >(); List <List <double> > yData = new List <List <double> >(); int rows_count = data.Length - padding.Right; for (int i = padding.Left; i < rows_count; i += sample_step) { List <double> xRow = GetInputRow(data, metadata, i, inputCount); xData.Add(xRow); List <double> yRow = GetOutputRow(data, metadata, i, outputCount, inputCount); yData.Add(yRow); } return(new PackedData(xData, yData)); }
/// <summary> /// Packing output data for series /// </summary> /// <param name="output_data">Data as numpy array</param> /// <param name="metadata">Model metadata</param> /// <returns>Packed data</returns> public static List <List <double> > PackOutputDataForSeries(double[][] output_data, BlackFoxMetadata metadata) { int outputCount = metadata.OutputShifts.Length; int sampleStep = metadata.OutputSampleStep; DataPadding padding = GetSeriesPadding(metadata); List <List <double> > yData = new List <List <double> >(); int rowsCount = output_data.Length - padding.Right; for (int i = padding.Left; i < rowsCount; i += sampleStep) { List <double> yRow = GetOutputRow(output_data, metadata, i, outputCount, 0); yData.Add(yRow); } return(yData); }
/// <summary> /// Packing input data for series /// </summary> /// <param name="inputData">Data as numpy array</param> /// <param name="metadata">Model metadata</param> /// <returns>Packed data</returns> public static List <List <double> > PackInputDataForSeries(IEnumerable <IEnumerable <double> > inputData, BlackFoxMetadata metadata) { DataPadding padding = GetSeriesPadding(metadata); List <List <double> > xData = new List <List <double> >();; int rowsCount = inputData.Count() - padding.Right; for (int i = padding.Left; i < rowsCount; i += metadata.OutputSampleStep) { List <double> xRow = GetInputRow(inputData, metadata, i, metadata.InputShifts.Length); xData.Add(xRow); } return(xData); }
private static List <double> GetInputRow(IEnumerable <IEnumerable <double> > data, BlackFoxMetadata metadata, int i, int input_count) { List <double> xRow = new List <double>(); for (int j = 0; j < input_count; j++) { int shift = metadata.InputShifts[j]; int window = metadata.InputWindows[j]; int step = metadata.InputSteps[j]; string aggregationType = metadata.InputAggregationTypes[j]; int offset = i + 1 + shift; double[] d = GetColumnData(data, offset - window, offset, j); int n = d.Length; List <double> newD = new List <double>(); if (aggregationType == "sum") { for (int s = (n) % step; s < n - 1; s += step) { newD.Add(GetColumnData(d, s, s + step).Sum()); } } else if (aggregationType == "avg") { for (int s = (n) % step; s < n - 1; s += step) { newD.Add(GetColumnData(d, s, s + step).Average()); } } else { for (int s = (n - 1) % step; s < n; s += step) { newD.Add(d[s]); } } xRow.AddRange(newD); } return(xRow); }
/// <summary> /// Remove insignificant columns from data, if model has feature selection /// </summary> /// <param name="data"></param> /// <param name="metadata">Model metadata</param> /// <returns>New data</returns> public static IEnumerable <IEnumerable <double> > RemoveNotUsedInputs(IEnumerable <IEnumerable <double> > data, BlackFoxMetadata metadata) { if (metadata.IsInputUsed != null) { List <List <double> > newData = new List <List <double> >(); int j = 0; foreach (var row in data) { j = 0; List <double> newRow = new List <double>(); foreach (var d in row) { if (metadata.IsInputUsed[j]) { newRow.Add(d); } j++; } newData.Add(newRow); } return(newData); } else { return(data); } }
/// <summary> /// Scale data from normalized values to real values.Use after prediction. /// </summary> /// <param name="outputData">Output data</param> /// <param name="metadata">Model metadata</param> /// <param name="ignoreIntegratedScaler">If False(default), only scale data if model does not contain integrated scaler</param> /// <returns>Scaled data</returns> public static IEnumerable <IEnumerable <double> > ScaleOutputData(IEnumerable <IEnumerable <double> > outputData, BlackFoxMetadata metadata, bool ignoreIntegratedScaler = false) { if (ignoreIntegratedScaler || !metadata.IsScalerIntegrated) { return(ScaleDataFromConfig( outputData, metadata.ScalerConfig.Output, metadata.ScalerName )); } else { return(outputData); } }
/// <summary> /// Prepare the input for prediction with the following steps /// 1. removing insignificant columns /// 2. packing data for series /// 3. scaling(normalizing) values /// </summary> /// <param name="inputData">Input data</param> /// <param name="metadata">Model metadata</param> /// <returns>Prepared values</returns> public static IEnumerable <IEnumerable <double> > PrepareInputData(IEnumerable <IEnumerable <double> > inputData, BlackFoxMetadata metadata) { var usedInputs = Input.RemoveNotUsedInputs(inputData, metadata); if (metadata.HasRolling) { usedInputs = Series.PackInputDataForSeries(usedInputs, metadata); } return(Scaler.ScaleInputData(usedInputs, metadata)); }