private static Lazy <TimeSeriesPredictionEngine <ModelInput, ModelOutput> > GetModel(MLContext context)
{
ITransformer model = context.Model.Load(Model_File, out _);
var engine = model.CreateTimeSeriesEngine <ModelInput, ModelOutput>(context);
return(new Lazy <TimeSeriesPredictionEngine <ModelInput, ModelOutput> >(engine));
}
public void Predict(string inputDataFile)
{
if (!File.Exists(ModelPath))
{
Console.WriteLine($"Failed to find model at {ModelPath}");
return;
}
if (!File.Exists(inputDataFile))
{
Console.WriteLine($"Failed to find input data at {inputDataFile}");
return;
}
ITransformer mlModel = MlContext.Model.Load(ModelPath, out var modelInputSchema);
if (mlModel == null)
{
Console.WriteLine("Failed to load model");
return;
}
// Using ML Transforms TimeSeries
var predictionEngine = mlModel.CreateTimeSeriesEngine <NetworkTrafficHistory, NetworkTrafficPrediction>(MlContext);
// var predictionEngine = MlContext.Model.CreatePredictionEngine<NetworkTrafficHistory, NetworkTrafficPrediction>(mlModel);
var inputData =
MlContext.Data.LoadFromTextFile <NetworkTrafficHistory>(inputDataFile, separatorChar: ',');
var rows = MlContext.Data.CreateEnumerable <NetworkTrafficHistory>(inputData, false);
Console.WriteLine($"Based on input file ({inputDataFile}):");
foreach (var row in rows)
{
var prediction = predictionEngine.Predict(row);
Console.Write($"HOST: {row.HostMachine} TIMESTAMP: {row.Timestamp} TRANSFER: {row.BytesTransferred} ");
Console.Write($"ALERT: {prediction.Prediction[0]} SCORE: {prediction.Prediction[1]:f2} P-VALUE: {prediction.Prediction[2]:F2}{Environment.NewLine}");
}
}
// This example creates a time series (list of Data with the i-th element
// corresponding to the i-th time slot). It demonstrates stateful prediction
// engine that updates the state of the model and allows for
// saving/reloading. The estimator is applied then to identify points where
// data distribution changed. This estimator can account for temporal
// seasonality in the data.
public static void Example()
{
// Create a new ML context, for ML.NET operations. It can be used for
// exception tracking and logging, as well as the source of randomness.
var ml = new MLContext();
// Generate sample series data with a recurring pattern
const int SeasonalitySize = 5;
const int TrainingSeasons = 3;
const int TrainingSize = SeasonalitySize * TrainingSeasons;
var data = new List <TimeSeriesData>()
{
new TimeSeriesData(0),
new TimeSeriesData(1),
new TimeSeriesData(2),
new TimeSeriesData(3),
new TimeSeriesData(4),
new TimeSeriesData(0),
new TimeSeriesData(1),
new TimeSeriesData(2),
new TimeSeriesData(3),
new TimeSeriesData(4),
new TimeSeriesData(0),
new TimeSeriesData(1),
new TimeSeriesData(2),
new TimeSeriesData(3),
new TimeSeriesData(4),
};
// Convert data to IDataView.
var dataView = ml.Data.LoadFromEnumerable(data);
// Setup SsaChangePointDetector arguments
var inputColumnName = nameof(TimeSeriesData.Value);
var outputColumnName = nameof(ChangePointPrediction.Prediction);
int confidence = 95;
int changeHistoryLength = 8;
// Train the change point detector.
ITransformer model = ml.Transforms.DetectChangePointBySsa(
outputColumnName, inputColumnName, confidence, changeHistoryLength,
TrainingSize, SeasonalitySize + 1).Fit(dataView);
// Create a prediction engine from the model for feeding new data.
var engine = model.CreateTimeSeriesEngine <TimeSeriesData,
ChangePointPrediction>(ml);
// Start streaming new data points with no change point to the
// prediction engine.
Console.WriteLine($"Output from ChangePoint predictions on new data:");
Console.WriteLine("Data\tAlert\tScore\tP-Value\tMartingale value");
// Output from ChangePoint predictions on new data:
// Data Alert Score P-Value Martingale value
for (int i = 0; i < 5; i++)
{
PrintPrediction(i, engine.Predict(new TimeSeriesData(i)));
}
// 0 0 -1.01 0.50 0.00
// 1 0 -0.24 0.22 0.00
// 2 0 -0.31 0.30 0.00
// 3 0 0.44 0.01 0.00
// 4 0 2.16 0.00 0.24
// Now stream data points that reflect a change in trend.
for (int i = 0; i < 5; i++)
{
int value = (i + 1) * 100;
PrintPrediction(value, engine.Predict(new TimeSeriesData(value)));
}
// 100 0 86.23 0.00 2076098.24
// 200 0 171.38 0.00 809668524.21
// 300 1 256.83 0.01 22130423541.93 <-- alert is on, note that delay is expected
// 400 0 326.55 0.04 241162710263.29
// 500 0 364.82 0.08 597660527041.45 <-- saved to disk
// Now we demonstrate saving and loading the model.
// Save the model that exists within the prediction engine.
// The engine has been updating this model with every new data point.
byte[] modelBytes;
using (var stream = new MemoryStream())
{
engine.CheckPoint(ml, stream);
modelBytes = stream.ToArray();
}
// Load the model.
using (var stream = new MemoryStream(modelBytes))
model = ml.Model.Load(stream, out DataViewSchema schema);
// We must create a new prediction engine from the persisted model.
engine = model.CreateTimeSeriesEngine <TimeSeriesData,
ChangePointPrediction>(ml);
// Run predictions on the loaded model.
for (int i = 0; i < 5; i++)
{
int value = (i + 1) * 100;
PrintPrediction(value, engine.Predict(new TimeSeriesData(value)));
}
// 100 0 -58.58 0.15 1096021098844.34 <-- loaded from disk and running new predictions
// 200 0 -41.24 0.20 97579154688.98
// 300 0 -30.61 0.24 95319753.87
// 400 0 58.87 0.38 14.24
// 500 0 219.28 0.36 0.05
}
public void ChangePointDetectionWithSeasonalityPredictionEngine()
{
const int changeHistorySize = 10;
const int seasonalitySize = 10;
const int numberOfSeasonsInTraining = 5;
const int maxTrainingSize = numberOfSeasonsInTraining * seasonalitySize;
List <Data> data = new List <Data>();
var ml = new MLContext(seed: 1);
var dataView = ml.Data.LoadFromEnumerable(data);
for (int j = 0; j < numberOfSeasonsInTraining; j++)
{
for (int i = 0; i < seasonalitySize; i++)
{
data.Add(new Data(i));
}
}
for (int i = 0; i < changeHistorySize; i++)
{
data.Add(new Data(i * 100));
}
// Pipeline.
var pipeline = ml.Transforms.Text.FeaturizeText("Text_Featurized", "Text")
.Append(ml.Transforms.Conversion.ConvertType("Value", "Value", DataKind.Single))
.Append(new SsaChangePointEstimator(ml, new SsaChangePointDetector.Options()
{
Confidence = 95,
Source = "Value",
Name = "Change",
ChangeHistoryLength = changeHistorySize,
TrainingWindowSize = maxTrainingSize,
SeasonalWindowSize = seasonalitySize
}));
// Train.
var model = pipeline.Fit(dataView);
//Model 1: Prediction #1.
var engine = model.CreateTimeSeriesEngine <Data, Prediction>(ml);
var prediction = engine.Predict(new Data(1));
Assert.Equal(0, prediction.Change[0], precision: 7); // Alert
Assert.Equal(1.1661833524703979, prediction.Change[1], precision: 5); // Raw score
Assert.Equal(0.5, prediction.Change[2], precision: 7); // P-Value score
Assert.Equal(5.1200000000000114E-08, prediction.Change[3], precision: 7); // Martingale score
//Model 1: Checkpoint.
var modelPath = "temp.zip";
engine.CheckPoint(ml, modelPath);
//Model 1: Prediction #2
prediction = engine.Predict(new Data(1));
Assert.Equal(0, prediction.Change[0], precision: 7); // Alert
Assert.Equal(0.12216401100158691, prediction.Change[1], precision: 5); // Raw score
Assert.Equal(0.14823824685192111, prediction.Change[2], precision: 5); // P-Value score
Assert.Equal(1.5292508189989167E-07, prediction.Change[3], precision: 7); // Martingale score
// Load Model 1.
ITransformer model2 = null;
using (var file = File.OpenRead(modelPath))
model2 = ml.Model.Load(file, out var schema);
//Predict and expect the same result after checkpointing(Prediction #2).
engine = model2.CreateTimeSeriesEngine <Data, Prediction>(ml);
prediction = engine.Predict(new Data(1));
Assert.Equal(0, prediction.Change[0], precision: 7); // Alert
Assert.Equal(0.12216401100158691, prediction.Change[1], precision: 5); // Raw score
Assert.Equal(0.14823824685192111, prediction.Change[2], precision: 5); // P-Value score
Assert.Equal(1.5292508189989167E-07, prediction.Change[3], precision: 5); // Martingale score
}
public void ChangePointDetectionWithSeasonalityPredictionEngineNoColumn()
{
const int changeHistorySize = 10;
const int seasonalitySize = 10;
const int numberOfSeasonsInTraining = 5;
const int maxTrainingSize = numberOfSeasonsInTraining * seasonalitySize;
List <Data> data = new List <Data>();
var ml = new MLContext(seed: 1);
var dataView = ml.Data.LoadFromEnumerable(data);
for (int j = 0; j < numberOfSeasonsInTraining; j++)
{
for (int i = 0; i < seasonalitySize; i++)
{
data.Add(new Data(i));
}
}
for (int i = 0; i < changeHistorySize; i++)
{
data.Add(new Data(i * 100));
}
// Pipeline.
var pipeline = ml.Transforms.Text.FeaturizeText("Text_Featurized", "Text")
.Append(new SsaChangePointEstimator(ml, new SsaChangePointDetector.Options()
{
Confidence = 95,
Source = "Value",
Name = "Change",
ChangeHistoryLength = changeHistorySize,
TrainingWindowSize = maxTrainingSize,
SeasonalWindowSize = seasonalitySize
}));
// Train.
var model = pipeline.Fit(dataView);
//Create prediction function.
var engine = model.CreateTimeSeriesEngine <Data, Prediction1>(ml);
//Checkpoint with no inputs passed at prediction.
var modelPath = "temp.zip";
engine.CheckPoint(ml, modelPath);
//Load time series model and we will use this to pass two inputs and compare the raw score
//with "engine".
ITransformer model2 = null;
using (var file = File.OpenRead(modelPath))
model2 = ml.Model.Load(file, out var schema);
//Raw score after state gets updated with two inputs.
var engine2 = model2.CreateTimeSeriesEngine <Data, Prediction>(ml);
var prediction2 = engine2.Predict(new Data(1));
//Raw score after first input.
Assert.Equal(1.1661833524703979, prediction2.Change[1], precision: 5); // Raw score
prediction2 = engine2.Predict(new Data(1));
//Raw score after second input.
Assert.Equal(0.12216401100158691, prediction2.Change[1], precision: 5); // Raw score
//Even though time series column is not requested it will
// pass the observation through time series transform and update the state with the first input.
var prediction = engine.Predict(new Data(1));
Assert.Equal(-1, prediction.Random);
//Save the model with state updated with just one input.
engine.CheckPoint(ml, modelPath + 1);
ITransformer model3 = null;
using (var file = File.OpenRead(modelPath + 1))
model3 = ml.Model.Load(file, out var schema);
//Load the model with state updated with just one input, then pass in the second input
//and raw score should match the raw score obtained by passing the two input in the first model.
var engine3 = model3.CreateTimeSeriesEngine <Data, Prediction>(ml);
var prediction3 = engine3.Predict(new Data(1));
Assert.Equal(0.12216401100158691, prediction2.Change[1], precision: 5); // Raw score
}
public async Task <IActionResult> GetProductUnitDemandEstimation(string productId)
{
// Get product history.
var productHistory = await _queries.GetProductDataAsync(productId);
// Supplement the history with synthetic data.
var supplementedProductHistory = TimeSeriesDataGenerator.SupplementData(mlContext, productHistory);
var supplementedProductHistoryLength = supplementedProductHistory.Count(); // 36
var supplementedProductDataView = mlContext.Data.LoadFromEnumerable(supplementedProductHistory);
// Create and add the forecast estimator to the pipeline.
IEstimator <ITransformer> forecastEstimator = mlContext.Forecasting.ForecastBySsa(
outputColumnName: nameof(ProductUnitTimeSeriesPrediction.ForecastedProductUnits),
inputColumnName: nameof(ProductData.units), // This is the column being forecasted.
windowSize: 12, // Window size is set to the time period represented in the product data cycle; our product cycle is based on 12 months, so this is set to a factor of 12, e.g. 3.
seriesLength: supplementedProductHistoryLength, // TODO: Need clarification on what this should be set to; assuming product series length for now.
trainSize: supplementedProductHistoryLength, // TODO: Need clarification on what this should be set to; assuming product series length for now.
horizon: 1, // Indicates the number of values to forecast; 1 indicates that the next month of product units will be forecasted.
confidenceLevel: 0.95f, // TODO: Is this the same as prediction interval, where this indicates that we are 95% confidence that the forecasted value will fall within the interval range?
confidenceLowerBoundColumn: nameof(ProductUnitTimeSeriesPrediction.ConfidenceLowerBound), // TODO: See above comment.
confidenceUpperBoundColumn: nameof(ProductUnitTimeSeriesPrediction.ConfidenceUpperBound)); // TODO: See above comment.
// Train the forecasting model for the specified product's data series.
ITransformer forecastTransformer = forecastEstimator.Fit(supplementedProductDataView);
// Create the forecast engine used for creating predictions.
TimeSeriesPredictionEngine <ProductData, ProductUnitTimeSeriesPrediction> forecastEngine = forecastTransformer.CreateTimeSeriesEngine <ProductData, ProductUnitTimeSeriesPrediction>(mlContext);
// Predict.
var nextMonthUnitDemandEstimation = forecastEngine.Predict();
return(Ok(nextMonthUnitDemandEstimation.ForecastedProductUnits.First()));
}
static void Predict()
{
//prediction engine based on our fitted model
TimeSeriesPredictionEngine <BTCDataModel, PredictedSeriesDataModel> forecastEngine = forecastTransformer.CreateTimeSeriesEngine <BTCDataModel, PredictedSeriesDataModel>(mlContext);
//call to predict the next 5 minutes
PredictedSeriesDataModel predictions = forecastEngine.Predict();
//write our predictions
for (int i = 0; i < predictions.ForecastedPrice.Count(); i++)
{
lastTime = lastTime.AddMinutes(1);
Console.WriteLine("{0} price: {1}, low: {2}, high: {3}, actual: {4}", lastTime, predictions.ForecastedPrice[i].ToString(), predictions.ConfidenceLowerBound[i].ToString(), predictions.ConfidenceUpperBound[i].ToString(), testingData[i].Amount);
}
//instead of saving, we use checkpoint. This allows us to continue training with updated data and not need to keep such a large data set
//so we can append Jan. 2021 without having everythign before to train the model speeding up the process
forecastEngine.CheckPoint(mlContext, fileName);
}
// This example creates a time series (list of Data with the i-th element
// corresponding to the i-th time slot). The estimator is applied then to
// identify spiking points in the series. This estimator can account for
// temporal seasonality in the data.
public static void Example()
{
// Create a new ML context, for ML.NET operations. It can be used for
// exception tracking and logging, as well as the source of randomness.
var ml = new MLContext();
// Generate sample series data with a recurring pattern
const int SeasonalitySize = 5;
const int TrainingSeasons = 3;
const int TrainingSize = SeasonalitySize * TrainingSeasons;
var data = new List <TimeSeriesData>()
{
new TimeSeriesData(0),
new TimeSeriesData(1),
new TimeSeriesData(2),
new TimeSeriesData(3),
new TimeSeriesData(4),
new TimeSeriesData(0),
new TimeSeriesData(1),
new TimeSeriesData(2),
new TimeSeriesData(3),
new TimeSeriesData(4),
new TimeSeriesData(0),
new TimeSeriesData(1),
new TimeSeriesData(2),
new TimeSeriesData(3),
new TimeSeriesData(4),
};
// Convert data to IDataView.
var dataView = ml.Data.LoadFromEnumerable(data);
// Setup IidSpikeDetector arguments
var inputColumnName = nameof(TimeSeriesData.Value);
var outputColumnName = nameof(SsaSpikePrediction.Prediction);
// Train the change point detector.
ITransformer model = ml.Transforms.DetectSpikeBySsa(outputColumnName,
inputColumnName, 95, 8, TrainingSize, SeasonalitySize + 1).Fit(
dataView);
// Create a prediction engine from the model for feeding new data.
var engine = model.CreateTimeSeriesEngine <TimeSeriesData,
SsaSpikePrediction>(ml);
// Start streaming new data points with no change point to the
// prediction engine.
Console.WriteLine($"Output from spike predictions on new data:");
Console.WriteLine("Data\tAlert\tScore\tP-Value");
// Output from spike predictions on new data:
// Data Alert Score P-Value
for (int j = 0; j < 2; j++)
{
for (int i = 0; i < 5; i++)
{
PrintPrediction(i, engine.Predict(new TimeSeriesData(i)));
}
}
// 0 0 -1.01 0.50
// 1 0 -0.24 0.22
// 2 0 -0.31 0.30
// 3 0 0.44 0.01
// 4 0 2.16 0.00
// 0 0 -0.78 0.27
// 1 0 -0.80 0.30
// 2 0 -0.84 0.31
// 3 0 0.33 0.31
// 4 0 2.21 0.07
// Now send a data point that reflects a spike.
PrintPrediction(100, engine.Predict(new TimeSeriesData(100)));
// 100 1 86.17 0.00 <-- alert is on, predicted spike
// Now we demonstrate saving and loading the model.
// Save the model that exists within the prediction engine.
// The engine has been updating this model with every new data point.
var modelPath = "model.zip";
engine.CheckPoint(ml, modelPath);
// Load the model.
using (var file = File.OpenRead(modelPath))
model = ml.Model.Load(file, out DataViewSchema schema);
// We must create a new prediction engine from the persisted model.
engine = model.CreateTimeSeriesEngine <TimeSeriesData,
SsaSpikePrediction>(ml);
// Run predictions on the loaded model.
for (int i = 0; i < 5; i++)
{
PrintPrediction(i, engine.Predict(new TimeSeriesData(i)));
}
// 0 0 -2.74 0.40 <-- saved to disk, re-loaded, and running new predictions
// 1 0 -1.47 0.42
// 2 0 -17.50 0.24
// 3 0 -30.82 0.16
// 4 0 -23.24 0.28
}
public IEnumerable <RatioAnalysis> Predict(IEnumerable <RatioAnalysis> data)
{
var mlContext = new MLContext(seed: 1);
IEstimator <ITransformer> costofgoodsForcaster = mlContext.Forecasting.ForecastBySsa(
outputColumnName: nameof(RatioAnalysisPrediction.Forecasted),
inputColumnName: nameof(RatioAnalysis.CostOfGoods), // This is the column being forecasted.
windowSize: 12, // Window size is set to the time period represented in the product data cycle; our product cycle is based on 12 months, so this is set to a factor of 12, e.g. 3.
seriesLength: data.Count(), // This parameter specifies the number of data points that are used when performing a forecast.
trainSize: data.Count(), // This parameter specifies the total number of data points in the input time series, starting from the beginning.
horizon: 3, // Indicates the number of values to forecast; 3 indicates that the next 3 months of product units will be forecasted.
confidenceLevel: 0.75f, // Indicates the likelihood the real observed value will fall within the specified interval bounds.
confidenceLowerBoundColumn: nameof(RatioAnalysisPrediction.ConfidenceLowerBound), //This is the name of the column that will be used to store the lower interval bound for each forecasted value.
confidenceUpperBoundColumn: nameof(RatioAnalysisPrediction.ConfidenceUpperBound)); //This is the name of the column that will be used to store the upper interval bound for each forecasted value.
IEstimator <ITransformer> inventoryForcaster = mlContext.Forecasting.ForecastBySsa(
outputColumnName: nameof(RatioAnalysisPrediction.Forecasted),
inputColumnName: nameof(RatioAnalysis.Inventory), // This is the column being forecasted.
windowSize: 12, // Window size is set to the time period represented in the product data cycle; our product cycle is based on 12 months, so this is set to a factor of 12, e.g. 3.
seriesLength: data.Count(), // This parameter specifies the number of data points that are used when performing a forecast.
trainSize: data.Count(), // This parameter specifies the total number of data points in the input time series, starting from the beginning.
horizon: 3, // Indicates the number of values to forecast; 3 indicates that the next 3 months of product units will be forecasted.
confidenceLevel: 0.75f, // Indicates the likelihood the real observed value will fall within the specified interval bounds.
confidenceLowerBoundColumn: nameof(RatioAnalysisPrediction.ConfidenceLowerBound), //This is the name of the column that will be used to store the lower interval bound for each forecasted value.
confidenceUpperBoundColumn: nameof(RatioAnalysisPrediction.ConfidenceUpperBound)); //This is the name of the column that will be used to store the upper interval bound for each forecasted value.
IEstimator <ITransformer> turnoverForcaster = mlContext.Forecasting.ForecastBySsa(
outputColumnName: nameof(RatioAnalysisPrediction.Forecasted),
inputColumnName: nameof(RatioAnalysis.Turnover), // This is the column being forecasted.
windowSize: 12, // Window size is set to the time period represented in the product data cycle; our product cycle is based on 12 months, so this is set to a factor of 12, e.g. 3.
seriesLength: data.Count(), // This parameter specifies the number of data points that are used when performing a forecast.
trainSize: data.Count(), // This parameter specifies the total number of data points in the input time series, starting from the beginning.
horizon: 3, // Indicates the number of values to forecast; 3 indicates that the next 3 months of product units will be forecasted.
confidenceLevel: 0.75f, // Indicates the likelihood the real observed value will fall within the specified interval bounds.
confidenceLowerBoundColumn: nameof(RatioAnalysisPrediction.ConfidenceLowerBound), //This is the name of the column that will be used to store the lower interval bound for each forecasted value.
confidenceUpperBoundColumn: nameof(RatioAnalysisPrediction.ConfidenceUpperBound)); //This is the name of the column that will be used to store the upper interval bound for each forecasted value.
// Fit the forecasting model to the specified product's data series.
ITransformer costofgoodsTransformer = costofgoodsForcaster.Fit(mlContext.Data.LoadFromEnumerable(data));
ITransformer inventoryTransformer = inventoryForcaster.Fit(mlContext.Data.LoadFromEnumerable(data));
ITransformer turnoverTransformer = turnoverForcaster.Fit(mlContext.Data.LoadFromEnumerable(data));
// Create the forecast engine used for creating predictions.
TimeSeriesPredictionEngine <RatioAnalysis, RatioAnalysisPrediction> inventoryEngine = inventoryTransformer.CreateTimeSeriesEngine <RatioAnalysis, RatioAnalysisPrediction>(mlContext);
TimeSeriesPredictionEngine <RatioAnalysis, RatioAnalysisPrediction> turneroverEngine = turnoverTransformer.CreateTimeSeriesEngine <RatioAnalysis, RatioAnalysisPrediction>(mlContext);
TimeSeriesPredictionEngine <RatioAnalysis, RatioAnalysisPrediction> costofgoodEngine = costofgoodsTransformer.CreateTimeSeriesEngine <RatioAnalysis, RatioAnalysisPrediction>(mlContext);
// Get the prediction; this will include the forecasted turnover for the next 3 months since this
//the time period specified in the `horizon` parameter when the forecast estimator was originally created.
var turnoverPrediction = turneroverEngine.Predict();
var costPrediction = costofgoodEngine.Predict();
var inventoryPrediction = inventoryEngine.Predict();
var last = data.Last();
var retVal = data.ToList();
for (int i = 0; i < turnoverPrediction.Forecasted.Count(); i++)
{
retVal.Add(new RatioAnalysis
{
Date = last.Date.AddMonths(i + 1),
CostOfGoods = costPrediction.Forecasted[i],
CostOfGoodsDelta = costPrediction.ConfidenceUpperBound[i] - costPrediction.Forecasted[i],
Inventory = inventoryPrediction.Forecasted[i],
InventoryDelta = inventoryPrediction.ConfidenceUpperBound[i] - inventoryPrediction.Forecasted[i],
Turnover = turnoverPrediction.Forecasted[i],
TurnoverDelta = turnoverPrediction.ConfidenceUpperBound[i] - turnoverPrediction.Forecasted[i]
});
}
return(retVal);
}
/// <summary>
/// Build model for predicting next month's product unit sales using time series forecasting.
/// </summary>
/// <param name="mlContext">ML.NET context.</param>
/// <param name="productDataSeries">ML.NET IDataView representing the loaded product data series.</param>
/// <param name="outputModelPath">Trained model path.</param>
private static void TrainAndSaveModel(MLContext mlContext, IDataView productDataView, string outputModelPath)
{
ConsoleWriteHeader("Training product forecasting Time Series model");
var supplementedProductDataSeries = TimeSeriesDataGenerator.SupplementData(mlContext, productDataView);
var supplementedProductDataSeriesLength = supplementedProductDataSeries.Count(); // 36
var supplementedProductDataView = mlContext.Data.LoadFromEnumerable(supplementedProductDataSeries, productDataView.Schema);
// Create and add the forecast estimator to the pipeline.
IEstimator <ITransformer> forecastEstimator = mlContext.Forecasting.ForecastBySsa(
outputColumnName: nameof(ProductUnitTimeSeriesPrediction.ForecastedProductUnits),
inputColumnName: nameof(ProductData.units), // This is the column being forecasted.
windowSize: 12, // Window size is set to the time period represented in the product data cycle; our product cycle is based on 12 months, so this is set to a factor of 12, e.g. 3.
seriesLength: supplementedProductDataSeriesLength, // TODO: Need clarification on what this should be set to; assuming product series length for now.
trainSize: supplementedProductDataSeriesLength, // TODO: Need clarification on what this should be set to; assuming product series length for now.
horizon: 2, // Indicates the number of values to forecast; 2 indicates that the next 2 months of product units will be forecasted.
confidenceLevel: 0.95f, // TODO: Is this the same as prediction interval, where this indicates that we are 95% confidence that the forecasted value will fall within the interval range?
confidenceLowerBoundColumn: nameof(ProductUnitTimeSeriesPrediction.ConfidenceLowerBound), // TODO: See above comment.
confidenceUpperBoundColumn: nameof(ProductUnitTimeSeriesPrediction.ConfidenceUpperBound)); // TODO: See above comment.
// Train the forecasting model for the specified product's data series.
ITransformer forecastTransformer = forecastEstimator.Fit(supplementedProductDataView);
// Create the forecast engine used for creating predictions.
TimeSeriesPredictionEngine <ProductData, ProductUnitTimeSeriesPrediction> forecastEngine = forecastTransformer.CreateTimeSeriesEngine <ProductData, ProductUnitTimeSeriesPrediction>(mlContext);
// Save the forecasting model so that it can be loaded within an end-user app.
forecastEngine.CheckPoint(mlContext, outputModelPath);
}
// This example creates a time series (list of Data with the i-th element
// corresponding to the i-th time slot). The estimator is applied then to
// identify points where data distribution changed.
public static void Example()
{
// Create a new ML context, for ML.NET operations. It can be used for
// exception tracking and logging, as well as the source of randomness.
var ml = new MLContext();
// Generate sample series data with a change
const int Size = 16;
var data = new List <TimeSeriesData>(Size)
{
new TimeSeriesData(5),
new TimeSeriesData(5),
new TimeSeriesData(5),
new TimeSeriesData(5),
new TimeSeriesData(5),
new TimeSeriesData(5),
new TimeSeriesData(5),
new TimeSeriesData(5),
//Change point data.
new TimeSeriesData(7),
new TimeSeriesData(7),
new TimeSeriesData(7),
new TimeSeriesData(7),
new TimeSeriesData(7),
new TimeSeriesData(7),
new TimeSeriesData(7),
new TimeSeriesData(7),
};
// Convert data to IDataView.
var dataView = ml.Data.LoadFromEnumerable(data);
// Setup IidSpikeDetector arguments
string outputColumnName = nameof(ChangePointPrediction.Prediction);
string inputColumnName = nameof(TimeSeriesData.Value);
// Time Series model.
ITransformer model = ml.Transforms.DetectIidChangePoint(
outputColumnName, inputColumnName, 95, Size / 4).Fit(dataView);
// Create a time series prediction engine from the model.
var engine = model.CreateTimeSeriesEngine <TimeSeriesData,
ChangePointPrediction>(ml);
Console.WriteLine($"{outputColumnName} column obtained " +
$"post-transformation.");
Console.WriteLine("Data\tAlert\tScore\tP-Value\tMartingale value");
// Data Alert Score P-Value Martingale value
// Create non-anomalous data and check for change point.
for (int index = 0; index < 8; index++)
{
// Anomaly change point detection.
PrintPrediction(5, engine.Predict(new TimeSeriesData(5)));
}
// 5 0 5.00 0.50 0.00 <-- Time Series 1.
// 5 0 5.00 0.50 0.00
// 5 0 5.00 0.50 0.00
// 5 0 5.00 0.50 0.00
// 5 0 5.00 0.50 0.00
// 5 0 5.00 0.50 0.00
// 5 0 5.00 0.50 0.00
// 5 0 5.00 0.50 0.00
// Change point
PrintPrediction(7, engine.Predict(new TimeSeriesData(7)));
// 7 1 7.00 0.00 10298.67 <-- alert is on, predicted changepoint (and model is checkpointed).
// Checkpoint the model.
var modelPath = "temp.zip";
engine.CheckPoint(ml, modelPath);
// Reference to current time series engine because in the next step
// "engine" will point to the
// checkpointed model being loaded from disk.
var timeseries1 = engine;
// Load the model.
using (var file = File.OpenRead(modelPath))
model = ml.Model.Load(file, out DataViewSchema schema);
// Create a time series prediction engine from the checkpointed model.
engine = model.CreateTimeSeriesEngine <TimeSeriesData,
ChangePointPrediction>(ml);
for (int index = 0; index < 8; index++)
{
// Anomaly change point detection.
PrintPrediction(7, engine.Predict(new TimeSeriesData(7)));
}
// 7 0 7.00 0.13 33950.16 <-- Time Series 2 : Model loaded back from disk and prediction is made.
// 7 0 7.00 0.26 60866.34
// 7 0 7.00 0.38 78362.04
// 7 0 7.00 0.50 0.01
// 7 0 7.00 0.50 0.00
// 7 0 7.00 0.50 0.00
// 7 0 7.00 0.50 0.00
// Prediction from the original time series engine should match the
// prediction from check pointed model.
engine = timeseries1;
for (int index = 0; index < 8; index++)
{
// Anomaly change point detection.
PrintPrediction(7, engine.Predict(new TimeSeriesData(7)));
}
// 7 0 7.00 0.13 33950.16 <-- Time Series 1 and prediction is made.
// 7 0 7.00 0.26 60866.34
// 7 0 7.00 0.38 78362.04
// 7 0 7.00 0.50 0.01
// 7 0 7.00 0.50 0.00
// 7 0 7.00 0.50 0.00
// 7 0 7.00 0.50 0.00
}
private static void SaveModel(MLContext context, ITransformer model)
{
TimeSeriesPredictionEngine <ModelInput, ModelOutput> forecastEngine = model.CreateTimeSeriesEngine <ModelInput, ModelOutput>(context);
forecastEngine.CheckPoint(context, MODEL_PATH);
}
/// <summary>
/// Build model for predicting next month's product unit sales using time series forecasting.
/// </summary>
/// <param name="mlContext">ML.NET context.</param>
/// <param name="productDataSeries">ML.NET IDataView representing the loaded product data series.</param>
/// <param name="outputModelPath">Model path.</param>
private static void FitAndSaveModel(MLContext mlContext, IDataView productDataSeries, string outputModelPath)
{
ConsoleWriteHeader("Fitting product forecasting Time Series model");
const int numSeriesDataPoints = 34; //The underlying data has a total of 34 months worth of data for each product
// Create and add the forecast estimator to the pipeline.
IEstimator <ITransformer> forecastEstimator = mlContext.Forecasting.ForecastBySsa(
outputColumnName: nameof(ProductUnitTimeSeriesPrediction.ForecastedProductUnits),
inputColumnName: nameof(ProductData.units), // This is the column being forecasted.
windowSize: 12, // Window size is set to the time period represented in the product data cycle; our product cycle is based on 12 months, so this is set to a factor of 12, e.g. 3.
seriesLength: numSeriesDataPoints, // This parameter specifies the number of data points that are used when performing a forecast.
trainSize: numSeriesDataPoints, // This parameter specifies the total number of data points in the input time series, starting from the beginning.
horizon: 2, // Indicates the number of values to forecast; 2 indicates that the next 2 months of product units will be forecasted.
confidenceLevel: 0.95f, // Indicates the likelihood the real observed value will fall within the specified interval bounds.
confidenceLowerBoundColumn: nameof(ProductUnitTimeSeriesPrediction.ConfidenceLowerBound), //This is the name of the column that will be used to store the lower interval bound for each forecasted value.
confidenceUpperBoundColumn: nameof(ProductUnitTimeSeriesPrediction.ConfidenceUpperBound)); //This is the name of the column that will be used to store the upper interval bound for each forecasted value.
// Fit the forecasting model to the specified product's data series.
ITransformer forecastTransformer = forecastEstimator.Fit(productDataSeries);
// Create the forecast engine used for creating predictions.
TimeSeriesPredictionEngine <ProductData, ProductUnitTimeSeriesPrediction> forecastEngine = forecastTransformer.CreateTimeSeriesEngine <ProductData, ProductUnitTimeSeriesPrediction>(mlContext);
// Save the forecasting model so that it can be loaded within an end-user app.
forecastEngine.CheckPoint(mlContext, outputModelPath);
}
/// <summary>
/// Crea el modelo y lo guarda en el path entrado
/// </summary>
/// <param name="mlContext">ML.NET context.</param>
/// <param name="productId">Id of the product series to forecast.</param>
/// <param name="dataPath">Input data file path.</param>
private static void CreateAndSaveModelTimeSeries(MLContext mlContext, string dataPath, float id)
{
var productModelPath = $"product{id}_month_timeSeriesSSA.zip";
if (File.Exists(productModelPath))
{
File.Delete(productModelPath);
}
IDataView saleDataView = LoadData(mlContext, dataPath, id);
var singleProductDataSeries = mlContext.Data.CreateEnumerable <SaleData>(saleDataView, false).OrderBy(p => p.ano).ThenBy(p => p.mes);
SaleData lastMonthSalesData = singleProductDataSeries.Last();
Console.WriteLine("Creando el modelo: Time Series Model");
const int numSeriesDataPoints = 34;
IEstimator <ITransformer> forecastEstimator = mlContext.Forecasting.ForecastBySsa(
outputColumnName: nameof(ProductUnitTimeSeriesPrediction.ForecastedProductUnits),
inputColumnName: nameof(SaleData.unidades), // This is the column being forecasted.
windowSize: 12, // Window size is set to the time period represented in the product data cycle; our product cycle is based on 12 months, so this is set to a factor of 12, e.g. 3.
seriesLength: numSeriesDataPoints, // This parameter specifies the number of data points that are used when performing a forecast.
trainSize: numSeriesDataPoints, // This parameter specifies the total number of data points in the input time series, starting from the beginning.
horizon: 2, // Indicates the number of values to forecast; 2 indicates that the next 2 months of product units will be forecasted.
confidenceLevel: 0.95f, // Indicates the likelihood the real observed value will fall within the specified interval bounds.
confidenceLowerBoundColumn: nameof(ProductUnitTimeSeriesPrediction.ConfidenceLowerBound), //This is the name of the column that will be used to store the lower interval bound for each forecasted value.
confidenceUpperBoundColumn: nameof(ProductUnitTimeSeriesPrediction.ConfidenceUpperBound)); //This is the name of the column that will be used to store the upper interval bound for each forecasted value.
// Fit the forecasting model to the specified product's data series.
ITransformer forecastTransformer = forecastEstimator.Fit(saleDataView);
// Create the forecast engine used for creating predictions.
TimeSeriesPredictionEngine <SaleData, ProductUnitTimeSeriesPrediction> forecastEngine = forecastTransformer.CreateTimeSeriesEngine <SaleData, ProductUnitTimeSeriesPrediction>(mlContext);
// Save the forecasting model so that it can be loaded within an end-user app.
forecastEngine.CheckPoint(mlContext, productModelPath);
TestPrediction(mlContext, lastMonthSalesData, productModelPath);
}
// This example creates a time series (list of Data with the i-th element corresponding to the i-th time slot).
// The estimator is applied then to identify spiking points in the series.
public static void Example()
{
// Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging,
// as well as the source of randomness.
var ml = new MLContext();
// Generate sample series data with a spike
const int Size = 10;
var data = new List <TimeSeriesData>(Size + 1)
{
new TimeSeriesData(5),
new TimeSeriesData(5),
new TimeSeriesData(5),
new TimeSeriesData(5),
new TimeSeriesData(5),
// This is a spike.
new TimeSeriesData(10),
new TimeSeriesData(5),
new TimeSeriesData(5),
new TimeSeriesData(5),
new TimeSeriesData(5),
new TimeSeriesData(5),
};
// Convert data to IDataView.
var dataView = ml.Data.LoadFromEnumerable(data);
// Setup IidSpikeDetector arguments
string outputColumnName = nameof(IidSpikePrediction.Prediction);
string inputColumnName = nameof(TimeSeriesData.Value);
// The transformed model.
ITransformer model = ml.Transforms.DetectIidSpike(outputColumnName, inputColumnName, 95, Size).Fit(dataView);
// Create a time series prediction engine from the model.
var engine = model.CreateTimeSeriesEngine <TimeSeriesData, IidSpikePrediction>(ml);
Console.WriteLine($"{outputColumnName} column obtained post-transformation.");
Console.WriteLine("Data\tAlert\tScore\tP-Value");
// Prediction column obtained post-transformation.
// Data Alert Score P-Value
// Create non-anomalous data and check for anomaly.
for (int index = 0; index < 5; index++)
{
// Anomaly spike detection.
PrintPrediction(5, engine.Predict(new TimeSeriesData(5)));
}
// 5 0 5.00 0.50
// 5 0 5.00 0.50
// 5 0 5.00 0.50
// 5 0 5.00 0.50
// 5 0 5.00 0.50
// Spike.
PrintPrediction(10, engine.Predict(new TimeSeriesData(10)));
// 10 1 10.00 0.00 <-- alert is on, predicted spike (check-point model)
// Checkpoint the model.
var modelPath = "temp.zip";
engine.CheckPoint(ml, modelPath);
// Load the model.
using (var file = File.OpenRead(modelPath))
model = ml.Model.Load(file, out DataViewSchema schema);
for (int index = 0; index < 5; index++)
{
// Anomaly spike detection.
PrintPrediction(5, engine.Predict(new TimeSeriesData(5)));
}
// 5 0 5.00 0.26 <-- load model from disk.
// 5 0 5.00 0.26
// 5 0 5.00 0.50
// 5 0 5.00 0.50
// 5 0 5.00 0.50
}
public async Task <IActionResult> GetProductUnitDemandEstimation(string productId)
{
// Get product history for the selected product
var productHistory = await _queries.GetProductDataAsync(productId);
var productDataView = mlContext.Data.LoadFromEnumerable(productHistory);
// Create and add the forecast estimator to the pipeline.
IEstimator <ITransformer> forecastEstimator = mlContext.Forecasting.ForecastBySsa(
outputColumnName: nameof(ProductUnitTimeSeriesPrediction.ForecastedProductUnits),
inputColumnName: nameof(ProductData.units), // This is the column being forecasted.
windowSize: 12, // Window size is set to the time period represented in the product data cycle; our product cycle is based on 12 months, so this is set to a factor of 12, e.g. 3.
seriesLength: productHistory.Count(), // This parameter specifies the number of data points that are used when performing a forecast.
trainSize: productHistory.Count(), // This parameter specifies the total number of data points in the input time series, starting from the beginning.
horizon: 1, // Indicates the number of values to forecast; 1 indicates that the next month of product units will be forecasted.
confidenceLevel: 0.95f, // Indicates the likelihood the real observed value will fall within the specified interval bounds.
confidenceLowerBoundColumn: nameof(ProductUnitTimeSeriesPrediction.ConfidenceLowerBound), //This is the name of the column that will be used to store the lower interval bound for each forecasted value.
confidenceUpperBoundColumn: nameof(ProductUnitTimeSeriesPrediction.ConfidenceUpperBound)); //This is the name of the column that will be used to store the upper interval bound for each forecasted value.
// Train the forecasting model for the specified product's data series.
ITransformer forecastTransformer = forecastEstimator.Fit(productDataView);
// Create the forecast engine used for creating predictions.
TimeSeriesPredictionEngine <ProductData, ProductUnitTimeSeriesPrediction> forecastEngine = forecastTransformer.CreateTimeSeriesEngine <ProductData, ProductUnitTimeSeriesPrediction>(mlContext);
// Predict.
var nextMonthUnitDemandEstimation = forecastEngine.Predict();
return(Ok(nextMonthUnitDemandEstimation.ForecastedProductUnits.First()));
}
// This example creates a time series (list of Data with the i-th element
// corresponding to the i-th time slot). The estimator is applied then to
// identify spiking points in the series.
public static void Example()
{
// Create a new ML context, for ML.NET operations. It can be used for
// exception tracking and logging, as well as the source of randomness.
var ml = new MLContext();
// Generate sample series data with an anomaly
var data = new List <TimeSeriesData>();
for (int index = 0; index < 20; index++)
{
data.Add(new TimeSeriesData(5));
}
data.Add(new TimeSeriesData(10));
for (int index = 0; index < 5; index++)
{
data.Add(new TimeSeriesData(5));
}
// Convert data to IDataView.
var dataView = ml.Data.LoadFromEnumerable(data);
// Setup the estimator arguments
string outputColumnName = nameof(SrCnnAnomalyDetection.Prediction);
string inputColumnName = nameof(TimeSeriesData.Value);
// The transformed model.
ITransformer model = ml.Transforms.DetectAnomalyBySrCnn(
outputColumnName, inputColumnName, 16, 5, 5, 3, 8, 0.35).Fit(
dataView);
// Create a time series prediction engine from the model.
var engine = model.CreateTimeSeriesEngine <TimeSeriesData,
SrCnnAnomalyDetection>(ml);
Console.WriteLine($"{outputColumnName} column obtained post-" +
$"transformation.");
Console.WriteLine("Data\tAlert\tScore\tMag");
// Prediction column obtained post-transformation.
// Data Alert Score Mag
// Create non-anomalous data and check for anomaly.
for (int index = 0; index < 20; index++)
{
// Anomaly detection.
PrintPrediction(5, engine.Predict(new TimeSeriesData(5)));
}
//5 0 0.00 0.00
//5 0 0.00 0.00
//5 0 0.00 0.00
//5 0 0.00 0.00
//5 0 0.00 0.00
//5 0 0.00 0.00
//5 0 0.00 0.00
//5 0 0.00 0.00
//5 0 0.00 0.00
//5 0 0.00 0.00
//5 0 0.00 0.00
//5 0 0.00 0.00
//5 0 0.00 0.00
//5 0 0.00 0.00
//5 0 0.00 0.00
//5 0 0.03 0.18
//5 0 0.03 0.18
//5 0 0.03 0.18
//5 0 0.03 0.18
//5 0 0.03 0.18
// Anomaly.
PrintPrediction(10, engine.Predict(new TimeSeriesData(10)));
//10 1 0.47 0.93 <-- alert is on, predicted anomaly
// Checkpoint the model.
var modelPath = "temp.zip";
engine.CheckPoint(ml, modelPath);
// Load the model.
using (var file = File.OpenRead(modelPath))
model = ml.Model.Load(file, out DataViewSchema schema);
for (int index = 0; index < 5; index++)
{
// Anomaly detection.
PrintPrediction(5, engine.Predict(new TimeSeriesData(5)));
}
//5 0 0.31 0.50
//5 0 0.05 0.30
//5 0 0.01 0.23
//5 0 0.00 0.21
//5 0 0.01 0.25
}
static void Main(string[] args)
{
var mlContext = new MLContext();
int c = 0;
float predictedPrice = 0;
while (c < 10)
{
IEnumerable <BTC_TimeSeries> data = GetTrainingData(mlContext);
IDataView trainingData = mlContext.Data.LoadFromEnumerable <BTC_TimeSeries>(data);
var offset = TimeSpan.FromSeconds(data.Last().TimeStamp);
var epoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc);
DateTime lastTime = epoch.Add(offset).ToLocalTime();
int seriesLength = data.Count();
var estimator = mlContext.Forecasting.ForecastBySsa(outputColumnName: nameof(PredictedSeries.ForecastedPrice),
inputColumnName: nameof(BTC_TimeSeries.Price),
windowSize: 360,
seriesLength: seriesLength,
trainSize: seriesLength,
horizon: 5,
confidenceLevel: 0.95f,
confidenceLowerBoundColumn: nameof(PredictedSeries.ConfidenceLowerBound),
confidenceUpperBoundColumn: nameof(PredictedSeries.ConfidenceUpperBound)
);
Console.WriteLine("Training model");
ITransformer forecastTransformer = estimator.Fit(trainingData);
TimeSeriesPredictionEngine <BTC_TimeSeries, PredictedSeries> forecastEngine = forecastTransformer.CreateTimeSeriesEngine <BTC_TimeSeries, PredictedSeries>(mlContext);
PredictedSeries predictions = forecastEngine.Predict();
Console.WriteLine("last read time {0}: last read price {1}", lastTime, data.Last().Price);
for (int i = 0; i < predictions.ForecastedPrice.Count(); i++)
{
lastTime = lastTime.AddMinutes(15);
Console.WriteLine("{0} price: {1}, low: {2}, high: {3}", lastTime, predictions.ForecastedPrice[i].ToString(), predictions.ConfidenceLowerBound[i].ToString(), predictions.ConfidenceUpperBound[i].ToString());
}
float delta = predictedPrice - data.Last().Price;
Console.WriteLine("Delta, {0}", delta);
predictedPrice = predictions.ForecastedPrice[0];
forecastEngine.CheckPoint(mlContext, "forecastmodel.zip");
System.Threading.Thread.Sleep(300000);
c++;
}
Console.Read();
}
