public void TestSsaSpikeEstimator() { int confidence = 95; int pValueHistorySize = 10; int seasonalitySize = 10; int numberOfSeasonsInTraining = 5; int maxTrainingSize = numberOfSeasonsInTraining * seasonalitySize; List <Data> data = new List <Data>(); var dataView = ML.Data.LoadFromEnumerable(data); for (int j = 0; j < numberOfSeasonsInTraining; j++) { for (int i = 0; i < seasonalitySize; i++) { data.Add(new Data(i)); } } for (int i = 0; i < pValueHistorySize; i++) { data.Add(new Data(i * 100)); } var pipe = new SsaSpikeEstimator(Env, "Change", confidence, pValueHistorySize, maxTrainingSize, seasonalitySize, "Value"); var xyData = new List <TestDataXY> { new TestDataXY() { A = new float[InputSize] } }; var stringData = new List <TestDataDifferentType> { new TestDataDifferentType() { data_0 = new string[InputSize] } }; var invalidDataWrongNames = ML.Data.LoadFromEnumerable(xyData); var invalidDataWrongTypes = ML.Data.LoadFromEnumerable(stringData); TestEstimatorCore(pipe, dataView, invalidInput: invalidDataWrongTypes); TestEstimatorCore(pipe, dataView, invalidInput: invalidDataWrongNames); Done(); }
void TestSsaSpikeEstimator() { int Confidence = 95; int PValueHistorySize = 10; int SeasonalitySize = 10; int NumberOfSeasonsInTraining = 5; int MaxTrainingSize = NumberOfSeasonsInTraining * SeasonalitySize; List <Data> data = new List <Data>(); var dataView = ML.Data.ReadFromEnumerable(data); for (int j = 0; j < NumberOfSeasonsInTraining; j++) { for (int i = 0; i < SeasonalitySize; i++) { data.Add(new Data(i)); } } for (int i = 0; i < PValueHistorySize; i++) { data.Add(new Data(i * 100)); } var pipe = new SsaSpikeEstimator(Env, "Change", Confidence, PValueHistorySize, MaxTrainingSize, SeasonalitySize, "Value"); var xyData = new List <TestDataXY> { new TestDataXY() { A = new float[inputSize] } }; var stringData = new List <TestDataDifferntType> { new TestDataDifferntType() { data_0 = new string[inputSize] } }; var invalidDataWrongNames = ML.Data.ReadFromEnumerable(xyData); var invalidDataWrongTypes = ML.Data.ReadFromEnumerable(stringData); TestEstimatorCore(pipe, dataView, invalidInput: invalidDataWrongTypes); TestEstimatorCore(pipe, dataView, invalidInput: invalidDataWrongNames); Done(); }
// This example creates a time series (list of Data with the i-th element corresponding to the i-th time slot). // SsaSpikeDetector is applied then to identify spiking points in the series. // SsaSpikeDetector differs from IidSpikeDetector in that it can account for temporal seasonality // in the data. public static void SsaSpikeDetectorTransform() { // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, // as well as the source of randomness. var ml = new MLContext(); // Generate sample series data with a recurring pattern and a spike within the pattern const int SeasonalitySize = 5; const int TrainingSeasons = 3; const int TrainingSize = SeasonalitySize * TrainingSeasons; var data = new List <SsaSpikeData>(); for (int i = 0; i < TrainingSeasons; i++) { for (int j = 0; j < SeasonalitySize; j++) { data.Add(new SsaSpikeData(j)); } } // This is a spike data.Add(new SsaSpikeData(100)); for (int i = 0; i < SeasonalitySize; i++) { data.Add(new SsaSpikeData(i)); } // Convert data to IDataView. var dataView = ml.Data.ReadFromEnumerable(data); // Setup IidSpikeDetector arguments var inputColumnName = nameof(SsaSpikeData.Value); var outputColumnName = nameof(SsaSpikePrediction.Prediction); var args = new SsaSpikeDetector.Arguments() { Source = inputColumnName, Name = outputColumnName, Confidence = 95, // The confidence for spike detection in the range [0, 100] PvalueHistoryLength = 8, // The size of the sliding window for computing the p-value; shorter windows are more sensitive to spikes. TrainingWindowSize = TrainingSize, // The number of points from the beginning of the sequence used for training. SeasonalWindowSize = SeasonalitySize + 1 // An upper bound on the largest relevant seasonality in the input time series." }; // The transformed data. var transformedData = new SsaSpikeEstimator(ml, args).Fit(dataView).Transform(dataView); // Getting the data of the newly created column as an IEnumerable of SsaSpikePrediction. var predictionColumn = ml.CreateEnumerable <SsaSpikePrediction>(transformedData, reuseRowObject: false); Console.WriteLine($"{outputColumnName} column obtained post-transformation."); Console.WriteLine("Data\tAlert\tScore\tP-Value"); int k = 0; foreach (var prediction in predictionColumn) { Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", data[k++].Value, prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2]); } Console.WriteLine(""); // Prediction column obtained post-transformation. // Data Alert Score P-Value // 0 0 - 2.53 0.50 // 1 0 - 0.01 0.01 // 2 0 0.76 0.14 // 3 0 0.69 0.28 // 4 0 1.44 0.18 // 0 0 - 1.84 0.17 // 1 0 0.22 0.44 // 2 0 0.20 0.45 // 3 0 0.16 0.47 // 4 0 1.33 0.18 // 0 0 - 1.79 0.07 // 1 0 0.16 0.50 // 2 0 0.09 0.50 // 3 0 0.08 0.45 // 4 0 1.31 0.12 // 100 1 98.21 0.00 <-- alert is on, predicted spike // 0 0 - 13.83 0.29 // 1 0 - 1.74 0.44 // 2 0 - 0.47 0.46 // 3 0 - 16.50 0.29 // 4 0 - 29.82 0.21 }
// This example shows spike detection as above, but demonstrates how to train a model // that can run predictions on streaming data, and how to persist the trained model and then re-load it. public static void SsaSpikeDetectorPrediction() { // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, // as well as the source of randomness. var ml = new MLContext(); // Generate sample series data with a recurring pattern const int SeasonalitySize = 5; const int TrainingSeasons = 3; const int TrainingSize = SeasonalitySize * TrainingSeasons; var data = new List <SsaSpikeData>(); for (int i = 0; i < TrainingSeasons; i++) { for (int j = 0; j < SeasonalitySize; j++) { data.Add(new SsaSpikeData(j)); } } // Convert data to IDataView. var dataView = ml.Data.ReadFromEnumerable(data); // Setup IidSpikeDetector arguments var inputColumnName = nameof(SsaSpikeData.Value); var outputColumnName = nameof(SsaSpikePrediction.Prediction); var args = new SsaSpikeDetector.Arguments() { Source = inputColumnName, Name = outputColumnName, Confidence = 95, // The confidence for spike detection in the range [0, 100] PvalueHistoryLength = 8, // The size of the sliding window for computing the p-value; shorter windows are more sensitive to spikes. TrainingWindowSize = TrainingSize, // The number of points from the beginning of the sequence used for training. SeasonalWindowSize = SeasonalitySize + 1 // An upper bound on the largest relevant seasonality in the input time series." }; // Train the change point detector. ITransformer model = new SsaSpikeEstimator(ml, args).Fit(dataView); // Create a prediction engine from the model for feeding new data. var engine = model.CreateTimeSeriesPredictionFunction <SsaSpikeData, SsaSpikePrediction>(ml); // Start streaming new data points with no change point to the prediction engine. Console.WriteLine($"Output from spike predictions on new data:"); Console.WriteLine("Data\tAlert\tScore\tP-Value"); SsaSpikePrediction prediction = null; for (int j = 0; j < 2; j++) { for (int i = 0; i < 5; i++) { var value = i; prediction = engine.Predict(new SsaSpikeData(value)); Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", value, prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2]); } } // Now send a data point that reflects a spike. var newValue = 100; prediction = engine.Predict(new SsaSpikeData(newValue)); Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", newValue, prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2]); // Now we demonstrate saving and loading the model. // Save the model that exists within the prediction engine. // The engine has been updating this model with every new data point. var modelPath = "model.zip"; engine.CheckPoint(ml, modelPath); // Load the model. using (var file = File.OpenRead(modelPath)) model = TransformerChain.LoadFrom(ml, file); // We must create a new prediction engine from the persisted model. engine = model.CreateTimeSeriesPredictionFunction <SsaSpikeData, SsaSpikePrediction>(ml); // Run predictions on the loaded model. for (int i = 0; i < 5; i++) { var value = i; prediction = engine.Predict(new SsaSpikeData(value)); Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", value, prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2]); } // Output from spike predictions on new data: // Data Alert Score P-Value // 0 0 - 1.01 0.50 // 1 0 - 0.24 0.22 // 2 0 - 0.31 0.30 // 3 0 0.44 0.01 // 4 0 2.16 0.00 // 0 0 - 0.78 0.27 // 1 0 - 0.80 0.30 // 2 0 - 0.84 0.31 // 3 0 0.33 0.31 // 4 0 2.21 0.07 // 100 1 86.17 0.00 <-- alert is on, predicted spike // 0 0 - 2.74 0.40 <-- saved to disk, re-loaded, and running new predictions // 1 0 - 1.47 0.42 // 2 0 - 17.50 0.24 // 3 0 - 30.82 0.16 // 4 0 - 23.24 0.28 }
// This example creates a time series (list of Data with the i-th element corresponding to the i-th time slot). // SsaSpikeDetector is applied then to identify spiking points in the series. public static void SsaSpikeDetectorTransform() { // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, // as well as the source of randomness. var ml = new MLContext(); // Generate sample series data with a spike const int size = 16; var data = new List <Data>(size); for (int i = 0; i < size / 2; i++) { data.Add(new Data(5)); } // This is a spike data.Add(new Data(10)); for (int i = 0; i < size / 2; i++) { data.Add(new Data(5)); } // Convert data to IDataView. var dataView = ml.CreateStreamingDataView(data); // Setup IidSpikeDetector arguments string outputColumnName = "Prediction"; string inputColumnName = "Value"; var args = new SsaSpikeDetector.Arguments() { Source = inputColumnName, Name = outputColumnName, Confidence = 95, // The confidence for spike detection in the range [0, 100] PvalueHistoryLength = size / 4, // The size of the sliding window for computing the p-value TrainingWindowSize = size / 2, // The number of points from the beginning of the sequence used for training. SeasonalWindowSize = size / 8, // An upper bound on the largest relevant seasonality in the input time - series." }; // The transformed data. var transformedData = new SsaSpikeEstimator(ml, args).Fit(dataView).Transform(dataView); // Getting the data of the newly created column as an IEnumerable of SpikePrediction. var predictionColumn = transformedData.AsEnumerable <SpikePrediction>(ml, reuseRowObject: false); Console.WriteLine($"{outputColumnName} column obtained post-transformation."); Console.WriteLine("Alert\tScore\tP-Value"); foreach (var prediction in predictionColumn) { Console.WriteLine("{0}\t{1:0.00}\t{2:0.00}", prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2]); } Console.WriteLine(""); // Prediction column obtained post-transformation. // Alert Score P-Value // 0 0.00 0.50 // 0 0.00 0.50 // 0 0.00 0.50 // 0 0.00 0.50 // 0 0.00 0.50 // 0 0.00 0.50 // 0 0.00 0.50 // 0 0.00 0.50 // 1 5.00 0.00 <-- alert is on, predicted spike // 0 -2.50 0.09 // 0 -2.50 0.22 // 0 0.00 0.47 // 0 0.00 0.47 // 0 0.00 0.26 // 0 0.00 0.38 // 0 0.00 0.50 // 0 0.00 0.50 }
static void Main(string[] args) { MLContext mlContext = new MLContext(); IDataView dataMeterFileRaw = mlContext.Data.LoadFromTextFile <PowerMeterRecord>("power-export_min.csv", separatorChar: ',', hasHeader: true); string outputColumn = nameof(PowerMeterSpikePrediction.MeterPrediction); string inputColumn = nameof(PowerMeterRecord.ConsumptionDiffNormalized); SsaSpikeEstimator ssaSpikeEstimator = mlContext.Transforms.DetectSpikeBySsa(outputColumn, inputColumn, 95, 30, 70, 30); SsaSpikeDetector detector = ssaSpikeEstimator.Fit(dataMeterFileRaw); SrCnnAnomalyEstimator ssrAnomalyEstimator = mlContext.Transforms.DetectAnomalyBySrCnn(outputColumn, inputColumn, 30); SrCnnAnomalyDetector srCnnAnomalyDetector = ssrAnomalyEstimator.Fit(dataMeterFileRaw); mlContext.Model.Save(detector, dataMeterFileRaw.Schema, "powermeterspikemodel.zip"); Console.WriteLine("Saved"); ITransformer trainedModelFromDisk = mlContext.Model.Load("powermeterspikemodel.zip", out var dataLoader); IDataView estimateSpikeData = trainedModelFromDisk.Transform(dataMeterFileRaw); IEnumerable <PowerMeterSpikePrediction> powerMeterSpikePredictions = mlContext.Data.CreateEnumerable <PowerMeterSpikePrediction>(estimateSpikeData, false); IDataView srCnnAnomaly = srCnnAnomalyDetector.Transform(dataMeterFileRaw); IEnumerable <PowerMeterSpikePrediction> powerMeterSrCnnAnomalyPredictions = mlContext.Data.CreateEnumerable <PowerMeterSpikePrediction>(srCnnAnomaly, false); var normalizedConsumptionDiff = dataMeterFileRaw.GetColumn <float>(nameof(PowerMeterRecord.ConsumptionDiffNormalized)).ToArray(); var time = dataMeterFileRaw.GetColumn <DateTime>(nameof(PowerMeterRecord.ConsumptionTime)).ToArray(); Console.WriteLine("======Displaying anomalies in the Power meter data========="); Console.WriteLine("Date \tReadingDiff\tAlert\tScore\tP-Value"); StreamWriter textWriterSpikePredictions = new StreamWriter("SpikePredictions.csv"); textWriterSpikePredictions.WriteLine("Date \tReadingDiff\tAlert\tScore\tP-Value"); int i = 0; foreach (var spike in powerMeterSpikePredictions) { if (spike.MeterPrediction[0] == 1) { Console.BackgroundColor = ConsoleColor.DarkYellow; Console.ForegroundColor = ConsoleColor.Black; } Console.WriteLine("{0}\t{1:0.0000}\t{2:0.00}\t{3:0.00}\t{4:0.00}", time[i], normalizedConsumptionDiff[i], spike.MeterPrediction[0], spike.MeterPrediction[1], spike.MeterPrediction[2]); textWriterSpikePredictions.WriteLine("{0}\t{1:0.0000}\t{2:0.00}\t{3:0.00}\t{4:0.00}", time[i], normalizedConsumptionDiff[i], spike.MeterPrediction[0], spike.MeterPrediction[1], spike.MeterPrediction[2]); Console.ResetColor(); i++; } Console.WriteLine("======Displaying anomalies in the Power meter data========="); Console.WriteLine("Date \tReadingDiff\tSrCNNAlert\tScore\tP-Value"); StreamWriter textWriterSrCnnAnomalyPredictions = new StreamWriter("SrCnnAnomalyPredictionsPredictions.csv"); textWriterSrCnnAnomalyPredictions.WriteLine("Date \tReadingDiff\tSrCNNAlert\tScore\tP-Value"); i = 0; foreach (var spike in powerMeterSrCnnAnomalyPredictions) { if (spike.MeterPrediction[0] == 1) { Console.BackgroundColor = ConsoleColor.DarkYellow; Console.ForegroundColor = ConsoleColor.Black; } Console.WriteLine("{0}\t{1:0.0000}\t{2:0.00}\t{3:0.00}\t{4:0.00}", time[i], normalizedConsumptionDiff[i], spike.MeterPrediction[0], spike.MeterPrediction[1], spike.MeterPrediction[2]); textWriterSrCnnAnomalyPredictions.WriteLine("{0}\t{1:0.0000}\t{2:0.00}\t{3:0.00}\t{4:0.00}", time[i], normalizedConsumptionDiff[i], spike.MeterPrediction[0], spike.MeterPrediction[1], spike.MeterPrediction[2]); Console.ResetColor(); i++; } textWriterSpikePredictions.Flush(); textWriterSrCnnAnomalyPredictions.Flush(); }