// This example creates a time series (list of Data with the i-th element corresponding to the i-th time slot).
// IidSpikeDetector is applied then to identify spiking points in the series.
public static IEnumerable <IidSpikePrediction> IidSpikeDetectorTransform()
{
// Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging,
// as well as the source of randomness.
var ml = new MLContext();
int Size = 10;
// Generate sample series data with a spike
//var data = GenerateFakeDataSet();
var data = GenerateDirtySinusoidal();
//var data = GenerateInitialExample();
// Convert data to IDataView.
var dataView = ml.CreateStreamingDataView(data);
// Setup IidSpikeDetector arguments
string outputColumnName = nameof(IidSpikePrediction.Prediction);
string inputColumnName = nameof(IidSpikeData.Value);
var args = new IidSpikeDetector.Arguments()
{
Source = inputColumnName,
Name = outputColumnName,
Confidence = 95, // The confidence for spike detection in the range [0, 100]
PvalueHistoryLength = Size / 4 // The size of the sliding window for computing the p-value; shorter windows are more sensitive to spikes.
};
// The transformed data.
var transformedData = new IidSpikeEstimator(ml, args).Fit(dataView).Transform(dataView);
// Getting the data of the newly created column as an IEnumerable of IidSpikePrediction.
var predictionColumn = transformedData.AsEnumerable <IidSpikePrediction>(ml, reuseRowObject: false);
Console.WriteLine($"{outputColumnName} column obtained post-transformation.");
Console.WriteLine("Alert\tScore\tP-Value");
foreach (var prediction in predictionColumn)
{
Console.WriteLine("{0}\t{1:0.00}\t{2:0.00}", prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2]);
}
Console.WriteLine("");
// Prediction column obtained post-transformation.
// Alert Score P-Value
// 0 5.00 0.50
// 0 5.00 0.50
// 0 5.00 0.50
// 0 5.00 0.50
// 0 5.00 0.50
// 1 10.00 0.00 <-- alert is on, predicted spike
// 0 5.00 0.26
// 0 5.00 0.26
// 0 5.00 0.50
// 0 5.00 0.50
// 0 5.00 0.50
return(predictionColumn);
}
public void TestIidSpikeEstimator()
{
int confidence = 95;
int pValueHistorySize = 10;
List <Data> data = new List <Data>();
var dataView = ML.Data.LoadFromEnumerable(data);
for (int i = 0; i < pValueHistorySize; i++)
{
data.Add(new Data(i * 100));
}
var pipe = new IidSpikeEstimator(Env,
"Change", confidence, pValueHistorySize, "Value");
var xyData = new List <TestDataXY> {
new TestDataXY()
{
A = new float[InputSize]
}
};
var stringData = new List <TestDataDifferentType> {
new TestDataDifferentType()
{
data_0 = new string[InputSize]
}
};
var invalidDataWrongNames = ML.Data.LoadFromEnumerable(xyData);
var invalidDataWrongTypes = ML.Data.LoadFromEnumerable(stringData);
TestEstimatorCore(pipe, dataView, invalidInput: invalidDataWrongTypes);
TestEstimatorCore(pipe, dataView, invalidInput: invalidDataWrongNames);
Done();
}
void TestIidSpikeEstimator()
{
int Confidence = 95;
int PValueHistorySize = 10;
List <Data> data = new List <Data>();
var dataView = Env.CreateStreamingDataView(data);
for (int i = 0; i < PValueHistorySize; i++)
{
data.Add(new Data(i * 100));
}
var pipe = new IidSpikeEstimator(Env,
"Value", "Change", Confidence, PValueHistorySize);
var xyData = new List <TestDataXY> {
new TestDataXY()
{
A = new float[inputSize]
}
};
var stringData = new List <TestDataDifferntType> {
new TestDataDifferntType()
{
data_0 = new string[inputSize]
}
};
var invalidDataWrongNames = ComponentCreation.CreateDataView(Env, xyData);
var invalidDataWrongTypes = ComponentCreation.CreateDataView(Env, stringData);
TestEstimatorCore(pipe, dataView, invalidInput: invalidDataWrongTypes);
TestEstimatorCore(pipe, dataView, invalidInput: invalidDataWrongNames);
Done();
}
public static void IidSpikeDetectorPrediction()
{
// Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging,
// as well as the source of randomness.
var ml = new MLContext();
// Generate sample series data with a spike
const int Size = 10;
var data = new List <IidSpikeData>(Size);
for (int i = 0; i < Size / 2; i++)
{
data.Add(new IidSpikeData(5));
}
// This is a spike
data.Add(new IidSpikeData(10));
for (int i = 0; i < Size / 2; i++)
{
data.Add(new IidSpikeData(5));
}
// Convert data to IDataView.
var dataView = ml.CreateStreamingDataView(data);
// Setup IidSpikeDetector arguments
string outputColumnName = nameof(IidSpikePrediction.Prediction);
string inputColumnName = nameof(IidSpikeData.Value);
var args = new IidSpikeDetector.Arguments()
{
Source = inputColumnName,
Name = outputColumnName,
Confidence = 95, // The confidence for spike detection in the range [0, 100]
PvalueHistoryLength = Size / 4 // The size of the sliding window for computing the p-value; shorter windows are more sensitive to spikes.
};
// The transformed model.
ITransformer model = new IidSpikeEstimator(ml, args).Fit(dataView);
// Create a time series prediction engine from the model.
var engine = model.CreateTimeSeriesPredictionFunction <IidSpikeData, IidSpikePrediction>(ml);
for (int index = 0; index < 5; index++)
{
// Anomaly spike detection.
var prediction = engine.Predict(new IidSpikeData(5));
Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", 5, prediction.Prediction[0],
prediction.Prediction[1], prediction.Prediction[2]);
}
// Spike.
var spikePrediction = engine.Predict(new IidSpikeData(10));
Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", 10, spikePrediction.Prediction[0],
spikePrediction.Prediction[1], spikePrediction.Prediction[2]);
// Checkpoint the model.
var modelPath = "temp.zip";
engine.CheckPoint(ml, modelPath);
// Load the model.
using (var file = File.OpenRead(modelPath))
model = TransformerChain.LoadFrom(ml, file);
for (int index = 0; index < 5; index++)
{
// Anomaly spike detection.
var prediction = engine.Predict(new IidSpikeData(5));
Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", 5, prediction.Prediction[0],
prediction.Prediction[1], prediction.Prediction[2]);
}
// Data Alert Score P-Value
// 5 0 5.00 0.50
// 5 0 5.00 0.50
// 5 0 5.00 0.50
// 5 0 5.00 0.50
// 5 0 5.00 0.50
// 10 1 10.00 0.00 <-- alert is on, predicted spike (check-point model)
// 5 0 5.00 0.26 <-- load model from disk.
// 5 0 5.00 0.26
// 5 0 5.00 0.50
// 5 0 5.00 0.50
// 5 0 5.00 0.50
}
// This example creates a time series (list of Data with the i-th element corresponding to the i-th time slot).
// IidSpikeDetector is applied then to identify spiking points in the series.
public static void IidSpikeDetectorTransform()
{
// Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging,
// as well as the source of randomness.
var ml = new MLContext();
// Generate sample series data with a spike
const int size = 10;
var data = new List <Data>(size);
for (int i = 0; i < size / 2; i++)
{
data.Add(new Data(5));
}
// This is a spike
data.Add(new Data(10));
for (int i = 0; i < size / 2; i++)
{
data.Add(new Data(5));
}
// Convert data to IDataView.
var dataView = ml.CreateStreamingDataView(data);
// Setup IidSpikeDetector arguments
string outputColumnName = "Prediction";
string inputColumnName = "Value";
var args = new IidSpikeDetector.Arguments()
{
Source = inputColumnName,
Name = outputColumnName,
Confidence = 95, // The confidence for spike detection in the range [0, 100]
PvalueHistoryLength = size / 4 // The size of the sliding window for computing the p-value
};
// The transformed data.
var transformedData = new IidSpikeEstimator(ml, args).Fit(dataView).Transform(dataView);
// Getting the data of the newly created column as an IEnumerable of SpikePrediction.
var predictionColumn = transformedData.AsEnumerable <SpikePrediction>(ml, reuseRowObject: false);
Console.WriteLine($"{outputColumnName} column obtained post-transformation.");
Console.WriteLine("Alert\tScore\tP-Value");
foreach (var prediction in predictionColumn)
{
Console.WriteLine("{0}\t{1:0.00}\t{2:0.00}", prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2]);
}
Console.WriteLine("");
// Prediction column obtained post-transformation.
// Alert Score P-Value
// 0 5.00 0.50
// 0 5.00 0.50
// 0 5.00 0.50
// 0 5.00 0.50
// 0 5.00 0.50
// 1 10.00 0.00 <-- alert is on, predicted spike
// 0 5.00 0.26
// 0 5.00 0.26
// 0 5.00 0.50
// 0 5.00 0.50
// 0 5.00 0.50
}
