public DataStreamUnit[] Execute(DataStreamUnit[] input)
{
DataStreamUnit[] retVal = _Execute(input);
if (_nextFilter != null)
{
retVal = _nextFilter.Execute(retVal);
}
return retVal;
}
private DataStreamUnit ConvertLineToUnit(string line)
{
var resultUnit = new DataStreamUnit();
var values = line.Split(';').Where(x => !String.IsNullOrWhiteSpace(x));
var paramIndex = 0;
resultUnit.Values = new System.Collections.Concurrent.ConcurrentDictionary<string, object>(values.ToDictionary(x => headers[paramIndex++].Trim(), x => (object)x));
resultUnit.TimeStamp = DateTime.Now;
return resultUnit;
}
public DataStreamUnit[] Dequeue(string measurementId, string queueId)
{
var timeStamp = DateTime.Now;
var values = new ConcurrentDictionary<string, object>();
values["Value"] = 50000.0;
values["TimeStamp"] = timeStamp;
var unit = new DataStreamUnit() {TimeStamp = timeStamp, Values = values};
return new DataStreamUnit[] { unit };
}
protected override DataStreamUnit[] _Execute(DataStreamUnit[] input)
{
var newValue = input[0];
var sw = new Stopwatch();
sw.Start();
newValue.Values["Stopwatch"] = sw;
var dateTimeStr = newValue.Values["TimeStamp"].ToString();
DateTime dateTime;
DateTime.TryParse(dateTimeStr, out dateTime);
//нужно, чтобы симулировать приход значений раз в 15 минут
//то есть чтобы онлайн ждал, пока оффлайн дотренирует НС до текущей даты
while(exchange["TrainingInProcess"] as bool? == true && dateTime >= (exchange["LastDate"] as DateTime?))
{
Thread.Sleep(100);
}
if (dateTime > waitUntil && !trainingFinished)
{
while (parameters["TrainingFinished"] == null)
{
Thread.Sleep(100);
}
trainingFinished = true;
}
var realValue = double.Parse(newValue.Values["Value"].ToString());
this.timeSeries.timeseries.Add(realValue);
this.timeSeries.timestamps.Add(dateTime);
//this.TimeSeries.outlierLabel.Add(int.Parse(newValue.Values["Result"].ToString()));
this.timeSeries.temperature.Add(0);
this.timeSeries.status.Add(0);
this.timeSeries.cluster.Add(0);
var forecastResult = model.Forecast(timeSeries, 0, settings);
if(forecastResult.HasValue) forecastResult = Math.Max(forecastResult.Value, 0.0);
newValue.Values["PredictedValue"] = forecastResult;
newValue.Values["EnergyLag"] = waitUntil;
if (model.NetworkSet.Count > globalCounter % NN_NUMBER)
{
parameters.Values["NeuralNetwork"] = model.NetworkSet[globalCounter % NN_NUMBER];
}
globalCounter++;
return input;
}
protected override DataStreamUnit[] _Execute(DataStreamUnit[] input)
{
var newValue = input[0];
var sw = new Stopwatch();
sw.Start();
var currentTimeSeriesNumber = counter % PACKAGE_SIZE;
var dateTimeStr = newValue.Values["TimeStamp"].ToString();
DateTime dateTime;
DateTime.TryParse(dateTimeStr, out dateTime);
var currentTimeSeries = this.timeSeriesEnsemble[currentTimeSeriesNumber];
currentTimeSeries.timeseries.Add(double.Parse(newValue.Values["Value"].ToString()));
currentTimeSeries.timestamps.Add(dateTime);
currentTimeSeries.temperature.Add(0);
currentTimeSeries.status.Add(0);
currentTimeSeries.cluster.Add(0);
counter++;
if (counter == PACKAGE_SIZE)
{
exchange["TrainingInProcess"] = true;
exchange["LastDate"] = dateTime;
var model = parameters["model"] as MultipleNeuralNetworksModel;
model.Train(timeSeriesEnsemble, settings, true);
counter = 0;
this.time[globalCounter + "Training"] = 1;
exchange["TrainingInProcess"] = false;
}
else
{
this.time[globalCounter + "Training"] = 0;
}
if (dateTime > waitUntil && (parameters["TrainingFinished"] == null || (bool)parameters["TrainingFinished"] == false))
{
parameters["TrainingFinished"] = true;
}
sw.Stop();
this.time[globalCounter + "Value"] = (int)sw.ElapsedTicks;
globalCounter++;
return input;
}
public DataStreamUnit[] Execute(DataStreamUnit[] source)
{
IFilter root = null;
IFilter previous = null;
foreach (IFilter phrase in filters)
{
if (root == null)
{
root = phrase;
}
else
{
previous.Register(phrase);
}
previous = phrase;
}
var result = root == null ? source : root.Execute(source);
return result;
}
protected override DataStreamUnit[] _Execute(DataStreamUnit[] source)
{
try
{
if (!_initialized)
{
_forecast = new Forecast() { ForecastModelId = this.ForecastModelId };
_measurement = repository.FirstOrDefault(x => x.MeasurementId == this.MeasurementId);
if (_measurement == null)
{
_measurement = new Measurement {MeasurementId = this.MeasurementId};
}
_measurement.Forecasts.Add(_forecast);
_initialized = true;
}
var data = source.ToList();
var result = new ForecastResult();
if (data[0].Values.ContainsKey("MAPE"))
{
result.Error = data[0].Values["MAPE"] as double?;
}
result.TimeStamp = (DateTime)data[0].Values["TimeStamp"];
result.RealValue = data[0].Values["Value"] as double?;
result.PredictedValue = data[0].Values["PredictedValue"] as double?;
_forecast.Error = result.Error;
_forecast.Results.Add(result);
repository.Update(_measurement);
}
catch (Exception e)
{
Logging.Info(e.Message);
}
return source;
}
protected override DataStreamUnit[] _Execute(DataStreamUnit[] input)
{
var newValue = input[0];
var predictedValue = 0.0;
if (newValue.Values.ContainsKey("Value"))
{
var actualValue = double.Parse(newValue.Values["Value"].ToString());
counter++;
sum += actualValue;
predictedValue = sum / counter;
if (actualValue > 0)
{
errorSum += Math.Abs((actualValue - predictedValue)/actualValue);
newValue.Values["MAPE"] = errorSum / counter;
}
}
newValue.Values["PredictedValue"] = predictedValue;
return new DataStreamUnit[] { newValue };
}
private DataStreamUnit[] ConvertToUnits(byte[] source)
{
string message = System.Text.Encoding.UTF8.GetString(source);
JObject unit = JObject.Parse(message);
DataStreamUnit dataStreamUnit = new DataStreamUnit();
var valuesDict = new Dictionary<string, object>();
foreach (var child in unit.Children())
{
var prop = (child as JProperty);
dataStreamUnit.Values[prop.Name] = prop.Value.ToString();
}
var unitArray = new DataStreamUnit[] { dataStreamUnit };
return unitArray;
}
protected abstract DataStreamUnit[] _Execute(DataStreamUnit[] input);
protected override DataStreamUnit[] _Execute(DataStreamUnit[] input)
{
var value = input[0];
value.Values["Number"] = counter;
var waitUntil = DateTime.Parse(value.Values["EnergyLag"].ToString());
var dateTime = DateTime.Parse(value.Values["TimeStamp"].ToString());
if (dateTime > waitUntil)
{
List<double> absoluteErrors = new List<double>();
neuralNetwork = parameters["NeuralNetwork"] as MultiLayersNN;
if (neuralNetwork != null)
{
absoluteErrors = neuralNetwork.absoluteErrors;
}
var realValue = Double.Parse(value.Values["Value"].ToString());
var predictedValue = value.Values["PredictedValue"] != null ? Double.Parse(value.Values["PredictedValue"].ToString()) : (double?)null;
if (predictedValue != null)
{
var absoluteError = Math.Abs(predictedValue.Value - realValue);
if (realValue != 0)
{
counter++;
mapeSum += Math.Abs(absoluteError/realValue);
}
testCounter++;
value.Values["MAPE"] = mapeSum/testCounter;
}
D = (int)Math.Round(absoluteErrors.Count / 1.0, MidpointRounding.AwayFromZero);
value.Values["D"] = D;
double errorSum = 0;
for (int i = absoluteErrors.Count - D; i < absoluteErrors.Count; i++)
{
errorSum += absoluteErrors[i];
}
var mu = errorSum / D;
value.Values["mu"] = mu;
double sum = 0;
for (int i = makeNonNegative(absoluteErrors.Count - D) + 1; i < absoluteErrors.Count; i++)
{
sum += Math.Pow(absoluteErrors[i] - mu, 2);
}
var resultError = Math.Sqrt(sum) / D;
value.Values["Error"] = resultError;
}
else
{
value.Values["MAPE"] = null;
}
return input;
}
