public void AddSink(IDataSink dataSink)
{
if (!DataSinks.Contains(dataSink))
{
DataSinks.Add(dataSink);
}
}
public Pipeline(IDataSource <Message> dataSource, IDataSink <Message> dataSink)
{
Add(new AppendAFilter());
Add(new AppendBFilter());
this.dataSource = dataSource;
this.dataSink = dataSink;
}
public AzureBlobDataWriter(
IAzureBlobContainerClientFactory containerClientFactory,
IDataSink dataSink,
ILogger <AzureBlobDataWriter> logger)
{
EnsureArg.IsNotNull(containerClientFactory, nameof(containerClientFactory));
EnsureArg.IsNotNull(logger, nameof(logger));
_containerClient = containerClientFactory.Create(dataSink.StorageUrl, dataSink.Location);
_logger = logger;
}
public EnrichmentMessageRouter(IDataSource dataSource, IDataSink dataSink, IIoTDeviceDataEnricher dataEnricher, ITelemetryClient telemetryClient, IConfiguration config, ILogger <EnrichmentMessageRouter> logger)
{
this.dataSource = dataSource;
this.dataSink = dataSink;
this.dataEnricher = dataEnricher;
this.telemetryClient = telemetryClient;
this.logger = logger;
this.config = config;
// handle messages as they arrive
this.MessageBatchReceived += (sender, e) =>
{
// process with parallelism
Parallel.ForEach(e.Messages, async(message) =>
{
// get the device id
var deviceId = message.GetDeviceId();
// get the metadata
var startTime = DateTime.UtcNow;
var timer = System.Diagnostics.Stopwatch.StartNew();
try
{
var metadata = await this.dataEnricher.GetMetadataAsync(deviceId);
if (metadata != null)
{
// enrich the message
message.EnrichMessage(metadata);
// output the message
await this.dataSink.WriteMessageAsync(message);
// send the telemetry
timer.Stop();
telemetryClient.TrackDependency("gRPC call", "IoTClient", "GetMetadataAzync", startTime, timer.Elapsed, true);
}
}
catch
{
// send the telemetry
timer.Stop();
logger.LogDebug($"gRPC call took {timer.Elapsed.Milliseconds} ms");
telemetryClient.TrackDependency("gRPC call", "IoTClient", "GetMetadataAzync", startTime, timer.Elapsed, false);
}
finally
{
Interlocked.Decrement(ref waiting);
}
});
};
}
public QueueAnalyser(IDataSink dataSink, ISvcBusService svcBusSvc, IConfiguration config, ILogger <QueueAnalyser> logger)
{
_dataSink = dataSink ?? throw new Exception("data sink is null");
_svcBusSvc = svcBusSvc ?? throw new Exception("service is null");
_config = config ?? throw new Exception("config is null");
_logger = logger ?? throw new Exception("Logger is null");
if (_logger.IsEnabled(LogLevel.Debug))
{
_logger.LogDebug($"_dataSink is: {_dataSink}");
_logger.LogDebug($"_svcBusSvc is: {_svcBusSvc}");
_logger.LogDebug($"_config is: {_config}");
_logger.LogDebug($"_logger is: {_logger}");
}
}
public static void WritePedanticData(IDataSink fixture, byte[] testStringBytes)
{
Assert.IsFalse(fixture.IsDataCompleted);
for (int counter = 0;counter < testStringBytes.Length;++counter)
{
Assert.AreEqual(1, fixture.AddBytes(testStringBytes, counter, 1));
if (counter == (testStringBytes.Length - 1))
{
Assert.IsTrue(fixture.IsDataCompleted);
}
else
{
Assert.IsFalse(fixture.IsDataCompleted);
}
}
}
public bool InitializePrepocessing()
{
try
{
string sourceDataFolder = string.IsNullOrEmpty(this._runtimeSettings.SourceDataFolder) ? Directory.GetCurrentDirectory() : this._runtimeSettings.SourceDataFolder;
string[] sources = this._diskIOHandler.DirectoryGetFiles(Path.GetDirectoryName(sourceDataFolder), this._runtimeSettings.LookUpFilePattern, SearchOption.AllDirectories);
foreach (string item in sources)
{
using (IDataSource dataSource = this._dataSourceFactory.GetDataSource(item))
{
IDataLookUpCollection lookUpCollection = new DataLookUpCollection(dataSource, this._logger);
string key = Path.GetFileName(item);
this._toSinkDataChainBuilder.LookUps.Add(key, new HashSet <string>(dataSource.GetHeaders()));
this._dataMapHandler.AddNewDataLookUp(key, lookUpCollection);
}
}
foreach (IDataMapper item in this._dataMapHandler.GetAllMappers())
{
this._toSinkDataChainBuilder.LookUps.Add(item.Name, item.GetAssociatedColumns());
}
using (IDataSource dataSource = this._dataSourceFactory.GetDataSource(this._runtimeSettings.DataSourceFileName))
{
this._toSinkDataChainBuilder.AddSourceColumns(dataSource.GetHeaders());
}
using (IDataSink dataSink = this._dataSinkFactory.GetDataSink(this._runtimeSettings.DataSinkFileName, this._runtimeSettings.OutConfigFileName))
{
this._toSinkDataChainBuilder.AddSinkColumns(dataSink.Columns);
}
Dictionary <string, List <string> > mappingRules = JsonConvert.DeserializeObject <Dictionary <string, List <string> > >(this._diskIOHandler.FileReadAllText(this._runtimeSettings.MappingRulesSourcePath));
foreach (KeyValuePair <string, List <string> > item in mappingRules)
{
this._dataMapHandler.AddNewMappingRule(item.Key, item.Value);
}
this._toSinkDataChainBuilder.BuildChain();
return(true);
}
catch (Exception ex)
{
this._logger.Log("Initialization Failed : see logs for mor information", EventLevel.Error, ex);
}
return(false);
}
public bool PerformTransformation()
{
int ingestRowsCount = 0;
int egressRowsCount = 0;
// Console.Clear();
using (IDataSource dataSource = this._dataSourceFactory.GetDataSource(this._runtimeSettings.DataSourceFileName))
{
using (IDataSink dataSink = this._dataSinkFactory.GetDataSink(this._runtimeSettings.DataSinkFileName, this._runtimeSettings.OutConfigFileName))
{
string[] rowValues = new string[dataSink.Columns.Length];
Dictionary <string, string> context = new Dictionary <string, string>();
foreach (DataCellCollection row in dataSource.GetDataRowEntries())
{
ingestRowsCount += 1;
DataCellCollection outRowCollection = new DataCellCollection();
for (int i = 0; i < dataSink.Columns.Length; i++)
{
Dictionary <string, string> steps = this._toSinkDataChainBuilder.GetSteps(dataSink.Columns[i]);
try
{
outRowCollection = this._dataMapHandler.Resolve(row, new Column()
{
Name = dataSink.Columns[i]
}, outRowCollection, steps, context);
}
catch (Exception ex)
{
this._logger.Log($"resolving failed for {dataSink.Columns[i]} \n raw row data {row.Cells.Select(c => $"{c.Column.Name}: {c.Value}").ToArray()}", EventLevel.Error, ex);
}
}
if (outRowCollection.Cells.Count != dataSink.Columns.Length)
{
this._logger.Log($"resolving failed for raw row data {row.Cells.Select(c => $"{c.Column.Name}: {c.Value}").ToArray()}", EventLevel.Error);
continue;
}
dataSink.AddRecordsToSink(outRowCollection.Cells);
egressRowsCount += 1;
this.ShowProgress(egressRowsCount);
}
}
}
this._logger.LogInformation($"Processing Compeleted");
this._logger.LogInformation($"Ingest = {ingestRowsCount} egress={egressRowsCount}");
this._logger.LogInformation($"Initiating Post Run Analysis ");
egressRowsCount = 0;
using (StreamReader stream = new StreamReader(this._diskIOHandler.FileReadTextStream(this._runtimeSettings.DataSinkFileName)))
{
// int headerCount = CsvParseHelper.GetAllFields(stream.ReadLine()).Length;
while (stream.EndOfStream == false)
{
string line = stream.ReadLine();
//int cellcount = CsvParseHelper.GetAllFields(line).Length;
//if (cellcount != headerCount)
//{
// this._logger.Log($"Error Data Alignment mismatch cellcount {cellcount } != headerCount {headerCount } att position {textLines} , line :{line}", EventLevel.Error);
// noError++;
//}
//textLines += 1;
this.ShowProgress(egressRowsCount++);
}
}
this._logger.LogInformation($"\nPost Analysis Completed egress={--egressRowsCount}");
this._logger.LogInformation($"\nYour output is ready : {this._runtimeSettings.DataSinkFileName}");
return(ingestRowsCount == egressRowsCount);
}
public PostProcessBase(IDataSink sink)
{
this.Sink = sink;
this.Status = new List <string>();
}
public void TempAddSink(IDataSink sink)
{
_tempSerial.AddSink(sink);
}
private static void Run()
{
Log.AttachLogSink(new RS232Writer()); // get this in early
Log.WriteLine("\n--- Logging Initialisation ---");
Log.AttachLogSink(new TextFileWriter("Logs", "Log"));
Log.WriteLine("\n--- AeroDataLogger: Boot Sequence Start ---");
MPU6050Device mpu6050 = new MPU6050Device(); // initalise this before the compass!
MS5611Baro baro = new MS5611Baro();
HMC5883L compass = new HMC5883L();
// Wait for user to start the recording...
var controller = new Controller();
Log.WriteLine("Initialisation successful! Waiting to begin recording...\n");
controller.NotifyReady();
while (!controller.Recording)
{
Thread.Sleep(100);
}
;
// Recording begins...
Log.WriteLine("Starting data capture.");
AccelerationAndGyroData inertialResult;
RawData rawMagnetrometry;
double temp = 0;
double pressure = 0;
using (var fileDataSink = new FileDataSink())
{
var dataSinks = new IDataSink[] { fileDataSink, new DebuggerDataSink() };
DataHandler dataHandler = new DataHandler(dataSinks);
dataHandler.WriteHeader(new[] { "Ax", "Ay", "Az", "Rx", "Ry", "Rz", "Mx", "My", "Mz", "P" });
while (controller.Recording)
{
// Gather data from sensors
inertialResult = mpu6050.GetSensorData();
baro.ReadTemperatureAndPressure(out temp, out pressure);
rawMagnetrometry = compass.Raw;
// Emit data to handlers
dataHandler.HandleData(new object[]
{
inertialResult.Ax,
inertialResult.Ay,
inertialResult.Az,
inertialResult.Rx,
inertialResult.Ry,
inertialResult.Rz,
rawMagnetrometry.X,
rawMagnetrometry.Y,
rawMagnetrometry.Z,
pressure
});
Thread.Sleep(100);
}
}
Log.WriteLine("Data capture stopped.");
Log.WriteLine("Exiting.\n\n");
Log.Close();
}
public void Connect(IDataSink sink)
{
_sink = sink;
}
public Pipeline(IPump <T> pump, IDataSink <T> sink, string pipelineName = "")
{
_pump = pump;
_sink = sink;
_pipelineName = pipelineName;
}
public SinkPostProcessStorage(IDataSink sink, AzureBlobStorageConfiguration config)
: base(sink)
{
this.Configuration = config;
}
public SinkPostProcess(IDataSink sink)
: base(sink)
{
}
public void RemoveSink(IDataSink dataSink)
{
DataSinks.Remove(dataSink);
}
public Pipeline(IDataSource <Message> dataSource, IDataSink <Message> dataSink)
{
this.dataSource = dataSource;
this.dataSink = dataSink;
}
