public void Run()
{
var rx = QueueFramework
.FromTopicSubscription(
_config.ReceiveQueue.GetConnectionString(), _config.ReceiveQueue.QueueName, "blobwriter")
.WithPrefetchCount(1000)
.WithMaxConcurrency(1)
.OutputToReactive(_serializer as DatasourceRecordSerializer);
Log.DebugFormat("DeviceDataBlobWriter Starting");
//TODO: What if we die while filling the buffer?
rx.Timestamp().Buffer(1000).Do(message => Log.Debug("writing message")).Subscribe(messages =>
{
//write to blob storeage
// Create file name
var firstTimeStamp = messages[0].Timestamp;
var filename = firstTimeStamp.Year + "\\" + String.Format("{0:MM}", firstTimeStamp)
+ "\\" + String.Format("{0:dd}", firstTimeStamp)
+ "\\" + String.Format("{0:HH}", firstTimeStamp)
+ "\\" + String.Format("{0:mm}", firstTimeStamp)
+ "-" + String.Format("{0:ss}", firstTimeStamp)
+ "-" + String.Format("{0:ffff}", firstTimeStamp);
CloudBlockBlob blockBlob = new CloudBlockBlob(new Uri(_storageAccount.BlobEndpoint +
_config.DataStorageConfiguration.RawDataContainer + "/" + filename), _storageAccount.Credentials);
blockBlob.Properties.ContentType = "text/plain";
blockBlob.Properties.ContentEncoding = "utf8";
using (MemoryStream ms = new MemoryStream())
{
byte[] dataToWrite;
//Header
dataToWrite = Encoding.UTF8.GetBytes("Rx Timestamp, Datasource ID, DataType, EncodedDataType, IntervalSeconds, Source Timestamp, Value\r\n");
ms.Write(dataToWrite, 0, dataToWrite.Length);
//Body
foreach (var m in messages)
{
dataToWrite = Encoding.UTF8.GetBytes(String.Format("{0:yyyy/MM/dd HH:mm:ss.ffff}", m.Timestamp) + "," + m.Value.DatasourceId + "," + m.Value.DataType + "," + m.Value.EncodedDataType + "," + m.Value.IntervalSeconds + "," + String.Format("{0:yyyy/MM/dd HH:mm:ss.ffff}", m.Value.Timestamp) + "," + m.Value.GetDecimalValue() + "\r\n");
ms.Write(dataToWrite, 0, dataToWrite.Length);
}
ms.Position = 0;
blockBlob.UploadFromStream(ms);
}
});
}
public async void Run()
{
rx =
QueueFramework.FromTopicSubscription(_config.ReceiveQueue.GetConnectionString(),
_config.ReceiveQueue.QueueName, "rxtest")
.WithPrefetchCount(1000)
.WithMaxConcurrency(1)
.OutputToReactive(_serializer as DatasourceRecordSerializer);
//rx.Sample(TimeSpan.FromSeconds(1)).Subscribe(x => Log.Debug(x.GetDecimalValue()));
var eventsProcessed = 0;
var init = DateTime.UtcNow;
var start = DateTime.UtcNow;
DateTime end;
rx.Subscribe(x => { eventsProcessed++; }, ex => Log.DebugFormat("OnError: {0}", ex.Message), () =>
{
end = DateTime.UtcNow;
Log.DebugFormat("OnCompleted: {0} events processed in {1}", eventsProcessed, (end - init).TotalSeconds);
});
// Analytic Subscriptions
var rawValueStream = rx.Select <DatasourceRecord, double>(x =>
{
if (x.DataType == DatasourceRecord.DataTypeEnum.Double)
{
return(x.GetDoubleValue());
}
else
{
return(0.0D);
}
});
// Calculate Avg and Max values every second
seqWindowed = rawValueStream.Window(() =>
{
var seqWindowControl = Observable.Interval(TimeSpan.FromSeconds(1));
return(seqWindowControl);
});
var total = 0.0D;
var count = 0;
var max = 0.0D;
seqWindowed.Subscribe(seqWindow =>
{
var thisWindowCount = 0;
seqWindow.Subscribe(x =>
{
total += x;
count++;
thisWindowCount++;
if (x > max)
{
max = x;
}
//Console.WriteLine( "Integer : {0}", x );
}, () =>
{
// Emit objects to new Obersevable
averageSubj.OnNext((total / count).ToString("F2"));
maxSubj.OnNext(max.ToString("F2"));
sampleSubj.OnNext(String.Format("Window Avg: {0}, Count: {1} ", total / count, thisWindowCount));
});
});
}