Exemple #1
0
        public void Run()
        {
            var rx = QueueFramework
                     .FromTopicSubscription(
                _config.ReceiveQueue.GetConnectionString(), _config.ReceiveQueue.QueueName, "blobwriter")
                     .WithPrefetchCount(1000)
                     .WithMaxConcurrency(1)

                     .OutputToReactive(_serializer as DatasourceRecordSerializer);

            Log.DebugFormat("DeviceDataBlobWriter Starting");

            //TODO: What if we die while filling the buffer?
            rx.Timestamp().Buffer(1000).Do(message => Log.Debug("writing message")).Subscribe(messages =>
            {
                //write to blob storeage

                // Create file name
                var firstTimeStamp = messages[0].Timestamp;
                var filename       = firstTimeStamp.Year + "\\" + String.Format("{0:MM}", firstTimeStamp)
                                     + "\\" + String.Format("{0:dd}", firstTimeStamp)
                                     + "\\" + String.Format("{0:HH}", firstTimeStamp)
                                     + "\\" + String.Format("{0:mm}", firstTimeStamp)
                                     + "-" + String.Format("{0:ss}", firstTimeStamp)
                                     + "-" + String.Format("{0:ffff}", firstTimeStamp);


                CloudBlockBlob blockBlob = new CloudBlockBlob(new Uri(_storageAccount.BlobEndpoint +
                                                                      _config.DataStorageConfiguration.RawDataContainer + "/" + filename), _storageAccount.Credentials);

                blockBlob.Properties.ContentType     = "text/plain";
                blockBlob.Properties.ContentEncoding = "utf8";

                using (MemoryStream ms = new MemoryStream())
                {
                    byte[] dataToWrite;

                    //Header
                    dataToWrite = Encoding.UTF8.GetBytes("Rx Timestamp, Datasource ID, DataType, EncodedDataType, IntervalSeconds, Source Timestamp, Value\r\n");
                    ms.Write(dataToWrite, 0, dataToWrite.Length);

                    //Body
                    foreach (var m in messages)
                    {
                        dataToWrite = Encoding.UTF8.GetBytes(String.Format("{0:yyyy/MM/dd HH:mm:ss.ffff}", m.Timestamp) + "," + m.Value.DatasourceId + "," + m.Value.DataType + "," + m.Value.EncodedDataType + "," + m.Value.IntervalSeconds + "," + String.Format("{0:yyyy/MM/dd HH:mm:ss.ffff}", m.Value.Timestamp) + "," + m.Value.GetDecimalValue() + "\r\n");
                        ms.Write(dataToWrite, 0, dataToWrite.Length);
                    }
                    ms.Position = 0;
                    blockBlob.UploadFromStream(ms);
                }
            });
        }
Exemple #2
0
        public async void Run()
        {
            rx =
                QueueFramework.FromTopicSubscription(_config.ReceiveQueue.GetConnectionString(),
                                                     _config.ReceiveQueue.QueueName, "rxtest")
                .WithPrefetchCount(1000)
                .WithMaxConcurrency(1)
                .OutputToReactive(_serializer as DatasourceRecordSerializer);

            //rx.Sample(TimeSpan.FromSeconds(1)).Subscribe(x => Log.Debug(x.GetDecimalValue()));

            var      eventsProcessed = 0;
            var      init            = DateTime.UtcNow;
            var      start           = DateTime.UtcNow;
            DateTime end;

            rx.Subscribe(x => { eventsProcessed++; }, ex => Log.DebugFormat("OnError: {0}", ex.Message), () =>
            {
                end = DateTime.UtcNow;
                Log.DebugFormat("OnCompleted: {0} events processed in {1}", eventsProcessed, (end - init).TotalSeconds);
            });

            // Analytic Subscriptions
            var rawValueStream = rx.Select <DatasourceRecord, double>(x =>
            {
                if (x.DataType == DatasourceRecord.DataTypeEnum.Double)
                {
                    return(x.GetDoubleValue());
                }
                else
                {
                    return(0.0D);
                }
            });

            // Calculate Avg and Max values every second

            seqWindowed = rawValueStream.Window(() =>
            {
                var seqWindowControl = Observable.Interval(TimeSpan.FromSeconds(1));
                return(seqWindowControl);
            });

            var total = 0.0D;
            var count = 0;
            var max   = 0.0D;

            seqWindowed.Subscribe(seqWindow =>
            {
                var thisWindowCount = 0;
                seqWindow.Subscribe(x =>
                {
                    total += x;
                    count++;
                    thisWindowCount++;
                    if (x > max)
                    {
                        max = x;
                    }
                    //Console.WriteLine( "Integer : {0}", x );
                }, () =>
                {
                    // Emit objects to new Obersevable
                    averageSubj.OnNext((total / count).ToString("F2"));
                    maxSubj.OnNext(max.ToString("F2"));
                    sampleSubj.OnNext(String.Format("Window Avg: {0}, Count: {1} ", total / count, thisWindowCount));
                });
            });
        }