public static IObservable <DateTimeOffset> GetExportTicks(DateTimeOffset start, DateTimeOffset finish, ExportOptions options, IScheduler scheduler) { return(Observable.Create <DateTimeOffset>(observer => { return scheduler.ScheduleAsync( start, async(schdlr, timestamp, cancellationToken) => { try { var step = TimeSpan.FromMinutes(1); while (timestamp < finish) { var now = schdlr.Now; if (timestamp < now - options.MinLatency) { observer.OnNext(timestamp); } else { await schdlr.Sleep(now - options.MinLatency, cancellationToken); } timestamp += step; } observer.OnCompleted(); } catch (OperationCanceledException exception) when(exception.CancellationToken == cancellationToken) { observer.OnCompleted(); } catch (Exception exception) { observer.OnError(exception); } }); })); }
public static IObservable <Timestamped <string> > CreateJsonExport(this CloudBlobContainer container, DateTimeOffset start, DateTimeOffset finish, ExportOptions options, IScheduler scheduler) => container.CreateJsonExportObservables(start, finish, options, scheduler) .Select(_ => _.Value.Select(json => new Timestamped <string>(json, _.Timestamp))) .Concat();
public static IObservable <Timestamped <IObservable <string> > > CreateJsonExportObservables(this CloudBlobContainer container, DateTimeOffset start, ExportOptions options, IScheduler scheduler) => container.CreateJsonExportObservables(start, DateTimeOffset.MaxValue, options, scheduler);
public static IObservable <Timestamped <DeviceLog[]> > CreateExport(this CloudBlobContainer container, DateTimeOffset start, ExportOptions options) => container.CreateExport(start, DateTimeOffset.MaxValue, options, Scheduler.Default);
public static IObservable <Timestamped <DeviceLog[]> > CreateExport(this CloudBlobContainer container, DateTimeOffset start, DateTimeOffset finish, ExportOptions options, IScheduler scheduler) { if (start > finish) { throw new ArgumentException("start must be less than finish", nameof(finish)); } return(container.CreateJsonExport(start, finish, options, scheduler) .Select(_ => { if (string.IsNullOrEmpty(_.Value)) { return new Timestamped <DeviceLog[]>(Array.Empty <DeviceLog>(), _.Timestamp); } try { return new Timestamped <DeviceLog[]>(JsonConvert.DeserializeObject <DeviceLog[]>(_.Value), _.Timestamp); } catch (Exception exception) { Log.Error(exception, "Error deserializing {JSON} as a device log", _.Value); return new Timestamped <DeviceLog[]>(Array.Empty <DeviceLog>(), _.Timestamp); } })); }
public static IObservable <CloudBlockBlob> CreateMinuteJsonExport(this CloudBlobContainer container, DateTimeOffset timestamp, int maxResults, ExportOptions options, IScheduler scheduler) { const bool useFlatBlobListing = true; const BlobListingDetails blobListingDetails = BlobListingDetails.None; var prefix = $"{timestamp.ToString("yyyy/MM/dd/HH/mm", CultureInfo.InvariantCulture)}/logs"; return(Observable.Create <CloudBlockBlob>(observer => { return scheduler.ScheduleAsync(async(schdlr, cancellationToken) => { try { BlobContinuationToken continuationToken = null; do { var segment = await GetNextBlobSegmentAsync(continuationToken, cancellationToken).ConfigureAwait(false); PublishBlobs(observer, segment.Results); continuationToken = segment.ContinuationToken; } while (!(cancellationToken.IsCancellationRequested || continuationToken is null)); observer.OnCompleted(); } catch (OperationCanceledException exception) when(exception.CancellationToken == cancellationToken) { observer.OnCompleted(); } catch (Exception exception) { observer.OnError(exception); } }); })); Task <BlobResultSegment> GetNextBlobSegmentAsync(BlobContinuationToken continuationToken, CancellationToken cancellationToken) => container.ListBlobsSegmentedAsync(prefix, useFlatBlobListing, blobListingDetails, maxResults, continuationToken, options.BlobRequestOptions, options.OperationContext, cancellationToken); void PublishBlobs(IObserver <CloudBlockBlob> observer, IEnumerable <IListBlobItem> blobs) { foreach (var blob in blobs.Where(b => b.Uri.AbsolutePath.EndsWith("logs.v1.data")).OfType <CloudBlockBlob>()) { observer.OnNext(blob); } } }
public static IObservable <Timestamped <IObservable <string> > > CreateJsonExportObservables(this CloudBlobContainer container, DateTimeOffset start, DateTimeOffset finish, ExportOptions options, IScheduler scheduler) { const int maxResults = 100; return(GetExportTicks(start, finish, options, scheduler) .Select(timestamp => { var observable = container .CreateMinuteJsonExport(timestamp, maxResults, options, scheduler) .SelectMany(blob => { try { return blob.DownloadTextAsync(); } catch (Exception exception) { Log.Error(exception, "Error downloading from {SourceBLOB}", blob.Uri); return Task.FromResult(string.Empty); } }); return new Timestamped <IObservable <string> >(observable, timestamp); })); }
public static async Task Main() { Log.Logger = new LoggerConfiguration() .WriteTo.Console() .CreateLogger(); using (var cts = new CancellationTokenSource()) { var cancellationToken = cts.Token; var start = new DateTimeOffset(2018, 6, 1, 0, 0, 0, TimeSpan.Zero); var options = new ExportOptions(); var logCount = 0UL; var timer = new Stopwatch(); // Setup data source var inputContainer = await CreateInputContainerAsync(InputConnectionString, cts.Token).ConfigureAwait(false); // Create the core observable, and publish it to support multiple subscribers var observable = inputContainer.CreateExport(start, options).Publish(); //// Setup BLOB output var blobSink = await CreateBlobSink(OutputConnectionString, OutputContainerName, options, cancellationToken); observable .Subscribe( onNext: _ => blobSink.Post(_), onError: e => Log.Error(e, "Azure BLOB target stream completed in error"), onCompleted: () => Log.Information("Azure BLOB target stream completed"), token: cancellationToken); // Setup Event Hubs output var eventHubSinks = await CreateEventHubsSink(OutputEventHubsConnectionString, options); observable .SelectMany(_ => _.Value) .GroupBy(deviceLog => GetPartition(deviceLog, (uint)eventHubSinks.Length)) .SelectMany(partition => partition .Buffer(options.EventHubBufferTimeSpan, options.EventHubBufferCount) .Where(batch => batch.Count > 0) .Select(batch => (PartitionId: (int)partition.Key, Payload: batch))) .Subscribe( onNext: batch => { try { eventHubSinks[batch.PartitionId].Post(batch.Payload); } catch (Exception exception) { Log.Error(exception, "Unexpected error posting to Event Hub sender"); } }, onError: e => Log.Error(e, "Event Hubs target stream completed in error"), onCompleted: () => Log.Information("Event Hubs target stream completed"), token: cancellationToken); // Setup console output observable .Subscribe( onNext: _ => { var elapsed = timer.Elapsed; var batchSize = (ulong)_.Value.Length; logCount += batchSize; var rate = Math.Round(logCount / elapsed.TotalSeconds); Log.Information("{Timestamp}: Received a batch of {BatchSize:N0} log(s). Total logs={TotalLogs:N0}. Average rate={LogRate:N0} logs/sec", _.Timestamp, batchSize, logCount, rate); }, onError: e => Log.Error(e, "Event Hubs target stream completed in error"), onCompleted: () => Log.Information("Event Hubs target stream completed"), token: cancellationToken); using (observable.Connect()) { timer.Start(); WriteLine("Press ENTER to terminate..."); ReadLine(); cts.Cancel(); } } }
private static async Task <ImmutableArray <ITargetBlock <IList <DeviceLog> > > > CreateEventHubsSink(string connectionString, ExportOptions options) { var client = EventHubClient.CreateFromConnectionString(connectionString); var information = await client.GetRuntimeInformationAsync().ConfigureAwait(false); var senderOptions = new ExecutionDataflowBlockOptions { BoundedCapacity = options.PartitionQueueLength, EnsureOrdered = true, SingleProducerConstrained = true }; return(information.PartitionIds .Select(partitionId => { var sender = client.CreatePartitionSender(partitionId); // A block that sends batches to the Event Hub partition var sendBlock = new ActionBlock <IList <DeviceLog> >(async batch => { var timer = Stopwatch.StartNew(); var json = JsonConvert.SerializeObject(batch, Formatting.None); var bytes = Encoding.UTF8.GetBytes(json); try { using (var eventData = new EventData(bytes)) await sender.SendAsync(eventData).ConfigureAwait(false); Log.Information("Sent a {Bytes:N0} byte device log batch to Event Hub partition {PartitionId} in {EventHubPartitionSendTime}", bytes.Length, partitionId, timer.Elapsed); } catch (Exception exception) { // Swallow errors Log.Error(exception, "Error sending {Bytes:N0} byte device log batch to Event Hub partition {PartitionId}", bytes.Length, partitionId); } }, senderOptions); return (ITargetBlock <IList <DeviceLog> >)sendBlock; }) .ToImmutableArray()); }
public JsonBlobWriter(CloudBlobContainer container, JsonSerializer serializer, ExportOptions options, Encoding encoding, int bufferSize = 1024) { Container = container ?? throw new ArgumentNullException(nameof(container)); Encoding = encoding ?? Encoding.UTF8; Options = options ?? throw new ArgumentNullException(nameof(options)); Serializer = serializer ?? throw new ArgumentNullException(nameof(serializer)); BufferSize = bufferSize; }