public static IObservable <DateTimeOffset> GetExportTicks(DateTimeOffset start, DateTimeOffset finish, ExportOptions options, IScheduler scheduler)
{
return(Observable.Create <DateTimeOffset>(observer =>
{
return scheduler.ScheduleAsync(
start,
async(schdlr, timestamp, cancellationToken) =>
{
try
{
var step = TimeSpan.FromMinutes(1);
while (timestamp < finish)
{
var now = schdlr.Now;
if (timestamp < now - options.MinLatency)
{
observer.OnNext(timestamp);
}
else
{
await schdlr.Sleep(now - options.MinLatency, cancellationToken);
}
timestamp += step;
}
observer.OnCompleted();
}
catch (OperationCanceledException exception) when(exception.CancellationToken == cancellationToken)
{
observer.OnCompleted();
}
catch (Exception exception)
{
observer.OnError(exception);
}
});
}));
}
public static IObservable <Timestamped <string> > CreateJsonExport(this CloudBlobContainer container, DateTimeOffset start, DateTimeOffset finish, ExportOptions options, IScheduler scheduler) => container.CreateJsonExportObservables(start, finish, options, scheduler) .Select(_ => _.Value.Select(json => new Timestamped <string>(json, _.Timestamp))) .Concat();
public static IObservable <Timestamped <IObservable <string> > > CreateJsonExportObservables(this CloudBlobContainer container, DateTimeOffset start, ExportOptions options, IScheduler scheduler) => container.CreateJsonExportObservables(start, DateTimeOffset.MaxValue, options, scheduler);
public static IObservable <Timestamped <DeviceLog[]> > CreateExport(this CloudBlobContainer container, DateTimeOffset start, ExportOptions options) => container.CreateExport(start, DateTimeOffset.MaxValue, options, Scheduler.Default);
public static IObservable <Timestamped <DeviceLog[]> > CreateExport(this CloudBlobContainer container, DateTimeOffset start, DateTimeOffset finish, ExportOptions options, IScheduler scheduler)
{
if (start > finish)
{
throw new ArgumentException("start must be less than finish", nameof(finish));
}
return(container.CreateJsonExport(start, finish, options, scheduler)
.Select(_ =>
{
if (string.IsNullOrEmpty(_.Value))
{
return new Timestamped <DeviceLog[]>(Array.Empty <DeviceLog>(), _.Timestamp);
}
try
{
return new Timestamped <DeviceLog[]>(JsonConvert.DeserializeObject <DeviceLog[]>(_.Value), _.Timestamp);
}
catch (Exception exception)
{
Log.Error(exception, "Error deserializing {JSON} as a device log", _.Value);
return new Timestamped <DeviceLog[]>(Array.Empty <DeviceLog>(), _.Timestamp);
}
}));
}
public static IObservable <CloudBlockBlob> CreateMinuteJsonExport(this CloudBlobContainer container, DateTimeOffset timestamp, int maxResults, ExportOptions options, IScheduler scheduler)
{
const bool useFlatBlobListing = true;
const BlobListingDetails blobListingDetails = BlobListingDetails.None;
var prefix = $"{timestamp.ToString("yyyy/MM/dd/HH/mm", CultureInfo.InvariantCulture)}/logs";
return(Observable.Create <CloudBlockBlob>(observer =>
{
return scheduler.ScheduleAsync(async(schdlr, cancellationToken) =>
{
try
{
BlobContinuationToken continuationToken = null;
do
{
var segment = await GetNextBlobSegmentAsync(continuationToken, cancellationToken).ConfigureAwait(false);
PublishBlobs(observer, segment.Results);
continuationToken = segment.ContinuationToken;
} while (!(cancellationToken.IsCancellationRequested || continuationToken is null));
observer.OnCompleted();
}
catch (OperationCanceledException exception) when(exception.CancellationToken == cancellationToken)
{
observer.OnCompleted();
}
catch (Exception exception)
{
observer.OnError(exception);
}
});
}));
Task <BlobResultSegment> GetNextBlobSegmentAsync(BlobContinuationToken continuationToken, CancellationToken cancellationToken)
=> container.ListBlobsSegmentedAsync(prefix, useFlatBlobListing, blobListingDetails, maxResults, continuationToken, options.BlobRequestOptions, options.OperationContext, cancellationToken);
void PublishBlobs(IObserver <CloudBlockBlob> observer, IEnumerable <IListBlobItem> blobs)
{
foreach (var blob in blobs.Where(b => b.Uri.AbsolutePath.EndsWith("logs.v1.data")).OfType <CloudBlockBlob>())
{
observer.OnNext(blob);
}
}
}
public static IObservable <Timestamped <IObservable <string> > > CreateJsonExportObservables(this CloudBlobContainer container, DateTimeOffset start, DateTimeOffset finish, ExportOptions options, IScheduler scheduler)
{
const int maxResults = 100;
return(GetExportTicks(start, finish, options, scheduler)
.Select(timestamp =>
{
var observable = container
.CreateMinuteJsonExport(timestamp, maxResults, options, scheduler)
.SelectMany(blob =>
{
try
{
return blob.DownloadTextAsync();
}
catch (Exception exception)
{
Log.Error(exception, "Error downloading from {SourceBLOB}", blob.Uri);
return Task.FromResult(string.Empty);
}
});
return new Timestamped <IObservable <string> >(observable, timestamp);
}));
}
public static async Task Main()
{
Log.Logger = new LoggerConfiguration()
.WriteTo.Console()
.CreateLogger();
using (var cts = new CancellationTokenSource())
{
var cancellationToken = cts.Token;
var start = new DateTimeOffset(2018, 6, 1, 0, 0, 0, TimeSpan.Zero);
var options = new ExportOptions();
var logCount = 0UL;
var timer = new Stopwatch();
// Setup data source
var inputContainer = await CreateInputContainerAsync(InputConnectionString, cts.Token).ConfigureAwait(false);
// Create the core observable, and publish it to support multiple subscribers
var observable = inputContainer.CreateExport(start, options).Publish();
//// Setup BLOB output
var blobSink = await CreateBlobSink(OutputConnectionString, OutputContainerName, options, cancellationToken);
observable
.Subscribe(
onNext: _ => blobSink.Post(_),
onError: e => Log.Error(e, "Azure BLOB target stream completed in error"),
onCompleted: () => Log.Information("Azure BLOB target stream completed"),
token: cancellationToken);
// Setup Event Hubs output
var eventHubSinks = await CreateEventHubsSink(OutputEventHubsConnectionString, options);
observable
.SelectMany(_ => _.Value)
.GroupBy(deviceLog => GetPartition(deviceLog, (uint)eventHubSinks.Length))
.SelectMany(partition =>
partition
.Buffer(options.EventHubBufferTimeSpan, options.EventHubBufferCount)
.Where(batch => batch.Count > 0)
.Select(batch => (PartitionId: (int)partition.Key, Payload: batch)))
.Subscribe(
onNext: batch =>
{
try
{
eventHubSinks[batch.PartitionId].Post(batch.Payload);
}
catch (Exception exception)
{
Log.Error(exception, "Unexpected error posting to Event Hub sender");
}
},
onError: e => Log.Error(e, "Event Hubs target stream completed in error"),
onCompleted: () => Log.Information("Event Hubs target stream completed"),
token: cancellationToken);
// Setup console output
observable
.Subscribe(
onNext: _ =>
{
var elapsed = timer.Elapsed;
var batchSize = (ulong)_.Value.Length;
logCount += batchSize;
var rate = Math.Round(logCount / elapsed.TotalSeconds);
Log.Information("{Timestamp}: Received a batch of {BatchSize:N0} log(s). Total logs={TotalLogs:N0}. Average rate={LogRate:N0} logs/sec", _.Timestamp, batchSize, logCount, rate);
},
onError: e => Log.Error(e, "Event Hubs target stream completed in error"),
onCompleted: () => Log.Information("Event Hubs target stream completed"),
token: cancellationToken);
using (observable.Connect())
{
timer.Start();
WriteLine("Press ENTER to terminate...");
ReadLine();
cts.Cancel();
}
}
}
private static async Task <ImmutableArray <ITargetBlock <IList <DeviceLog> > > > CreateEventHubsSink(string connectionString, ExportOptions options)
{
var client = EventHubClient.CreateFromConnectionString(connectionString);
var information = await client.GetRuntimeInformationAsync().ConfigureAwait(false);
var senderOptions = new ExecutionDataflowBlockOptions
{
BoundedCapacity = options.PartitionQueueLength,
EnsureOrdered = true,
SingleProducerConstrained = true
};
return(information.PartitionIds
.Select(partitionId =>
{
var sender = client.CreatePartitionSender(partitionId);
// A block that sends batches to the Event Hub partition
var sendBlock = new ActionBlock <IList <DeviceLog> >(async batch =>
{
var timer = Stopwatch.StartNew();
var json = JsonConvert.SerializeObject(batch, Formatting.None);
var bytes = Encoding.UTF8.GetBytes(json);
try
{
using (var eventData = new EventData(bytes))
await sender.SendAsync(eventData).ConfigureAwait(false);
Log.Information("Sent a {Bytes:N0} byte device log batch to Event Hub partition {PartitionId} in {EventHubPartitionSendTime}", bytes.Length, partitionId, timer.Elapsed);
}
catch (Exception exception)
{
// Swallow errors
Log.Error(exception, "Error sending {Bytes:N0} byte device log batch to Event Hub partition {PartitionId}", bytes.Length, partitionId);
}
},
senderOptions);
return (ITargetBlock <IList <DeviceLog> >)sendBlock;
})
.ToImmutableArray());
}
public JsonBlobWriter(CloudBlobContainer container, JsonSerializer serializer, ExportOptions options, Encoding encoding, int bufferSize = 1024)
{
Container = container ?? throw new ArgumentNullException(nameof(container));
Encoding = encoding ?? Encoding.UTF8;
Options = options ?? throw new ArgumentNullException(nameof(options));
Serializer = serializer ?? throw new ArgumentNullException(nameof(serializer));
BufferSize = bufferSize;
}