public Task ProcessChangesAsync(ChangeFeedObserverContext context, IReadOnlyList <Document> docs)
{
return(this.executor.TryExecuteAsync(new TriggeredFunctionData()
{
TriggerValue = docs
}, CancellationToken.None));
}
/// <summary>
/// Called when change feed observer is closed;
/// this function prints out observer partition key id and reason for shut down.
/// </summary>
/// <param name="context">The context specifying partition for this observer, etc.</param>
/// <param name="reason">Specifies the reason the observer is closed.</param>
/// <returns>A Task to allow asynchronous execution</returns>
public Task CloseAsync(ChangeFeedObserverContext context, ChangeFeedObserverCloseReason reason)
{
Console.ForegroundColor = ConsoleColor.Cyan;
Console.WriteLine("Observer closed, {0}", context.PartitionKeyRangeId);
Console.WriteLine("Reason for shutdown, {0}", reason);
return(Task.CompletedTask);
}
public Task CloseAsync(ChangeFeedObserverContext context, ChangeFeedObserverCloseReason reason)
{
Console.WriteLine($"Worker closed, {context.PartitionKeyRangeId}, reason {reason}");
return(Task.CompletedTask);
}
public async Task ProcessChangesAsync(ChangeFeedObserverContext context, IReadOnlyList <Document> docs)
{
var sw = new Stopwatch();
sw.Start();
var documentHashCalculator = ServiceLocator.GetService <IDocumentHashCalculator>();
var documentChanges = new List <DocumentChangeInfo>();
foreach (var doc in docs)
{
int contentHash = documentHashCalculator.CalculateHash(doc);
documentChanges.Add(new DocumentChangeInfo
{
PartitionKey = doc.GetPropertyValue <string>(_config.MonitoredCollectionPartitionKeyName),
Id = doc.Id,
ContentHash = contentHash
});
}
var changeTracker = ServiceLocator.GetService <IChangeTracker>();
await changeTracker.ProcessContentHash(documentChanges);
Console.WriteLine($"Handled {docs.Count} documents in {sw.ElapsedMilliseconds} ms - avarage of {sw.ElapsedMilliseconds / docs.Count}");
}
/// <summary>
/// Called when change feed observer is opened;
/// this function prints out observer partition key id.
/// </summary>
/// <param name="context">The context specifying partition for this observer, etc.</param>
/// <returns>A Task to allow asynchronous execution</returns>
public Task OpenAsync(ChangeFeedObserverContext context)
{
EchoApp.Program.logDebug("OpenAsync");
Console.ForegroundColor = ConsoleColor.Magenta;
Console.WriteLine("Observer opened for partition Key Range: {0}", context.PartitionKeyRangeId);
return(Task.CompletedTask);
}
public Task ProcessChangesAsync(ChangeFeedObserverContext context, IReadOnlyList <Document> docs)
{
foreach (var doc in docs)
{
this.DocumentReceived(doc);
}
return(Task.CompletedTask);
}
public Task OpenAsync(ChangeFeedObserverContext context)
{
if (context == null)
{
throw new ArgumentNullException("context", "Missing observer context");
}
return(Task.CompletedTask);
}
public Task ProcessChangesAsync(ChangeFeedObserverContext context, IReadOnlyList <Document> docs)
{
foreach (Document document in docs)
{
Console.WriteLine($"Found document with ID '{document.Id}'.");
}
return(Task.FromResult <object>(null));
}
/// <summary>
/// When document changes are available on change feed, changes are copied to destination connection;
/// this function prints out the changed document ID.
/// </summary>
/// <param name="context">The context specifying partition for this observer, etc.</param>
/// <param name="docs">The documents changed.</param>
/// <returns>A Task to allow asynchronous execution</returns>
public async Task ProcessChangesAsync(ChangeFeedObserverContext context, IReadOnlyList <Document> docs)
{
Console.WriteLine("Change feed: total {0} doc(s)", Interlocked.Add(ref totalDocs, docs.Count));
foreach (Document doc in docs)
{
Console.WriteLine(doc.Id.ToString());
await this.client.UpsertDocumentAsync(this.destinationCollectionUri, doc);
}
}
/// <summary>
/// Called after OpenAsnc
/// </summary>
/// <param name="context"></param>
/// <param name="docs"></param>
/// <returns></returns>
public Task ProcessChangesAsync(ChangeFeedObserverContext context, IReadOnlyList <Document> docs)
{
Console.WriteLine($" Processing partition key range: {context.PartitionKeyRangeId} - {docs.Count()} changed documents...");
foreach (Person changedPerson in docs)
{
Console.WriteLine($" Changed Person: {changedPerson.name}!");
}
return(Task.CompletedTask);
}
public async Task ProcessChanges_WhenCheckpointThrows_ShouldThrow()
{
checkpointFrequency.TimeInterval = TimeSpan.Zero;
ChangeFeedObserverContext observerContext = Mock.Of <ChangeFeedObserverContext>();
Mock.Get(observerContext).Setup(abs => abs.CheckpointAsync()).Throws(new LeaseLostException());
await Assert.ThrowsExceptionAsync <LeaseLostException>(() => sut.ProcessChangesAsync(observerContext, documents, CancellationToken.None));
}
public Task OpenAsync(ChangeFeedObserverContext context)
{
if (this.openProcessor != null)
{
return(this.openProcessor(context));
}
else
{
return(Task.CompletedTask);
}
}
public Task ProcessChangesAsync(ChangeFeedObserverContext context, IReadOnlyList <Document> docs)
{
foreach (var doc in docs)
{
if (destinationCollectionUri != null)
{
client.UpsertDocumentAsync(destinationCollectionUri, doc);
}
}
return(Task.CompletedTask);
}
public Task CloseAsync(ChangeFeedObserverContext context, ChangeFeedObserverCloseReason reason)
{
if (this.closeProcessor != null)
{
return(this.closeProcessor(context, reason));
}
else
{
return(Task.CompletedTask);
}
}
public Task ProcessChangesAsync(ChangeFeedObserverContext context, IReadOnlyList <Document> docs)
{
if (this.changeProcessor != null)
{
return(this.changeProcessor(context, docs));
}
else
{
return(Task.CompletedTask);
}
}
public async Task RunAsync(CancellationToken shutdownToken)
{
var context = new ChangeFeedObserverContext(this.lease.PartitionId);
await this.observer.OpenAsync(context).ConfigureAwait(false);
this.processorCancellation = CancellationTokenSource.CreateLinkedTokenSource(shutdownToken);
Task processorTask = this.processor.RunAsync(this.processorCancellation.Token);
processorTask.ContinueWith(_ => this.renewerCancellation.Cancel()).LogException();
Task renewerTask = this.renewer.RunAsync(this.renewerCancellation.Token);
renewerTask.ContinueWith(_ => this.processorCancellation.Cancel()).LogException();
ChangeFeedObserverCloseReason closeReason = shutdownToken.IsCancellationRequested ?
ChangeFeedObserverCloseReason.Shutdown :
ChangeFeedObserverCloseReason.Unknown;
try
{
await Task.WhenAll(processorTask, renewerTask).ConfigureAwait(false);
}
catch (LeaseLostException)
{
closeReason = ChangeFeedObserverCloseReason.LeaseLost;
throw;
}
catch (PartitionSplitException)
{
closeReason = ChangeFeedObserverCloseReason.LeaseGone;
throw;
}
catch (OperationCanceledException) when(shutdownToken.IsCancellationRequested)
{
closeReason = ChangeFeedObserverCloseReason.Shutdown;
}
catch (ObserverException)
{
closeReason = ChangeFeedObserverCloseReason.ObserverError;
throw;
}
catch (Exception) when(processorTask.IsFaulted)
{
closeReason = ChangeFeedObserverCloseReason.Unknown;
throw;
}
finally
{
await this.observer.CloseAsync(context, closeReason).ConfigureAwait(false);
}
}
public Task CloseAsync(ChangeFeedObserverContext context, ChangeFeedObserverCloseReason reason)
{
if (reason == ChangeFeedObserverCloseReason.LeaseLost)
{
// The lease is lost by ChangeFeedEventHost but it still exists and ChangeFeedEventHost
// keeps calling ProcessAsync which is IMHO a bug.
// There is an open issue on git hub: https://github.com/Azure/azure-documentdb-dotnet/issues/237
// Just be aware of this, if you create/dispose objects. Do not dispose objects immediatly in case of "LeaseLost"
// as workaround. At some later point in time the ChangeFeedEventHost disposes the Factory and creates a new one
}
return(Task.CompletedTask); // Framework 4.6 above
}
public ObserverExceptionWrappingChangeFeedObserverDecoratorTests()
{
this.observer = new Mock <ChangeFeedObserver <MyDocument> >();
this.changeFeedObserverContext = Mock.Of <ChangeFeedObserverContext>();
this.observerWrapper = new FeedProcessing.ObserverExceptionWrappingChangeFeedObserverDecorator <MyDocument>(this.observer.Object);
var document = new MyDocument();
documents = new List <MyDocument> {
document
};
}
/// <summary>
/// When document changes are available on change feed, changes are copied to destination connection;
/// this function prints out the changed document ID.
/// </summary>
/// <param name="context">The context specifying partition for this observer, etc.</param>
/// <param name="docs">The documents changed.</param>
/// <returns>A Task to allow asynchronous execution</returns>
public Task ProcessChangesAsync(ChangeFeedObserverContext context, IReadOnlyList <Document> docs)
{
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("Change feed: PartitionId {0} total {1} doc(s)", context.PartitionKeyRangeId, Interlocked.Add(ref _totalDocs, docs.Count));
foreach (Document doc in docs)
{
// EXERCISE:
// Write code to set the name of the planet into the document.
// We've already instantiated _planetsTable for you.
}
return(Task.CompletedTask);
}
/// <summary>
/// When document changes are available on change feed, changes are copied to destination connection;
/// this function prints out the changed document ID.
/// </summary>
/// <param name="context">The context specifying partition for this observer, etc.</param>
/// <param name="docs">The documents changed.</param>
/// <returns>A Task to allow asynchronous execution</returns>
public async Task ProcessChangesAsync(ChangeFeedObserverContext context, IReadOnlyList <Document> docs)
{
Console.WriteLine("Change feed: PartitionId {0} total {1} doc(s)", context.PartitionKeyRangeId, Interlocked.Add(ref totalDocs, docs.Count));
foreach (Document doc in docs)
{
Console.WriteLine(doc.Id.ToString());
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine(doc.Id.ToString());
// if you want process each record, write your implemention here
// write your implemention here
//if you want to process batch
await _checkpoint(context, doc);
}
}
public AutoCheckPointTests()
{
changeFeedObserver = Mock.Of <ChangeFeedObserver <dynamic> >();
partitionCheckpointer = Mock.Of <PartitionCheckpointer>();
Mock.Get(partitionCheckpointer)
.Setup(checkPointer => checkPointer.CheckpointPartitionAsync(It.IsAny <string>()))
.Returns(Task.CompletedTask);
checkpointFrequency = new CheckpointFrequency();
sut = new AutoCheckpointer <dynamic>(checkpointFrequency, changeFeedObserver);
documents = Mock.Of <IReadOnlyList <dynamic> >();
observerContext = Mock.Of <ChangeFeedObserverContext>();
Mock.Get(observerContext)
.Setup(context => context.CheckpointAsync())
.Returns(partitionCheckpointer.CheckpointPartitionAsync("token"));
}
/// <summary>
/// When document changes are available on change feed, changes are copied to destination connection;
/// this function prints out the changed document ID.
/// </summary>
/// <param name="context">The context specifying partition for this observer, etc.</param>
/// <param name="docs">The documents changed.</param>
/// <returns>A Task to allow asynchronous execution</returns>
public Task ProcessChangesAsync(ChangeFeedObserverContext context, IReadOnlyList <Document> docs)
{
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("Change feed: PartitionId {0} total {1} doc(s)", context.PartitionKeyRangeId, Interlocked.Add(ref totalDocs, docs.Count));
foreach (Document doc in docs)
{
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine(doc.Id.ToString());
if (this.destinationCollectionUri != null)
{
this.client.UpsertDocumentAsync(this.destinationCollectionUri, doc);
}
}
return(Task.CompletedTask);
}
public async Task ProcessChangesAsync(ChangeFeedObserverContext context, IReadOnlyList <Document> docs)
{
foreach (var doc in docs)
{
var @event = DocumentDbStorageEvent.FromDocument(doc).ToStorageEvent(typeMap);
switch (@event.EventBody)
{
case OrderCreated body:
await Project(@event, body);
break;
case OrderDispatched e:
break;
}
}
Console.WriteLine($"Got {docs.Count} events!");
}
public Task ProcessChangesAsync(ChangeFeedObserverContext context, IReadOnlyList <Document> docs)
{
var partitionProcessors = new List <Task>();
var partitionGroups = Deconstruct <Item>(docs);
Log($@"Change feed: {docs.Count} Partition Group(s): {partitionGroups.Count()}");
foreach (var partition in partitionGroups)
{
Log($@"Processing Partition: {partition.Key}");
partitionProcessors.Add(Task.Factory.StartNew(() => ProcessPartition(partition.Key, partition.Value)));
}
Log($@"All Partions Hot");
Task.WaitAll(partitionProcessors.ToArray());
Log($@"All Partions Processed");
Log(FakeActor <Item> .DisplayDocuments());
return(Task.FromResult(1));
}
/// <summary>
/// When document changes are available on change feed, changes are copied to destination connection;
/// this function prints out the changed document ID.
/// </summary>
/// <param name="context">The context specifying partition for this observer, etc.</param>
/// <param name="docs">The documents changed.</param>
/// <returns>A Task to allow asynchronous execution</returns>
public Task ProcessChangesAsync(ChangeFeedObserverContext context, IReadOnlyList <Document> docs)
{
EchoApp.Program.logDebug("ProcessChangesAsync");
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("Change feed: PartitionId {0} total {1} doc(s)", context.PartitionKeyRangeId, Interlocked.Add(ref totalDocs, docs.Count));
foreach (Document doc in docs)
{
Console.ForegroundColor = ConsoleColor.Yellow;
// Console.WriteLine(doc.Id.ToString());
var content = doc.GetPropertyValue <String>("content");
// Console.WriteLine(content);
EchoApp.Startup.gotChanged(content);
if (this.destinationCollectionUri != null)
{
// this.client.UpsertDocumentAsync(this.destinationCollectionUri, doc);
}
}
return(Task.CompletedTask);
}
/// <summary>
/// Handles change feed documents called by host
/// </summary>
/// <param name="context"></param>
/// <param name="docs"></param>
/// <returns></returns>
async Task IChangeFeedObserver.ProcessChangesAsync(ChangeFeedObserverContext context, IReadOnlyList <Document> docs)
{
var documents = new List <Document>();
foreach (Document doc in docs)
{
var status = doc.GetPropertyValue <QueueItemStatus>("status");
if (status == QueueItemStatus.Pending)
{
documents.Add(doc);
}
}
// Lock queue items
var queueItems = await TryAdquireLock(documents);
if (queueItems.Count > 0)
{
await ProcessQueueItems(queueItems);
}
}
public Task OpenAsync(ChangeFeedObserverContext context)
{
Console.WriteLine("Worker opened, {0}", context.PartitionKeyRangeId);
return(Task.CompletedTask); // Requires targeting .NET 4.6+.
}
public Task ProcessChangesAsync(ChangeFeedObserverContext context, IReadOnlyList <Document> docs)
{
return(this.parent.ProcessChangesAsync(context, docs));
}
public Task CloseAsync(ChangeFeedObserverContext context, ChangeFeedObserverCloseReason reason)
{
return(parent.CloseAsync(context, reason));
}
public Task OpenAsync(ChangeFeedObserverContext context)
{
return(parent.OpenAsync(context));
}
