public WorkerData(Task task, IChangeFeedObserver observer, ChangeFeedObserverContext context, CancellationTokenSource cancellation)
{
this.Task = task;
this.Observer = observer;
this.Context = context;
this.Cancellation = cancellation;
}
public FeedProcessing.IChangeFeedObserver CreateObserver()
{
#pragma warning disable CS0618 // Type or member is obsolete
IChangeFeedObserver observer = this.factory.CreateObserver();
#pragma warning restore CS0618 // Type or member is obsolete
return(new ChangeFeedObserverAdapter(observer));
}
public IPartitionProcessor Create(ILease lease, IChangeFeedObserver observer)
{
if (observer == null)
{
throw new ArgumentNullException(nameof(observer));
}
if (lease == null)
{
throw new ArgumentNullException(nameof(lease));
}
var settings = new ProcessorSettings
{
CollectionSelfLink = this.collectionSelfLink,
StartContinuation = !string.IsNullOrEmpty(lease.ContinuationToken) ?
lease.ContinuationToken :
this.changeFeedProcessorOptions.StartContinuation,
PartitionKeyRangeId = lease.PartitionId,
FeedPollDelay = this.changeFeedProcessorOptions.FeedPollDelay,
MaxItemCount = this.changeFeedProcessorOptions.MaxItemCount,
StartFromBeginning = this.changeFeedProcessorOptions.StartFromBeginning,
StartTime = this.changeFeedProcessorOptions.StartTime,
SessionToken = this.changeFeedProcessorOptions.SessionToken,
};
var checkpointer = new PartitionCheckpointer(this.leaseCheckpointer, lease);
return(new PartitionProcessor(observer, this.documentClient, settings, checkpointer));
}
public PartitionControllerTests()
{
lease = Mock.Of <ILease>();
Mock.Get(lease)
.Setup(l => l.PartitionId)
.Returns("partitionId");
partitionProcessor = MockPartitionProcessor();
leaseRenewer = MockRenewer();
observer = MockObserver();
partitionSupervisorFactory = Mock.Of <IPartitionSupervisorFactory>(f => f.Create(lease) == new PartitionSupervisor(lease, observer, partitionProcessor, leaseRenewer));
leaseManager = Mock.Of <ILeaseManager>();
Mock.Get(leaseManager).Reset(); // Reset implicit/by default setup of properties.
Mock.Get(leaseManager)
.Setup(manager => manager.AcquireAsync(lease))
.ReturnsAsync(lease);
Mock.Get(leaseManager)
.Setup(manager => manager.ReleaseAsync(lease))
.Returns(Task.CompletedTask);
var leaseContainer = Mock.Of <ILeaseContainer>();
synchronizer = Mock.Of <IPartitionSynchronizer>();
sut = new PartitionController(leaseContainer, leaseManager, partitionSupervisorFactory, synchronizer);
}
public AutoCheckPointTests()
{
changeFeedObserver = Mock.Of <IChangeFeedObserver>();
partitionCheckpointer = Mock.Of <IPartitionCheckpointer>();
Mock.Get(partitionCheckpointer)
.Setup(checkPointer => checkPointer.CheckpointPartitionAsync(It.IsAny <string>()))
.Returns(Task.CompletedTask);
checkpointFrequency = new CheckpointFrequency();
sut = new AutoCheckpointer(checkpointFrequency, changeFeedObserver);
documents = Mock.Of <IReadOnlyList <Document> >();
feedResponse = Mock.Of <IFeedResponse <Document> >();
Mock.Get(feedResponse)
.Setup(response => response.Count)
.Returns(documents.Count);
Mock.Get(feedResponse)
.Setup(response => response.ResponseContinuation)
.Returns("token");
Mock.Get(feedResponse)
.Setup(response => response.GetEnumerator())
.Returns(documents.GetEnumerator());
observerContext = Mock.Of <IChangeFeedObserverContext>();
Mock.Get(observerContext)
.Setup(context => context.CheckpointAsync())
.Returns(partitionCheckpointer.CheckpointPartitionAsync("token"));
}
public PartitionSupervisor(ILease lease, IChangeFeedObserver observer, IPartitionProcessor processor, ILeaseRenewer renewer)
{
this.lease = lease;
this.observer = observer;
this.processor = processor;
this.renewer = renewer;
}
public PartitionProcessor(IChangeFeedObserver observer, IChangeFeedDocumentQuery <Document> query, ChangeFeedOptions options, ProcessorSettings settings, IPartitionCheckpointer checkpointer)
{
this.observer = observer;
this.settings = settings;
this.checkpointer = checkpointer;
this.options = options;
this.query = query;
}
public PartitionProcessorFactoryTests()
{
this.collectionSelfLink = this.fixture.Create <string>();
this.docClient = Mock.Of <IChangeFeedDocumentClient>();
Mock.Get(this.docClient)
.Setup(ex => ex.CreateDocumentChangeFeedQuery(this.collectionSelfLink, It.IsAny <ChangeFeedOptions>()))
.Returns(Mock.Of <IChangeFeedDocumentQuery <Document> >());
this.observer = Mock.Of <IChangeFeedObserver>();
}
/// <summary>
/// Creates a new instance of <see cref="Documents.ChangeFeedProcessor.IChangeFeedObserver"/>.
/// </summary>
/// <returns>Created instance of <see cref="Documents.ChangeFeedProcessor.IChangeFeedObserver"/>.</returns>
public IChangeFeedObserver CreateObserver()
{
IChangeFeedObserver observer = this.observerFactory.CreateObserver();
if (this.checkpointFrequency.ExplicitCheckpoint)
{
return(observer);
}
return(new AutoCheckpointer(this.checkpointFrequency, observer));
}
public ObserverExceptionWrappingChangeFeedObserverDecoratorTests()
{
this.observer = Mock.Of <IChangeFeedObserver>();
this.changeFeedObserverContext = Mock.Of <IChangeFeedObserverContext>();
this.observerWrapper = new FeedProcessing.ObserverExceptionWrappingChangeFeedObserverDecorator(observer);
var document = new Document();
documents = new List <Document> {
document
};
}
public IPartitionSupervisor Create(ILease lease)
{
if (lease == null)
{
throw new ArgumentNullException(nameof(lease));
}
IChangeFeedObserver changeFeedObserver = this.observerFactory.CreateObserver();
var processor = this.partitionProcessorFactory.Create(lease, this.leaseCheckpointer, changeFeedObserver);
var renewer = new LeaseRenewer(lease, this.leaseManager, this.changeFeedProcessorOptions.LeaseRenewInterval);
return(new PartitionSupervisor(lease, changeFeedObserver, processor, renewer));
}
public PartitionSupervisorTests()
{
lease = Mock.Of <ILease>();
Mock.Get(lease)
.Setup(l => l.PartitionId)
.Returns("partitionId");
leaseRenewer = Mock.Of <ILeaseRenewer>();
partitionProcessor = Mock.Of <IPartitionProcessor>();
observer = Mock.Of <IChangeFeedObserver>();
sut = new PartitionSupervisor(lease, observer, partitionProcessor, leaseRenewer);
}
public WorkerData(Task task, IChangeFeedObserver observer, ChangeFeedObserverContext context, CancellationTokenSource cancellation, DocumentServiceLease lease)
{
Debug.Assert(task != null);
Debug.Assert(observer != null);
Debug.Assert(context != null);
Debug.Assert(cancellation != null);
this.Task = task;
this.Observer = observer;
this.Context = context;
this.Cancellation = cancellation;
this.Lease = lease;
}
public AutoCheckpointer(CheckpointFrequency checkpointFrequency, IChangeFeedObserver observer)
{
if (checkpointFrequency == null)
{
throw new ArgumentNullException(nameof(checkpointFrequency));
}
if (observer == null)
{
throw new ArgumentNullException(nameof(observer));
}
this.checkpointFrequency = checkpointFrequency;
this.observer = observer;
}
public PartitionExceptionsTests()
{
processorSettings = new ProcessorSettings
{
CollectionSelfLink = "selfLink",
FeedPollDelay = TimeSpan.FromMilliseconds(16),
MaxItemCount = 5,
PartitionKeyRangeId = "keyRangeId",
RequestContinuation = "initialToken"
};
var document = new Document();
documents = new List <Document> {
document
};
feedResponse = Mock.Of <IFeedResponse <Document> >();
Mock.Get(feedResponse)
.Setup(response => response.Count)
.Returns(documents.Count);
Mock.Get(feedResponse)
.Setup(response => response.ResponseContinuation)
.Returns("token");
Mock.Get(feedResponse)
.Setup(response => response.GetEnumerator())
.Returns(documents.GetEnumerator());
documentQuery = Mock.Of <IChangeFeedDocumentQuery <Document> >();
Mock.Get(documentQuery)
.Setup(query => query.HasMoreResults)
.Returns(false);
docClient = Mock.Of <IChangeFeedDocumentClient>();
Mock.Get(docClient)
.Setup(ex => ex.CreateDocumentChangeFeedQuery(processorSettings.CollectionSelfLink, It.IsAny <ChangeFeedOptions>()))
.Returns(documentQuery);
observer = Mock.Of <IChangeFeedObserver>();
Mock.Get(observer)
.Setup(feedObserver => feedObserver
.ProcessChangesAsync(It.IsAny <ChangeFeedObserverContext>(), It.IsAny <IReadOnlyList <Document> >(), It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(false))
.Callback(cancellationTokenSource.Cancel);
var checkPointer = new Mock <IPartitionCheckpointer>();
partitionProcessor = new PartitionProcessor(observer, docClient, processorSettings, checkPointer.Object);
}
public PartitionProcessorTests()
{
processorSettings = new ProcessorSettings
{
CollectionSelfLink = "selfLink",
FeedPollDelay = TimeSpan.FromMilliseconds(16),
MaxItemCount = 5,
PartitionKeyRangeId = "keyRangeId",
StartContinuation = "initialToken"
};
var document = new Document();
documents = new List <Document> {
document
};
feedResponse = Mock.Of <IFeedResponse <Document> >();
Mock.Get(feedResponse)
.Setup(response => response.Count)
.Returns(documents.Count);
Mock.Get(feedResponse)
.Setup(response => response.ResponseContinuation)
.Returns("token");
Mock.Get(feedResponse)
.Setup(response => response.GetEnumerator())
.Returns(documents.GetEnumerator());
documentQuery = Mock.Of <IChangeFeedDocumentQuery <Document> >();
Mock.Get(documentQuery)
.Setup(query => query.HasMoreResults)
.Returns(false);
Mock.Get(documentQuery)
.Setup(query => query.ExecuteNextAsync <Document>(It.Is <CancellationToken>(token => token == cancellationTokenSource.Token)))
.ReturnsAsync(() => feedResponse)
.Callback(() => cancellationTokenSource.Cancel());
observer = Mock.Of <IChangeFeedObserver>();
var checkPointer = new Mock <IPartitionCheckpointer>();
sut = new PartitionProcessor(
new ObserverExceptionWrappingChangeFeedObserverDecorator(observer),
documentQuery,
new ChangeFeedOptions(),
processorSettings,
checkPointer.Object);
}
public PartitionProcessor(IChangeFeedObserver observer, IChangeFeedDocumentClient documentClient, ProcessorSettings settings, IPartitionCheckpointer checkpointer)
{
this.observer = observer;
this.settings = settings;
this.checkpointer = checkpointer;
this.options = new ChangeFeedOptions
{
MaxItemCount = settings.MaxItemCount,
PartitionKeyRangeId = settings.PartitionKeyRangeId,
SessionToken = settings.SessionToken,
StartFromBeginning = settings.StartFromBeginning,
RequestContinuation = settings.RequestContinuation,
StartTime = settings.StartTime,
};
this.query = documentClient.CreateDocumentChangeFeedQuery(settings.CollectionSelfLink, this.options);
}
public PartitionControllerTests()
{
lease = Mock.Of <ILease>();
Mock.Get(lease)
.Setup(l => l.PartitionId)
.Returns("partitionId");
partitionProcessor = MockPartitionProcessor();
leaseRenewer = MockRenewer();
observer = MockObserver();
partitionSupervisorFactory = Mock.Of <IPartitionSupervisorFactory>(f => f.Create(lease) == new PartitionSupervisor(lease, observer, partitionProcessor, leaseRenewer));
leaseManager = Mock.Of <ILeaseManager>();
Mock.Get(leaseManager)
.Setup(manager => manager.AcquireAsync(lease))
.ReturnsAsync(lease);
Mock.Get(leaseManager)
.Setup(manager => manager.ReleaseAsync(lease))
.Returns(Task.FromResult(false));
synchronizer = Mock.Of <IPartitionSynchronizer>();
sut = new PartitionController(leaseManager, partitionSupervisorFactory, synchronizer);
}
public TestObserver(IChangeFeedObserver parent)
{
Debug.Assert(parent != null);
this.parent = parent;
}
public PartitionChainingTests()
{
processorSettings = new ProcessorSettings
{
CollectionSelfLink = "selfLink",
MaxItemCount = 5,
FeedPollDelay = TimeSpan.FromMilliseconds(16),
PartitionKeyRangeId = "keyRangeId",
RequestContinuation = "initialToken"
};
var document = new Document();
batch1 = new List <Document> {
document
};
document = new Document();
batch2 = new List <Document> {
document
};
feedResponse1 = Mock.Of <IFeedResponse <Document> >();
Mock.Get(feedResponse1)
.Setup(response => response.Count)
.Returns(batch1.Count);
Mock.Get(feedResponse1)
.SetupSequence(response => response.ResponseContinuation)
.Returns("token1");
Mock.Get(feedResponse1)
.SetupSequence(response => response.GetEnumerator())
.Returns(batch1.GetEnumerator());
feedResponse2 = Mock.Of <IFeedResponse <Document> >();
Mock.Get(feedResponse2)
.Setup(response => response.Count)
.Returns(batch2.Count);
Mock.Get(feedResponse2)
.SetupSequence(response => response.ResponseContinuation)
.Returns("token2");
Mock.Get(feedResponse2)
.SetupSequence(response => response.GetEnumerator())
.Returns(batch2.GetEnumerator());
documentQuery = Mock.Of <IChangeFeedDocumentQuery <Document> >();
Mock.Get(documentQuery)
.SetupSequence(query => query.HasMoreResults)
.Returns(true)
.Returns(false);
Mock.Get(documentQuery)
.SetupSequence(query => query.ExecuteNextAsync <Document>(It.Is <CancellationToken>(token => token == cancellationTokenSource.Token)))
.Returns(Task.FromResult(feedResponse1))
.Returns(Task.FromResult(feedResponse2));
docClient = Mock.Of <IChangeFeedDocumentClient>();
Mock.Get(docClient)
.Setup(ex => ex.CreateDocumentChangeFeedQuery(processorSettings.CollectionSelfLink, It.IsAny <ChangeFeedOptions>()))
.Returns(documentQuery);
observer = Mock.Of <IChangeFeedObserver>();
var checkPointer = new Mock <IPartitionCheckpointer>();
partitionProcessor = new PartitionProcessor(observer, docClient, processorSettings, checkPointer.Object);
var i = 0;
Mock.Get(observer)
.Setup(feedObserver => feedObserver
.ProcessChangesAsync(It.IsAny <ChangeFeedObserverContext>(), It.IsAny <IReadOnlyList <Document> >(), It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(false))
.Callback(() =>
{
if (++i == 2)
{
cancellationTokenSource.Cancel();
}
});
}
public ChangeFeedProcessorTests()
{
var leaseQueryMock = new Mock <IDocumentQuery <Document> >();
var leaseDocumentClient = Mock.Of <IChangeFeedDocumentClient>();
Mock.Get(leaseDocumentClient)
.Setup(c => c.CreateDocumentQuery <Document>(collectionLink,
It.Is <SqlQuerySpec>(spec => spec.QueryText == "SELECT * FROM c WHERE STARTSWITH(c.id, @PartitionLeasePrefix)" &&
spec.Parameters.Count == 1 &&
spec.Parameters[0].Name == "@PartitionLeasePrefix" &&
(string)spec.Parameters[0].Value == storeNamePrefix + ".."
), null))
.Returns(leaseQueryMock.As <IQueryable <Document> >().Object);
leaseQueryMock
.Setup(q => q.HasMoreResults)
.Returns(false);
Mock.Get(leaseDocumentClient)
.Setup(ex => ex.ReadDatabaseAsync(It.IsAny <Uri>(), It.IsAny <RequestOptions>()))
.ReturnsAsync(new ResourceResponse <Database>(database));
Mock.Get(leaseDocumentClient)
.Setup(ex => ex.ReadDocumentCollectionAsync(It.IsAny <Uri>(), It.IsAny <RequestOptions>()))
.ReturnsAsync(new ResourceResponse <DocumentCollection>(collection));
var documents = new List <Document> {
};
var feedResponse = Mock.Of <IFeedResponse <Document> >();
Mock.Get(feedResponse)
.Setup(response => response.Count)
.Returns(documents.Count);
Mock.Get(feedResponse)
.Setup(response => response.GetEnumerator())
.Returns(documents.GetEnumerator());
var documentQuery = Mock.Of <IChangeFeedDocumentQuery <Document> >();
Mock.Get(documentQuery)
.Setup(query => query.HasMoreResults)
.Returns(false);
Mock.Get(documentQuery)
.Setup(query => query.ExecuteNextAsync <Document>(It.IsAny <CancellationToken>()))
.ReturnsAsync(() => feedResponse)
.Callback(cancellationTokenSource.Cancel);
var documentClient = Mock.Of <IChangeFeedDocumentClient>();
Mock.Get(documentClient)
.Setup(ex => ex.CreateDocumentChangeFeedQuery(It.IsAny <string>(), It.IsAny <ChangeFeedOptions>()))
.Callback((string s, ChangeFeedOptions o) => this.createDocumentChangeFeedQueryCallback(s, o))
.Returns(documentQuery);
Mock.Get(documentClient)
.Setup(ex => ex.ReadDatabaseAsync(It.IsAny <Uri>(), It.IsAny <RequestOptions>()))
.ReturnsAsync(new ResourceResponse <Database>(database));
Mock.Get(documentClient)
.Setup(ex => ex.ReadDocumentCollectionAsync(It.IsAny <Uri>(), It.IsAny <RequestOptions>()))
.ReturnsAsync(new ResourceResponse <DocumentCollection>(collection));
this.lease = Mock.Of <ILease>();
Mock.Get(this.lease)
.Setup(l => l.PartitionId)
.Returns("partitionId");
var leaseStore = Mock.Of <ILeaseStore>();
this.leaseStoreManager = Mock.Of <ILeaseStoreManager>();
Mock.Get(this.leaseStoreManager)
.Setup(store => store.IsInitializedAsync())
.ReturnsAsync(true);
Mock.Get(this.leaseStoreManager)
.Setup(manager => manager.AcquireAsync(lease))
.ReturnsAsync(lease);
Mock.Get(this.leaseStoreManager)
.Setup(manager => manager.ReleaseAsync(lease))
.Returns(Task.CompletedTask);
this.builder
.WithHostName("someHost")
.WithFeedDocumentClient(documentClient)
.WithFeedCollection(collectionInfo)
.WithProcessorOptions(new ChangeFeedProcessorOptions())
.WithLeaseStoreManager(leaseStoreManager)
.WithLeaseCollection(collectionInfo)
.WithLeaseDocumentClient(leaseDocumentClient);
this.observer = Mock.Of <IChangeFeedObserver>();
Mock.Get(observer)
.Setup(feedObserver => feedObserver
.ProcessChangesAsync(It.IsAny <ChangeFeedObserverContext>(), It.IsAny <IReadOnlyList <Document> >(), It.IsAny <CancellationToken>()))
.Returns(Task.CompletedTask)
.Callback(cancellationTokenSource.Cancel);
Mock.Get(observer)
.Setup(observer => observer.OpenAsync(It.IsAny <ChangeFeedObserverContext>()))
.Returns(Task.CompletedTask);
this.observerFactory = Mock.Of <IChangeFeedObserverFactory>();
Mock.Get(observerFactory)
.Setup(observer => observer.CreateObserver())
.Returns(observer);
}
async Task IPartitionObserver <DocumentServiceLease> .OnPartitionAcquiredAsync(DocumentServiceLease lease)
{
Debug.Assert(lease != null && !string.IsNullOrEmpty(lease.Owner), "lease");
TraceLog.Informational(string.Format("Host '{0}' partition {1}: acquired!", this.HostName, lease.PartitionId));
#if DEBUG
Interlocked.Increment(ref this.partitionCount);
#endif
IChangeFeedObserver observer = this.observerFactory.CreateObserver();
ChangeFeedObserverContext context = new ChangeFeedObserverContext {
PartitionKeyRangeId = lease.PartitionId
};
CancellationTokenSource cancellation = new CancellationTokenSource();
// Create ChangeFeedOptions to use for this worker.
ChangeFeedOptions options = new ChangeFeedOptions
{
MaxItemCount = this.changeFeedOptions.MaxItemCount,
PartitionKeyRangeId = this.changeFeedOptions.PartitionKeyRangeId,
SessionToken = this.changeFeedOptions.SessionToken,
StartFromBeginning = this.changeFeedOptions.StartFromBeginning,
RequestContinuation = this.changeFeedOptions.RequestContinuation
};
var workerTask = await Task.Factory.StartNew(async() =>
{
ChangeFeedObserverCloseReason?closeReason = null;
try
{
try
{
await observer.OpenAsync(context);
}
catch (Exception ex)
{
TraceLog.Error(string.Format("IChangeFeedObserver.OpenAsync exception: {0}", ex));
closeReason = ChangeFeedObserverCloseReason.ObserverError;
throw;
}
options.PartitionKeyRangeId = lease.PartitionId;
if (!string.IsNullOrEmpty(lease.ContinuationToken))
{
options.RequestContinuation = lease.ContinuationToken;
}
CheckpointStats checkpointStats = null;
if (!this.statsSinceLastCheckpoint.TryGetValue(lease.PartitionId, out checkpointStats) || checkpointStats == null)
{
// It could be that the lease was created by different host and we picked it up.
checkpointStats = this.statsSinceLastCheckpoint.AddOrUpdate(
lease.PartitionId,
new CheckpointStats(),
(partitionId, existingStats) => existingStats);
Trace.TraceWarning(string.Format("Added stats for partition '{0}' for which the lease was picked up after the host was started.", lease.PartitionId));
}
IDocumentQuery <Document> query = this.documentClient.CreateDocumentChangeFeedQuery(this.collectionSelfLink, options);
TraceLog.Verbose(string.Format("Worker start: partition '{0}', continuation '{1}'", lease.PartitionId, lease.ContinuationToken));
string lastContinuation = options.RequestContinuation;
try
{
while (this.isShutdown == 0)
{
do
{
ExceptionDispatchInfo exceptionDispatchInfo = null;
FeedResponse <Document> response = null;
try
{
response = await query.ExecuteNextAsync <Document>();
lastContinuation = response.ResponseContinuation;
}
catch (DocumentClientException ex)
{
exceptionDispatchInfo = ExceptionDispatchInfo.Capture(ex);
}
if (exceptionDispatchInfo != null)
{
DocumentClientException dcex = (DocumentClientException)exceptionDispatchInfo.SourceException;
if (StatusCode.NotFound == (StatusCode)dcex.StatusCode && SubStatusCode.ReadSessionNotAvailable != (SubStatusCode)GetSubStatusCode(dcex))
{
// Most likely, the database or collection was removed while we were enumerating.
// Shut down. The user will need to start over.
// Note: this has to be a new task, can't await for shutdown here, as shudown awaits for all worker tasks.
TraceLog.Error(string.Format("Partition {0}: resource gone (subStatus={1}). Aborting.", context.PartitionKeyRangeId, GetSubStatusCode(dcex)));
await Task.Factory.StartNew(() => this.StopAsync(ChangeFeedObserverCloseReason.ResourceGone));
break;
}
else if (StatusCode.Gone == (StatusCode)dcex.StatusCode)
{
SubStatusCode subStatusCode = (SubStatusCode)GetSubStatusCode(dcex);
if (SubStatusCode.PartitionKeyRangeGone == subStatusCode)
{
bool isSuccess = await HandleSplitAsync(context.PartitionKeyRangeId, lastContinuation, lease.Id);
if (!isSuccess)
{
TraceLog.Error(string.Format("Partition {0}: HandleSplit failed! Aborting.", context.PartitionKeyRangeId));
await Task.Factory.StartNew(() => this.StopAsync(ChangeFeedObserverCloseReason.ResourceGone));
break;
}
// Throw LeaseLostException so that we take the lease down.
throw new LeaseLostException(lease, exceptionDispatchInfo.SourceException, true);
}
else if (SubStatusCode.Splitting == subStatusCode)
{
TraceLog.Warning(string.Format("Partition {0} is splitting. Will retry to read changes until split finishes. {1}", context.PartitionKeyRangeId, dcex.Message));
}
else
{
exceptionDispatchInfo.Throw();
}
}
else if (StatusCode.TooManyRequests == (StatusCode)dcex.StatusCode ||
StatusCode.ServiceUnavailable == (StatusCode)dcex.StatusCode)
{
TraceLog.Warning(string.Format("Partition {0}: retriable exception : {1}", context.PartitionKeyRangeId, dcex.Message));
}
else
{
exceptionDispatchInfo.Throw();
}
await Task.Delay(dcex.RetryAfter != TimeSpan.Zero ? dcex.RetryAfter : this.options.FeedPollDelay, cancellation.Token);
}
if (response != null)
{
if (response.Count > 0)
{
List <Document> docs = new List <Document>();
docs.AddRange(response);
try
{
context.FeedResponse = response;
await observer.ProcessChangesAsync(context, docs);
}
catch (Exception ex)
{
TraceLog.Error(string.Format("IChangeFeedObserver.ProcessChangesAsync exception: {0}", ex));
closeReason = ChangeFeedObserverCloseReason.ObserverError;
throw;
}
finally
{
context.FeedResponse = null;
}
}
checkpointStats.ProcessedDocCount += (uint)response.Count;
if (IsCheckpointNeeded(lease, checkpointStats))
{
lease = await CheckpointAsync(lease, response.ResponseContinuation, context);
checkpointStats.Reset();
}
else if (response.Count > 0)
{
TraceLog.Informational(string.Format("Checkpoint: not checkpointing for partition {0}, {1} docs, new continuation '{2}' as frequency condition is not met", lease.PartitionId, response.Count, response.ResponseContinuation));
}
}
}while (query.HasMoreResults && this.isShutdown == 0);
if (this.isShutdown == 0)
{
await Task.Delay(this.options.FeedPollDelay, cancellation.Token);
}
} // Outer while (this.isShutdown == 0) loop.
closeReason = ChangeFeedObserverCloseReason.Shutdown;
}
catch (TaskCanceledException)
{
Debug.Assert(cancellation.IsCancellationRequested, "cancellation.IsCancellationRequested");
TraceLog.Informational(string.Format("Cancel signal received for partition {0} worker!", context.PartitionKeyRangeId));
}
}
catch (LeaseLostException ex)
{
closeReason = ex.IsGone ? ChangeFeedObserverCloseReason.LeaseGone : ChangeFeedObserverCloseReason.LeaseLost;
}
catch (Exception ex)
{
TraceLog.Error(string.Format("Partition {0} exception: {1}", context.PartitionKeyRangeId, ex));
if (!closeReason.HasValue)
{
closeReason = ChangeFeedObserverCloseReason.Unknown;
}
}
if (closeReason.HasValue)
{
TraceLog.Informational(string.Format("Releasing lease for partition {0} due to an error, reason: {1}!", context.PartitionKeyRangeId, closeReason.Value));
// Note: this has to be a new task, because OnPartitionReleasedAsync awaits for worker task.
await Task.Factory.StartNew(async() => await this.partitionManager.TryReleasePartitionAsync(context.PartitionKeyRangeId, true, closeReason.Value));
}
TraceLog.Informational(string.Format("Partition {0}: worker finished!", context.PartitionKeyRangeId));
});
var newWorkerData = new WorkerData(workerTask, observer, context, cancellation);
this.partitionKeyRangeIdToWorkerMap.AddOrUpdate(context.PartitionKeyRangeId, newWorkerData, (string id, WorkerData d) => { return(newWorkerData); });
}
public ObserverExceptionWrappingChangeFeedObserverDecorator(IChangeFeedObserver changeFeedObserver)
{
this.changeFeedObserver = changeFeedObserver;
}
async Task IPartitionObserver <DocumentServiceLease> .OnPartitionAcquiredAsync(DocumentServiceLease lease)
{
Debug.Assert(lease != null && !string.IsNullOrEmpty(lease.Owner), "lease");
TraceLog.Informational(string.Format("Host '{0}' partition {1}: acquired!", this.HostName, lease.PartitionId));
#if DEBUG
Interlocked.Increment(ref this.partitionCount);
#endif
IChangeFeedObserver observer = this.observerFactory.CreateObserver();
ChangeFeedObserverContext context = new ChangeFeedObserverContext {
PartitionKeyRangeId = lease.PartitionId
};
CancellationTokenSource cancellation = new CancellationTokenSource();
// Create ChangeFeedOptions to use for this worker.
ChangeFeedOptions options = new ChangeFeedOptions
{
MaxItemCount = this.changeFeedOptions.MaxItemCount,
PartitionKeyRangeId = this.changeFeedOptions.PartitionKeyRangeId,
SessionToken = this.changeFeedOptions.SessionToken,
StartFromBeginning = this.changeFeedOptions.StartFromBeginning,
RequestContinuation = this.changeFeedOptions.RequestContinuation
};
var workerTask = await Task.Factory.StartNew(async() =>
{
ChangeFeedObserverCloseReason?closeReason = null;
try
{
try
{
await observer.OpenAsync(context);
}
catch (Exception ex)
{
TraceLog.Error(string.Format("IChangeFeedObserver.OpenAsync exception: {0}", ex));
closeReason = ChangeFeedObserverCloseReason.ObserverError;
throw;
}
options.PartitionKeyRangeId = lease.PartitionId;
if (!string.IsNullOrEmpty(lease.ContinuationToken))
{
options.RequestContinuation = lease.ContinuationToken;
}
IDocumentQuery <Document> query = this.documentClient.CreateDocumentChangeFeedQuery(this.collectionSelfLink, options);
TraceLog.Verbose(string.Format("Worker start: partition '{0}', continuation '{1}'", lease.PartitionId, lease.ContinuationToken));
try
{
while (this.isShutdown == 0)
{
do
{
DocumentClientException dcex = null;
FeedResponse <Document> response = null;
try
{
response = await query.ExecuteNextAsync <Document>();
}
catch (DocumentClientException ex)
{
if (StatusCode.NotFound != (StatusCode)ex.StatusCode &&
StatusCode.TooManyRequests != (StatusCode)ex.StatusCode &&
StatusCode.ServiceUnavailable != (StatusCode)ex.StatusCode)
{
throw;
}
dcex = ex;
}
if (dcex != null)
{
const int ReadSessionNotAvailable = 1002;
if (StatusCode.NotFound == (StatusCode)dcex.StatusCode && GetSubStatusCode(dcex) != ReadSessionNotAvailable)
{
// Most likely, the database or collection was removed while we were enumerating.
// Shut down. The user will need to start over.
// Note: this has to be a new task, can't await for shutdown here, as shudown awaits for all worker tasks.
await Task.Factory.StartNew(() => this.StopAsync(ChangeFeedObserverCloseReason.ResourceGone));
break;
}
else
{
Debug.Assert(StatusCode.TooManyRequests == (StatusCode)dcex.StatusCode || StatusCode.ServiceUnavailable == (StatusCode)dcex.StatusCode);
TraceLog.Warning(string.Format("Partition {0}: retriable exception : {1}", context.PartitionKeyRangeId, dcex.Message));
await Task.Delay(dcex.RetryAfter != TimeSpan.Zero ? dcex.RetryAfter : this.options.FeedPollDelay, cancellation.Token);
}
}
if (response != null)
{
if (response.Count > 0)
{
List <Document> docs = new List <Document>();
docs.AddRange(response);
try
{
await observer.ProcessChangesAsync(context, docs);
}
catch (Exception ex)
{
TraceLog.Error(string.Format("IChangeFeedObserver.ProcessChangesAsync exception: {0}", ex));
closeReason = ChangeFeedObserverCloseReason.ObserverError;
throw;
}
// Checkpoint after every successful delivery to the client.
lease = await CheckpointAsync(lease, response.ResponseContinuation, context);
}
else if (string.IsNullOrEmpty(lease.ContinuationToken))
{
// Checkpoint if we've never done that for this lease.
lease = await CheckpointAsync(lease, response.ResponseContinuation, context);
}
}
}while (query.HasMoreResults && this.isShutdown == 0);
if (this.isShutdown == 0)
{
await Task.Delay(this.options.FeedPollDelay, cancellation.Token);
}
} // Outer while (this.isShutdown == 0) loop.
}
catch (TaskCanceledException)
{
Debug.Assert(cancellation.IsCancellationRequested, "cancellation.IsCancellationRequested");
TraceLog.Informational(string.Format("Cancel signal received for partition {0} worker!", context.PartitionKeyRangeId));
}
}
catch (LeaseLostException)
{
closeReason = ChangeFeedObserverCloseReason.LeaseLost;
}
catch (Exception ex)
{
TraceLog.Error(string.Format("Partition {0} exception: {1}", context.PartitionKeyRangeId, ex));
if (!closeReason.HasValue)
{
closeReason = ChangeFeedObserverCloseReason.Unknown;
}
}
if (closeReason.HasValue)
{
TraceLog.Informational(string.Format("Releasing lease for partition {0} due to an error, reason: {1}!", context.PartitionKeyRangeId, closeReason.Value));
// Note: this has to be a new task, because OnPartitionReleasedAsync awaits for worker task.
await Task.Factory.StartNew(async() => await this.partitionManager.TryReleasePartitionAsync(context.PartitionKeyRangeId, true, closeReason.Value));
}
TraceLog.Informational(string.Format("Partition {0}: worker finished!", context.PartitionKeyRangeId));
});
var newWorkerData = new WorkerData(workerTask, observer, context, cancellation);
this.partitionKeyRangeIdToWorkerMap.AddOrUpdate(context.PartitionKeyRangeId, newWorkerData, (string id, WorkerData d) => { return(newWorkerData); });
}
public IPartitionProcessor Create(ILease lease, ILeaseCheckpointer leaseCheckpointer, IChangeFeedObserver observer)
{
if (observer == null)
{
throw new ArgumentNullException(nameof(observer));
}
if (lease == null)
{
throw new ArgumentNullException(nameof(lease));
}
if (leaseCheckpointer == null)
{
throw new ArgumentNullException(nameof(leaseCheckpointer));
}
var settings = new ProcessorSettings
{
CollectionSelfLink = this.collectionSelfLink,
StartContinuation = !string.IsNullOrEmpty(lease.ContinuationToken) ?
lease.ContinuationToken :
this.changeFeedProcessorOptions.StartContinuation,
PartitionKeyRangeId = lease.PartitionId,
FeedPollDelay = this.changeFeedProcessorOptions.FeedPollDelay,
MaxItemCount = this.changeFeedProcessorOptions.MaxItemCount,
StartFromBeginning = this.changeFeedProcessorOptions.StartFromBeginning,
StartTime = this.changeFeedProcessorOptions.StartTime,
SessionToken = this.changeFeedProcessorOptions.SessionToken,
};
var checkpointer = new PartitionCheckpointer(leaseCheckpointer, lease);
var changeFeedOptions = new ChangeFeedOptions
{
MaxItemCount = settings.MaxItemCount,
PartitionKeyRangeId = settings.PartitionKeyRangeId,
SessionToken = settings.SessionToken,
StartFromBeginning = settings.StartFromBeginning,
RequestContinuation = settings.StartContinuation,
StartTime = settings.StartTime,
};
var changeFeedQuery = this.documentClient.CreateDocumentChangeFeedQuery(settings.CollectionSelfLink, changeFeedOptions);
changeFeedQuery = new ChangeFeedQueryTimeoutDecorator(changeFeedQuery, this.healthMonitor, this.changeFeedProcessorOptions.ChangeFeedTimeout, lease);
return(new PartitionProcessor(observer, changeFeedQuery, changeFeedOptions, settings, checkpointer));
}
public IPartitionProcessor Create(ILease lease, ILeaseCheckpointer leaseCheckpointer, IChangeFeedObserver observer)
{
return(this.partitionProcessorFactory.Create(lease, observer));
}
