private static async Task <IChangeFeedProcessor> StartChangeFeedProcessorAsync(
string dataCollectionName,
DocumentClient client,
string dbName,
string serverUri)
{
var leasesCollectionName = dataCollectionName + ".leases";
await client.CreateDocumentCollectionIfNotExistsAsync(UriFactory.CreateDatabaseUri(dbName), new DocumentCollection { Id = leasesCollectionName });
IChangeFeedProcessor processor = await new ChangeFeedProcessorBuilder()
.WithObserver <ConsoleObserver>()
.WithHostName("console_app_host")
.WithFeedDocumentClient(client)
.WithFeedCollection(new DocumentCollectionInfo
{
CollectionName = dataCollectionName,
DatabaseName = dbName,
Uri = new Uri(serverUri)
})
.WithLeaseDocumentClient(client)
.WithLeaseCollection(new DocumentCollectionInfo
{
CollectionName = leasesCollectionName,
DatabaseName = dbName
})
.WithProcessorOptions(new ChangeFeedProcessorOptions()
{
StartFromBeginning = true
})
.BuildAsync();
await processor.StartAsync().ConfigureAwait(false);
return(processor);
}
public async Task StartAsync()
{
var feedCollectionInfo = new DocumentCollectionInfo
{
DatabaseName = _database,
CollectionName = _eventContainer,
Uri = new Uri(_endpointUri),
MasterKey = _authKey
};
var leaseCollectionInfo = new DocumentCollectionInfo
{
DatabaseName = _database,
CollectionName = _leaseContainer,
Uri = new Uri(_endpointUri),
MasterKey = _authKey
};
var viewRepository = new CosmosDBViewRepository(_endpointUri, _authKey, _database);
var builder = new ChangeFeedProcessorBuilder();
_changeFeedProcessor = await builder
.WithHostName("ProjectionHost")
.WithFeedCollection(feedCollectionInfo)
.WithLeaseCollection(leaseCollectionInfo)
.WithObserverFactory(new EventObserverFactory(_projections, viewRepository))
.WithProcessorOptions(new ChangeFeedProcessorOptions {
StartFromBeginning = true
})
.BuildAsync();
await _changeFeedProcessor.StartAsync();
}
internal virtual async Task StartProcessorAsync()
{
if (_host == null)
{
_host = await _hostBuilder.BuildAsync();
}
await _host.StartAsync();
}
public async Task Schema_DefaultsToNoLeaseToken()
{
TestObserverFactory observerFactory = new TestObserverFactory(
openProcessor: null,
(FeedProcessing.IChangeFeedObserverContext context, IReadOnlyList <Document> docs) =>
{
return(Task.CompletedTask);
});
IChangeFeedProcessor changeFeedProcessorBuilder = await new ChangeFeedProcessorBuilder()
.WithObserverFactory(observerFactory)
.WithHostName("smoke_test")
.WithFeedCollection(this.MonitoredCollectionInfo)
.WithLeaseCollection(this.LeaseCollectionInfo)
.BuildAsync();
await changeFeedProcessorBuilder.StartAsync();
await this.WaitUntilLeaseStoreIsInitializedAsync(new CancellationTokenSource(5000).Token);
await changeFeedProcessorBuilder.StopAsync();
// Verify that no leases have LeaseToken (V3 contract)
int leasesProcessed = 0;
using (DocumentClient client = new DocumentClient(this.LeaseCollectionInfo.Uri, this.LeaseCollectionInfo.MasterKey, this.LeaseCollectionInfo.ConnectionPolicy))
{
Uri collectionUri = UriFactory.CreateDocumentCollectionUri(this.LeaseCollectionInfo.DatabaseName, this.LeaseCollectionInfo.CollectionName);
IDocumentQuery <JObject> query = client.CreateDocumentQuery <JObject>(collectionUri, "SELECT * FROM c").AsDocumentQuery();
while (query.HasMoreResults)
{
foreach (JObject lease in await query.ExecuteNextAsync())
{
string leaseId = lease.Value <string>("id");
if (leaseId.Contains(".info"))
{
// These are the store initialization marks
continue;
}
Assert.NotNull(lease.Value <string>("PartitionId"));
Assert.Null(lease.Value <string>("LeaseToken"));
leasesProcessed++;
}
}
}
Assert.True(leasesProcessed > 0);
}
private async void OnStarted()
// private void OnStarted()
{
_logger.LogInformation("OnStarted has been called.");
// Perform post-startup activities here
_logger.LogInformation("Log directory is " + localLogdirectory);
string hostName = Guid.NewGuid().ToString();
DocumentCollectionInfo documentCollectionInfo = new DocumentCollectionInfo
{
Uri = new Uri(_configuration["az_cosmos_uri"]),
MasterKey = _configuration["az_cosmos_key"],
DatabaseName = _configuration["az_cosmos_db_name"],
CollectionName = _configuration["az_cosmos_collection_name"]
};
DocumentCollectionInfo leaseCollectionInfo = new DocumentCollectionInfo
{
Uri = new Uri(_configuration["az_cosmos_uri"]),
MasterKey = _configuration["az_cosmos_key"],
DatabaseName = _configuration["az_cosmos_db_name"],
CollectionName = _configuration["az_cosmos_lease_collection_name"]
};
DocumentFeedObserverFactory docObserverFactory = new DocumentFeedObserverFactory();
ChangeFeedOptions feedOptions = new ChangeFeedOptions();
feedOptions.StartFromBeginning = true;
ChangeFeedProcessorOptions feedProcessorOptions = new ChangeFeedProcessorOptions();
feedProcessorOptions.LeaseRenewInterval = TimeSpan.FromSeconds(15);
this.builder
.WithHostName(hostName)
.WithFeedCollection(documentCollectionInfo)
.WithLeaseCollection(leaseCollectionInfo)
.WithProcessorOptions(feedProcessorOptions)
.WithObserverFactory(new DocumentFeedObserverFactory()); //;
// .WithObserver<DocumentFeedObserver>(); //or just pass a observer
iChangeFeedProcessor = await this.builder.BuildAsync();
await iChangeFeedProcessor.StartAsync();
}
public override async Task StartAsync(CancellationToken cancellationToken)
{
var hostName = Environment.MachineName;
var builder = new ChangeFeedProcessorBuilder()
.WithHostName(hostName)
.WithFeedCollection(_feedCollection)
.WithLeaseCollection(_leaseCollection)
.WithObserverFactory(this);
_processor = await builder.BuildAsync();
await _processor.StartAsync();
_logger.LogInformation($"Change feed processor started at {hostName}...");
await base.StartAsync(cancellationToken);
}
public async Task Schema_OnV2MigrationMaintainLeaseToken()
{
const int batchSize = 10;
int partitionCount = await IntegrationTestsHelper.GetPartitionCount(this.MonitoredCollectionInfo);
int openedCount = 0;
ManualResetEvent allObserversStarted = new ManualResetEvent(false);
List <int> expectedIds = Enumerable.Range(0, batchSize * 2).ToList();
ManualResetEvent firstSetOfResultsProcessed = new ManualResetEvent(false);
ManualResetEvent secondSetOfResultsProcessed = new ManualResetEvent(false);
List <int> receivedIds = new List <int>();
TestObserverFactory observerFactory = new TestObserverFactory(
context =>
{
int newCount = Interlocked.Increment(ref openedCount);
if (newCount == partitionCount)
{
allObserversStarted.Set();
}
return(Task.CompletedTask);
},
(FeedProcessing.IChangeFeedObserverContext context, IReadOnlyList <Document> docs) =>
{
foreach (Document doc in docs)
{
receivedIds.Add(int.Parse(doc.Id));
}
if (receivedIds.Count == batchSize)
{
firstSetOfResultsProcessed.Set();
}
if (receivedIds.Count == batchSize * 2)
{
secondSetOfResultsProcessed.Set();
}
return(Task.CompletedTask);
});
IChangeFeedProcessor changeFeedProcessorBuilder = await new ChangeFeedProcessorBuilder()
.WithObserverFactory(observerFactory)
.WithHostName("smoke_test")
.WithFeedCollection(this.MonitoredCollectionInfo)
.WithLeaseCollection(this.LeaseCollectionInfo)
.BuildAsync();
await changeFeedProcessorBuilder.StartAsync();
await this.WaitUntilLeaseStoreIsInitializedAsync(new CancellationTokenSource(5000).Token);
// Inserting some documents
using (DocumentClient client = new DocumentClient(this.MonitoredCollectionInfo.Uri, this.MonitoredCollectionInfo.MasterKey, this.MonitoredCollectionInfo.ConnectionPolicy))
{
Uri collectionUri = UriFactory.CreateDocumentCollectionUri(this.MonitoredCollectionInfo.DatabaseName, this.MonitoredCollectionInfo.CollectionName);
foreach (int id in expectedIds.Take(10))
{
await client.CreateDocumentAsync(collectionUri, new { id = id.ToString() });
}
}
Assert.True(firstSetOfResultsProcessed.WaitOne(IntegrationTest.changeWaitTimeout), "Timed out waiting for first set of items to be received.");
await changeFeedProcessorBuilder.StopAsync();
// At this point we have leases for V2, so we will simulate V3 by manually adding LeaseToken and removing PartitionId
using (DocumentClient client = new DocumentClient(this.LeaseCollectionInfo.Uri, this.LeaseCollectionInfo.MasterKey, this.LeaseCollectionInfo.ConnectionPolicy))
{
Uri collectionUri = UriFactory.CreateDocumentCollectionUri(this.LeaseCollectionInfo.DatabaseName, this.LeaseCollectionInfo.CollectionName);
IDocumentQuery <JObject> query = client.CreateDocumentQuery <JObject>(collectionUri, "SELECT * FROM c").AsDocumentQuery();
while (query.HasMoreResults)
{
foreach (JObject lease in await query.ExecuteNextAsync())
{
string leaseId = lease.Value <string>("id");
if (leaseId.Contains(".info"))
{
// These are the store initialization marks
continue;
}
// create the LeaseToken property
lease.Add("LeaseToken", lease.Value <string>("PartitionId"));
lease.Remove("PartitionId");
await client.UpsertDocumentAsync(collectionUri, lease);
}
}
}
// Now all leases are V3 leases, start another processor that should migrate to V2 schema and maintain LeaseToken for compatibility
openedCount = 0;
allObserversStarted.Reset();
changeFeedProcessorBuilder = await new ChangeFeedProcessorBuilder()
.WithObserverFactory(observerFactory)
.WithHostName("smoke_test")
.WithFeedCollection(this.MonitoredCollectionInfo)
.WithLeaseCollection(this.LeaseCollectionInfo)
.BuildAsync();
await changeFeedProcessorBuilder.StartAsync();
Assert.True(allObserversStarted.WaitOne(IntegrationTest.changeWaitTimeout + IntegrationTest.changeWaitTimeout), "Timed out waiting for observres to start");
// Create the rest of the documents
using (DocumentClient client = new DocumentClient(this.MonitoredCollectionInfo.Uri, this.MonitoredCollectionInfo.MasterKey, this.MonitoredCollectionInfo.ConnectionPolicy))
{
Uri collectionUri = UriFactory.CreateDocumentCollectionUri(this.MonitoredCollectionInfo.DatabaseName, this.MonitoredCollectionInfo.CollectionName);
foreach (int id in expectedIds.TakeLast(10))
{
await client.CreateDocumentAsync(collectionUri, new { id = id.ToString() });
}
}
Assert.True(secondSetOfResultsProcessed.WaitOne(IntegrationTest.changeWaitTimeout), "Timed out waiting for second set of items to be received.");
await changeFeedProcessorBuilder.StopAsync();
// Verify we processed all items (including when using the V3 leases)
Assert.True(!expectedIds.Except(receivedIds).Any() && expectedIds.Count == expectedIds.Count);
// Verify the after-migration leases have both PartitionId and LeaseToken with the same value
using (DocumentClient client = new DocumentClient(this.LeaseCollectionInfo.Uri, this.LeaseCollectionInfo.MasterKey, this.LeaseCollectionInfo.ConnectionPolicy))
{
Uri collectionUri = UriFactory.CreateDocumentCollectionUri(this.LeaseCollectionInfo.DatabaseName, this.LeaseCollectionInfo.CollectionName);
IDocumentQuery <JObject> query = client.CreateDocumentQuery <JObject>(collectionUri, "SELECT * FROM c").AsDocumentQuery();
while (query.HasMoreResults)
{
foreach (JObject lease in await query.ExecuteNextAsync())
{
string leaseId = lease.Value <string>("id");
if (leaseId.Contains(".info"))
{
// These are the store initialization marks
continue;
}
Assert.NotNull(lease.Value <string>("PartitionId"));
Assert.Null(lease.Value <string>("LeaseToken"));
}
}
}
}
public async Task WriteToDocDB_SouldGeneratesIncomingEvent_IfStartingFromDefaultPoint()
{
string sessionId = Guid.NewGuid().ToString();
IntegrationTestsHelper.GetConfigurationSettings(out DocumentCollectionInfo feedCollectionInfo, out DocumentCollectionInfo leaseCollectionInfo, out int feedOfferThroughput, out int leaseOfferThroughput);
feedCollectionInfo.CollectionName = Guid.NewGuid().ToString();
leaseCollectionInfo.CollectionName = feedCollectionInfo.CollectionName + "-lease";
using (var feedClient = new DocumentClient(
feedCollectionInfo.Uri,
feedCollectionInfo.MasterKey,
feedCollectionInfo.ConnectionPolicy))
{
await IntegrationTestsHelper.CreateDocumentCollectionAsync(feedClient, feedCollectionInfo.DatabaseName, new DocumentCollection { Id = feedCollectionInfo.CollectionName }, 400);
using (var client = new DocumentClient(leaseCollectionInfo.Uri, leaseCollectionInfo.MasterKey, leaseCollectionInfo.ConnectionPolicy))
{
await IntegrationTestsHelper.CreateDocumentCollectionAsync(
client,
leaseCollectionInfo.DatabaseName,
new DocumentCollection { Id = leaseCollectionInfo.CollectionName },
400);
}
TaskCompletionSource <bool> documentReceived = new TaskCompletionSource <bool>();
Task Process(IChangeFeedObserverContext changeFeedObserverContext, IReadOnlyList <Document> readOnlyList)
{
if (readOnlyList.Any(d => d.GetPropertyValue <string>("payload") == sessionId))
{
documentReceived.SetResult(true);
}
return(Task.CompletedTask);
}
FeedProcessing.IChangeFeedObserverFactory observerFactory = new DelegatingMemoryObserverFactory(
ctx => Task.CompletedTask,
(ctx, reason) => Task.CompletedTask,
Process);
IChangeFeedProcessor changeFeedProcessor = await new ChangeFeedProcessorBuilder()
.WithObserverFactory(observerFactory)
.WithHostName("smoke_test")
.WithFeedCollection(feedCollectionInfo)
.WithLeaseCollection(leaseCollectionInfo)
.BuildAsync();
await changeFeedProcessor.StartAsync().ConfigureAwait(false);
try
{
var generateDocsTask = await Task.Factory.StartNew(async() =>
{
while (!documentReceived.Task.IsCompleted)
{
var document = new Document();
document.SetPropertyValue("payload", sessionId);
var collectionUri = UriFactory.CreateDocumentCollectionUri(
feedCollectionInfo.DatabaseName,
feedCollectionInfo.CollectionName);
await feedClient.CreateDocumentAsync(collectionUri, document);
await Task.Delay(TimeSpan.FromMilliseconds(100));
}
});
await documentReceived.Task;
await generateDocsTask;
}
finally
{
await changeFeedProcessor.StopAsync();
}
}
}
public async Task <IChangeFeedProcessor> RunChangeFeedHostAsync()
{
string hostName = Guid.NewGuid().ToString();
Trace.TraceInformation("Host name {0}", hostName);
// monitored collection info
var documentCollectionLocation = new DocumentCollectionInfo
{
Uri = new Uri(this.config.MonitoredUri),
MasterKey = this.config.MonitoredSecretKey,
DatabaseName = this.config.MonitoredDbName,
CollectionName = this.config.MonitoredCollectionName
};
var policy = new ConnectionPolicy()
{
ConnectionMode = ConnectionMode.Direct,
ConnectionProtocol = Protocol.Tcp
};
policy.PreferredLocations.Add("North Europe");
// lease collection info
var leaseCollectionLocation = new DocumentCollectionInfo
{
Uri = new Uri(this.config.LeaseUri),
MasterKey = this.config.LeaseSecretKey,
DatabaseName = this.config.LeaseDbName,
CollectionName = this.config.LeaseCollectionName,
ConnectionPolicy = policy
};
// destination collection info
var destCollInfo = new DocumentCollectionInfo
{
Uri = new Uri(this.config.DestUri),
MasterKey = this.config.DestSecretKey,
DatabaseName = this.config.DestDbName,
CollectionName = this.config.DestCollectionName
};
var processorOptions = new ChangeFeedProcessorOptions();
if (config.DataAgeInHours.HasValue)
{
if (config.DataAgeInHours.Value >= 0)
{
processorOptions.StartTime = DateTime.UtcNow.Subtract(TimeSpan.FromHours(config.DataAgeInHours.Value));
}
}
else
{
processorOptions.StartFromBeginning = true;
}
processorOptions.LeaseRenewInterval = TimeSpan.FromSeconds(30);
Trace.TraceInformation("Processor options Starts from Beginning - {0}, Lease renew interval - {1}",
processorOptions.StartFromBeginning,
processorOptions.LeaseRenewInterval.ToString());
processorOptions.MaxItemCount = 1000;
var destClient = new DocumentClient(destCollInfo.Uri, destCollInfo.MasterKey,
new ConnectionPolicy()
{
ConnectionMode = ConnectionMode.Direct, ConnectionProtocol = Protocol.Tcp
},
ConsistencyLevel.Eventual);
var docTransformer = new DefaultDocumentTransformer();
BlobContainerClient containerClient = null;
if (!String.IsNullOrEmpty(config.BlobConnectionString))
{
BlobServiceClient blobServiceClient = new BlobServiceClient(config.BlobConnectionString);
containerClient = blobServiceClient.GetBlobContainerClient(config.BlobContainerName);
await containerClient.CreateIfNotExistsAsync();
}
var docObserverFactory = new DocumentFeedObserverFactory(config.SourcePartitionKeys, config.TargetPartitionKey, destClient, destCollInfo, docTransformer, containerClient);
changeFeedProcessor = await new ChangeFeedProcessorBuilder()
.WithObserverFactory(docObserverFactory)
.WithHostName(hostName)
.WithFeedCollection(documentCollectionLocation)
.WithLeaseCollection(leaseCollectionLocation)
.WithProcessorOptions(processorOptions)
.WithFeedDocumentClient(new DocumentClient(documentCollectionLocation.Uri, documentCollectionLocation.MasterKey, policy, ConsistencyLevel.Eventual))
.BuildAsync();
await changeFeedProcessor.StartAsync().ConfigureAwait(false);
return(changeFeedProcessor);
}