public async Task CursorTest()
{
BlobServiceClient service = GetServiceClient_SharedKey();
BlobChangeFeedClient blobChangeFeedClient = service.GetChangeFeedClient();
AsyncPageable <BlobChangeFeedEvent> blobChangeFeedAsyncPagable
= blobChangeFeedClient.GetChangesAsync();
IAsyncEnumerable <Page <BlobChangeFeedEvent> > asyncEnumerable = blobChangeFeedAsyncPagable.AsPages(pageSizeHint: 500);
Page <BlobChangeFeedEvent> page = await asyncEnumerable.FirstAsync();
foreach (BlobChangeFeedEvent changeFeedEvent in page.Values)
{
Console.WriteLine(changeFeedEvent);
}
Console.WriteLine("break");
string continuation = page.ContinuationToken;
AsyncPageable <BlobChangeFeedEvent> cursorBlobChangeFeedAsyncPagable
= blobChangeFeedClient.GetChangesAsync(continuation);
IList <BlobChangeFeedEvent> list = await cursorBlobChangeFeedAsyncPagable.ToListAsync();
foreach (BlobChangeFeedEvent e in list)
{
Console.WriteLine(e);
}
}
public async Task <Dictionary <string, List <BlobChangeFeedEvent> > > ReadChangeFeedAsync(string cursor)
{
// create blob service client
BlobServiceClient blobServiceClient = new BlobServiceClient(this.connectionString);
// get change feed client
BlobChangeFeedClient blobChangeFeedClient = blobServiceClient.GetChangeFeedClient();
List <BlobChangeFeedEvent> blobChangeFeedEvents = new List <BlobChangeFeedEvent>();
IAsyncEnumerator <Page <BlobChangeFeedEvent> > enumerator = blobChangeFeedClient.GetChangesAsync(continuationToken: cursor)
.AsPages(pageSizeHint: 5)
.GetAsyncEnumerator();
await enumerator.MoveNextAsync();
Dictionary <string, List <BlobChangeFeedEvent> > keyValuePairs = new Dictionary <string, List <BlobChangeFeedEvent> >();
if (enumerator.Current != null && enumerator.Current.Values != null)
{
foreach (BlobChangeFeedEvent changeFeedEvent in enumerator.Current.Values)
{
blobChangeFeedEvents.Add(changeFeedEvent);
}
keyValuePairs.Add(enumerator.Current.ContinuationToken, blobChangeFeedEvents);
}
return(keyValuePairs);
}
// Gets a cursor from container or creates a new cursor if it does not exist
private static string GetCursor(BlobClient blobClient, BlobChangeFeedClient changeFeedClient, ILogger log)
{
string continuationToken = null;
if (blobClient.Exists())
{
// If the continuationToken exists in blob, download and use it
var stream = new MemoryStream();
blobClient.DownloadTo(stream);
continuationToken = Encoding.UTF8.GetString(stream.ToArray());
}
else
{
// If the continuationToken does not exist in the blob, get the continuationToken from the first item
Page <BlobChangeFeedEvent> page = changeFeedClient.GetChanges().AsPages(pageSizeHint: 1).First <Page <BlobChangeFeedEvent> >();
BlobChangeFeedEvent changeFeedEvent = page.Values.First <BlobChangeFeedEvent>();
if (containerCheck.Invoke(changeFeedEvent) && eventTypeCheck.Invoke(changeFeedEvent) && blobCheck.Invoke(changeFeedEvent))
{
log.LogInformation($"event: {changeFeedEvent.EventType} at {changeFeedEvent.Subject.Replace("/blobServices/default", "")} on {changeFeedEvent.EventTime}");
}
continuationToken = page.ContinuationToken;
}
return(continuationToken);
}
public async Task TestNonRoundedBoundaries()
{
// This is hardcoded for playback stability. Feel free to modify but make sure recordings match.
DateTimeOffset startTime = new DateTimeOffset(2020, 8, 5, 16, 24, 00, TimeSpan.Zero);
DateTimeOffset endTime = new DateTimeOffset(2020, 8, 5, 18, 35, 00, TimeSpan.Zero);
DateTimeOffset roundedStartTime = new DateTimeOffset(2020, 8, 5, 16, 00, 00, TimeSpan.Zero);
DateTimeOffset roundedEndTime = new DateTimeOffset(2020, 8, 5, 19, 00, 00, TimeSpan.Zero);
BlobServiceClient service = GetServiceClient_SharedKey();
BlobChangeFeedClient blobChangeFeedClient = service.GetChangeFeedClient();
// Collect all events within range
AsyncPageable <BlobChangeFeedEvent> blobChangeFeedAsyncPagable
= blobChangeFeedClient.GetChangesAsync(
start: startTime,
end: endTime);
var eventList = new List <BlobChangeFeedEvent>(await blobChangeFeedAsyncPagable.ToListAsync());
// Assert
Assert.Greater(eventList.Count, 1);
Assert.IsNull(eventList.Find(e => e.EventTime < roundedStartTime.AddMinutes(-15)), "No event 15 minutes before start is present");
Assert.IsNull(eventList.Find(e => e.EventTime > roundedEndTime.AddMinutes(15)), "No event 15 minutes after end is present");
Assert.IsNotNull(eventList.Find(e => e.EventTime < roundedStartTime.AddMinutes(15)), "There is some event 15 minutes after start");
Assert.IsNotNull(eventList.Find(e => e.EventTime > roundedEndTime.AddMinutes(-15)), "There is some event 15 minutes before end");
}
public void SharedKeyAuth()
{
// Get a Storage account name, shared key, and endpoint Uri.
//
// You can obtain both from the Azure Portal by clicking Access
// Keys under Settings in the Portal Storage account blade.
//
// You can also get access to your account keys from the Azure CLI
// with:
//
// az storage account keys list --account-name <account_name> --resource-group <resource_group>
//
string accountName = StorageAccountName;
string accountKey = StorageAccountKey;
Uri serviceUri = StorageAccountBlobUri;
// Create a SharedKeyCredential that we can use to authenticate
StorageSharedKeyCredential credential = new StorageSharedKeyCredential(accountName, accountKey);
// Create a client that can authenticate with a connection string
BlobChangeFeedClient service = new BlobChangeFeedClient(serviceUri, credential);
List <BlobChangeFeedEvent> changeFeedEvents = new List <BlobChangeFeedEvent>();
// Make a service request to verify we've successfully authenticated
foreach (BlobChangeFeedEvent changeFeedEvent in service.GetChanges())
{
changeFeedEvents.Add(changeFeedEvent);
}
}
private async Task <List <BlobChangeFeedEvent> > ProcessChangeFeedTask
(string BlobStorageAccountName, DateTimeOffset start, DateTimeOffset end)
{
// Get a new blob service client.
string BlobUrl = $"https://{BlobStorageAccountName}.blob.core.windows.net/";
// Get a credential and create a client object for the blob container. Note using new Azure Core credential flow
BlobServiceClient blobServiceClient = new BlobServiceClient(new Uri(BlobUrl),
new DefaultAzureCredential());
// Get a new change feed client.
BlobChangeFeedClient changeFeedClient = blobServiceClient.GetChangeFeedClient();
List <BlobChangeFeedEvent> changeFeedEvents = new List <BlobChangeFeedEvent>();
IAsyncEnumerator <Page <BlobChangeFeedEvent> > enumerator = changeFeedClient
.GetChangesAsync(start, end)
.AsPages(pageSizeHint: 10)
.GetAsyncEnumerator();
await enumerator.MoveNextAsync();
foreach (BlobChangeFeedEvent changeFeedEvent in enumerator.Current.Values)
{
changeFeedEvents.Add(changeFeedEvent);
}
return(changeFeedEvents);
}
public void ChangeFeedBetweenDates()
{
// Get a connection string to our Azure Storage account.
string connectionString = ConnectionString;
// Get a new blob service client.
BlobServiceClient blobServiceClient = new BlobServiceClient(connectionString);
// Get a new change feed client.
BlobChangeFeedClient changeFeedClient = blobServiceClient.GetChangeFeedClient();
List <BlobChangeFeedEvent> changeFeedEvents = new List <BlobChangeFeedEvent>();
// Create the start and end time. The change feed client will round start time down to
// the nearest hour, and round endTime up to the next hour if you provide DateTimeOffsets
// with minutes and seconds.
DateTimeOffset startTime = new DateTimeOffset(2017, 3, 2, 15, 0, 0, TimeSpan.Zero);
DateTimeOffset endTime = new DateTimeOffset(2020, 10, 7, 2, 0, 0, TimeSpan.Zero);
// You can also provide just a start or end time.
foreach (BlobChangeFeedEvent changeFeedEvent in changeFeedClient.GetChanges(
start: startTime,
end: endTime))
{
changeFeedEvents.Add(changeFeedEvent);
}
}
public void ActiveDirectoryAuth()
{
// Create a token credential that can use our Azure Active
// Directory application to authenticate with Azure Storage
TokenCredential credential =
new ClientSecretCredential(
ActiveDirectoryTenantId,
ActiveDirectoryApplicationId,
ActiveDirectoryApplicationSecret,
new TokenCredentialOptions()
{
AuthorityHost = ActiveDirectoryAuthEndpoint
});
// Create a client that can authenticate using our token credential
BlobChangeFeedClient service = new BlobChangeFeedClient(ActiveDirectoryBlobUri, credential);
List <BlobChangeFeedEvent> changeFeedEvents = new List <BlobChangeFeedEvent>();
// Make a service request to verify we've successfully authenticated
foreach (BlobChangeFeedEvent changeFeedEvent in service.GetChanges())
{
changeFeedEvents.Add(changeFeedEvent);
}
}
public async Task ChangeFeedResumeWithCursorAsync()
{
// Get a connection string to our Azure Storage account.
string connectionString = ConnectionString;
// Get a new change feed client.
BlobChangeFeedClient changeFeedClient = new BlobChangeFeedClient(connectionString);
List <BlobChangeFeedEvent> changeFeedEvents = new List <BlobChangeFeedEvent>();
#region Snippet:SampleSnippetsChangeFeed_ResumeWithCursor
string continuationToken = null;
await foreach (Page <BlobChangeFeedEvent> page in changeFeedClient.GetChangesAsync().AsPages(pageSizeHint: 10))
{
foreach (BlobChangeFeedEvent changeFeedEvent in page.Values)
{
changeFeedEvents.Add(changeFeedEvent);
}
// Get the change feed continuation token. The continuation token is not required to get each page of events,
// it is intended to be saved and used to resume iterating at a later date.
continuationToken = page.ContinuationToken;
break;
}
// Resume iterating from the pervious position with the continuation token.
await foreach (BlobChangeFeedEvent changeFeedEvent in changeFeedClient.GetChangesAsync(
continuationToken: continuationToken))
{
changeFeedEvents.Add(changeFeedEvent);
}
#endregion
}
public async Task ChangeFeedBetweenDatesAsync()
{
// Get a connection string to our Azure Storage account.
string connectionString = ConnectionString;
// Get a new change feed client.
BlobChangeFeedClient changeFeedClient = new BlobChangeFeedClient(connectionString);
List <BlobChangeFeedEvent> changeFeedEvents = new List <BlobChangeFeedEvent>();
#region Snippet:SampleSnippetsChangeFeed_GetEventsBetweenStartAndEndTime
// Create the start and end time. The change feed client will round start time down to
// the nearest hour, and round endTime up to the next hour if you provide DateTimeOffsets
// with minutes and seconds.
DateTimeOffset startTime = new DateTimeOffset(2017, 3, 2, 15, 0, 0, TimeSpan.Zero);
DateTimeOffset endTime = new DateTimeOffset(2020, 10, 7, 2, 0, 0, TimeSpan.Zero);
// You can also provide just a start or end time.
await foreach (BlobChangeFeedEvent changeFeedEvent in changeFeedClient.GetChangesAsync(
start: startTime,
end: endTime))
{
changeFeedEvents.Add(changeFeedEvent);
}
#endregion
}
public async Task CursorFormatTest()
{
// This is hardcoded for playback stability. Feel free to modify but make sure recordings match.
DateTimeOffset startTime = new DateTimeOffset(2020, 7, 30, 23, 00, 00, TimeSpan.Zero);
DateTimeOffset endTime = new DateTimeOffset(2020, 7, 30, 23, 15, 00, TimeSpan.Zero);
BlobServiceClient service = GetServiceClient_SharedKey();
BlobChangeFeedClient blobChangeFeedClient = service.GetChangeFeedClient();
// Iterate over first two pages
var blobChangeFeedAsyncPagable = blobChangeFeedClient.GetChangesAsync(
start: startTime,
end: endTime);
IAsyncEnumerable <Page <BlobChangeFeedEvent> > asyncEnumerable = blobChangeFeedAsyncPagable.AsPages(pageSizeHint: 50);
Page <BlobChangeFeedEvent> lastPage = null;
int pages = 0;
await foreach (Page <BlobChangeFeedEvent> page in asyncEnumerable)
{
foreach (BlobChangeFeedEvent e in page.Values)
{
Console.WriteLine(e);
}
pages++;
lastPage = page;
if (pages > 2)
{
break;
}
}
// Act
string continuation = lastPage.ContinuationToken;
// Verify
// You may need to update expected values when re-recording
var cursor = (JsonSerializer.Deserialize(continuation, typeof(ChangeFeedCursor)) as ChangeFeedCursor);
Assert.AreEqual(new DateTimeOffset(2020, 7, 31, 00, 00, 00, TimeSpan.Zero), cursor.EndTime);
Assert.AreEqual(1, cursor.CursorVersion);
Assert.AreEqual("emilydevtest.blob.core.windows.net", cursor.UrlHost);
var currentSegmentCursor = cursor.CurrentSegmentCursor;
Assert.AreEqual("idx/segments/2020/07/30/2300/meta.json", currentSegmentCursor.SegmentPath);
Assert.AreEqual("log/00/2020/07/30/2300/", currentSegmentCursor.CurrentShardPath);
Assert.AreEqual(1, currentSegmentCursor.ShardCursors.Count);
var shardCursor = currentSegmentCursor.ShardCursors.First();
Assert.AreEqual("log/00/2020/07/30/2300/00000.avro", shardCursor.CurrentChunkPath);
Assert.AreEqual(96253, shardCursor.BlockOffset);
Assert.AreEqual(0, shardCursor.EventIndex);
}
public void Test()
{
BlobServiceClient service = GetServiceClient_SharedKey();
BlobChangeFeedClient blobChangeFeedClient = service.GetChangeFeedClient();
Pageable <BlobChangeFeedEvent> blobChangeFeedPagable
= blobChangeFeedClient.GetChanges();
IList <BlobChangeFeedEvent> list = blobChangeFeedPagable.ToList();
foreach (BlobChangeFeedEvent e in list)
{
Console.WriteLine(e);
}
}
public async Task TestLastHour()
{
BlobServiceClient service = GetServiceClient_SharedKey();
BlobChangeFeedClient blobChangeFeedClient = service.GetChangeFeedClient();
AsyncPageable <BlobChangeFeedEvent> blobChangeFeedAsyncPagable
= blobChangeFeedClient.GetChangesAsync(start: DateTime.Now, end: DateTime.Now);
IList <BlobChangeFeedEvent> list = await blobChangeFeedAsyncPagable.ToListAsync();
foreach (BlobChangeFeedEvent e in list)
{
Console.WriteLine(e);
}
}
public async Task CanReadTillEnd()
{
// Uncomment when recording.
//DateTimeOffset startTime = DateTimeOffset.Now;
// Update and uncomment after recording.
DateTimeOffset startTime = new DateTimeOffset(2020, 7, 31, 19, 00, 00, TimeSpan.Zero);
BlobServiceClient service = GetServiceClient_SharedKey();
BlobChangeFeedClient blobChangeFeedClient = service.GetChangeFeedClient();
AsyncPageable <BlobChangeFeedEvent> blobChangeFeedAsyncPagable
= blobChangeFeedClient.GetChangesAsync(startTime);
IList <BlobChangeFeedEvent> list = await blobChangeFeedAsyncPagable.ToListAsync();
CollectionAssert.IsNotEmpty(list);
}
public void ChangeFeed()
{
// Get a connection string to our Azure Storage account.
string connectionString = ConnectionString;
// Get a new change feed client.
BlobChangeFeedClient changeFeedClient = new BlobChangeFeedClient(connectionString);
// Get all the events in the change feed.
List <BlobChangeFeedEvent> changeFeedEvents = new List <BlobChangeFeedEvent>();
foreach (BlobChangeFeedEvent changeFeedEvent in changeFeedClient.GetChanges())
{
changeFeedEvents.Add(changeFeedEvent);
}
}
public async Task ChangeFeedAsync()
{
// Get a connection string to our Azure Storage account.
string connectionString = ConnectionString;
// Get a new change feed client.
BlobChangeFeedClient changeFeedClient = new BlobChangeFeedClient(connectionString);
#region Snippet:SampleSnippetsChangeFeed_GetAllEvents
// Get all the events in the change feed.
List <BlobChangeFeedEvent> changeFeedEvents = new List <BlobChangeFeedEvent>();
await foreach (BlobChangeFeedEvent changeFeedEvent in changeFeedClient.GetChangesAsync())
{
changeFeedEvents.Add(changeFeedEvent);
}
#endregion
}
public async Task ChangeFeedPollForEventsWithCursor()
{
// Get a connection string to our Azure Storage account.
string connectionString = ConnectionString;
// Get a new change feed client.
BlobChangeFeedClient changeFeedClient = new BlobChangeFeedClient(connectionString);
List <BlobChangeFeedEvent> changeFeedEvents = new List <BlobChangeFeedEvent>();
#region Snippet:SampleSnippetsChangeFeed_PollForEventsWithCursor
// Create the start time. The change feed client will round start time down to
// the nearest hour if you provide DateTimeOffsets
// with minutes and seconds.
DateTimeOffset startTime = DateTimeOffset.Now;
// Create polling interval.
TimeSpan pollingInterval = TimeSpan.FromMinutes(5);
// Get initial set of events.
IAsyncEnumerable <Page <BlobChangeFeedEvent> > pages = changeFeedClient.GetChangesAsync(start: startTime).AsPages();
string continuationToken = null;
while (true)
{
await foreach (Page <BlobChangeFeedEvent> page in pages)
{
foreach (BlobChangeFeedEvent changeFeedEvent in page.Values)
{
changeFeedEvents.Add(changeFeedEvent);
}
// Get the change feed continuation token. The continuation token is not required to get each page of events,
// it is intended to be saved and used to resume iterating at a later date.
// For the purpose of actively listening to events the continuation token from last page is used.
continuationToken = page.ContinuationToken;
}
// Wait before processing next batch of events.
await Task.Delay(pollingInterval);
// Resume from last continuation token and fetch latest set of events.
pages = changeFeedClient.GetChangesAsync(continuationToken).AsPages();
}
#endregion
}
public async Task PageSizeTest()
{
int pageSize = 100;
BlobServiceClient service = GetServiceClient_SharedKey();
BlobChangeFeedClient blobChangeFeedClient = service.GetChangeFeedClient();
IAsyncEnumerator <Page <BlobChangeFeedEvent> > asyncEnumerator
= blobChangeFeedClient.GetChangesAsync().AsPages(pageSizeHint: pageSize).GetAsyncEnumerator();
List <int> pageSizes = new List <int>();
while (await asyncEnumerator.MoveNextAsync())
{
pageSizes.Add(asyncEnumerator.Current.Values.Count);
}
// All pages except the last should have a count == pageSize.
for (int i = 0; i < pageSizes.Count - 1; i++)
{
Assert.AreEqual(pageSize, pageSizes[i]);
}
}
public async Task ImmediateResumeFromEndOfCurrentHourYieldsEmptyResult()
{
// Uncomment when recording.
//DateTimeOffset startTime = DateTimeOffset.Now;
// Update and uncomment after recording.
DateTimeOffset startTime = new DateTimeOffset(2020, 8, 11, 21, 00, 00, TimeSpan.Zero);
BlobServiceClient service = GetServiceClient_SharedKey();
BlobChangeFeedClient blobChangeFeedClient = service.GetChangeFeedClient();
// Collect all events within range
AsyncPageable <BlobChangeFeedEvent> blobChangeFeedAsyncPagable
= blobChangeFeedClient.GetChangesAsync(
start: startTime);
ISet <string> AllEventIds = new HashSet <string>();
IAsyncEnumerable <Page <BlobChangeFeedEvent> > asyncEnumerable = blobChangeFeedAsyncPagable.AsPages(pageSizeHint: 50);
Page <BlobChangeFeedEvent> lastPage = null;
await foreach (Page <BlobChangeFeedEvent> page in asyncEnumerable)
{
foreach (BlobChangeFeedEvent e in page.Values)
{
AllEventIds.Add(e.Id.ToString());
}
lastPage = page;
}
string continuation = lastPage.ContinuationToken;
// Act
blobChangeFeedAsyncPagable = blobChangeFeedClient.GetChangesAsync(continuation);
IList <BlobChangeFeedEvent> tail = await blobChangeFeedAsyncPagable.ToListAsync();
// Assert
Assert.AreEqual(0, tail.Count);
Assert.Greater(AllEventIds.Count, 0);
}
public void SharedAccessSignatureAuth()
{
// Create a blob level SAS that allows reading change events from blob
// level APIs
AccountSasBuilder sas = new AccountSasBuilder
{
// Allow access to blobs
Services = AccountSasServices.Blobs,
// Allow access to the service level APIs
ResourceTypes = AccountSasResourceTypes.All,
// Access expires in 1 hour!
ExpiresOn = DateTimeOffset.UtcNow.AddHours(1)
};
// Allow all access
sas.SetPermissions(AccountSasPermissions.All);
// Create a SharedKeyCredential that we can use to sign the SAS token
StorageSharedKeyCredential credential = new StorageSharedKeyCredential(StorageAccountName, StorageAccountKey);
// Build a SAS URI
UriBuilder sasUri = new UriBuilder(StorageAccountBlobUri);
sasUri.Query = sas.ToSasQueryParameters(credential).ToString();
// Create a client that can authenticate with the SAS URI
BlobChangeFeedClient service = new BlobChangeFeedClient(sasUri.Uri);
List <BlobChangeFeedEvent> changeFeedEvents = new List <BlobChangeFeedEvent>();
// Make a service request to verify we've successfully authenticated
foreach (BlobChangeFeedEvent changeFeedEvent in service.GetChanges())
{
changeFeedEvents.Add(changeFeedEvent);
}
}
public async Task ResumeFromEndInThePastYieldsEmptyResult()
{
// This is hardcoded for playback stability. Feel free to modify but make sure recordings match.
DateTimeOffset startTime = new DateTimeOffset(2020, 7, 30, 23, 00, 00, TimeSpan.Zero);
DateTimeOffset endTime = new DateTimeOffset(2020, 7, 30, 23, 15, 00, TimeSpan.Zero);
BlobServiceClient service = GetServiceClient_SharedKey();
BlobChangeFeedClient blobChangeFeedClient = service.GetChangeFeedClient();
// Collect all events within range
AsyncPageable <BlobChangeFeedEvent> blobChangeFeedAsyncPagable
= blobChangeFeedClient.GetChangesAsync(
start: startTime,
end: endTime);
ISet <string> AllEventIds = new HashSet <string>();
IAsyncEnumerable <Page <BlobChangeFeedEvent> > asyncEnumerable = blobChangeFeedAsyncPagable.AsPages(pageSizeHint: 50);
Page <BlobChangeFeedEvent> lastPage = null;
await foreach (Page <BlobChangeFeedEvent> page in asyncEnumerable)
{
foreach (BlobChangeFeedEvent e in page.Values)
{
AllEventIds.Add(e.Id.ToString());
}
lastPage = page;
}
string continuation = lastPage.ContinuationToken;
// Act
blobChangeFeedAsyncPagable = blobChangeFeedClient.GetChangesAsync(continuation);
IList <BlobChangeFeedEvent> tail = await blobChangeFeedAsyncPagable.ToListAsync();
// Assert
Assert.AreEqual(0, tail.Count);
Assert.Greater(AllEventIds.Count, 0);
}
public void ChangeFeedResumeWithCursor()
{
// Get a connection string to our Azure Storage account.
string connectionString = ConnectionString;
// Get a new blob service client.
BlobServiceClient blobServiceClient = new BlobServiceClient(connectionString);
// Get a new change feed client.
BlobChangeFeedClient changeFeedClient = blobServiceClient.GetChangeFeedClient();
List <BlobChangeFeedEvent> changeFeedEvents = new List <BlobChangeFeedEvent>();
IEnumerator <Page <BlobChangeFeedEvent> > enumerator = changeFeedClient
.GetChanges()
.AsPages(pageSizeHint: 10)
.GetEnumerator();
;
enumerator.MoveNext();
foreach (BlobChangeFeedEvent changeFeedEvent in enumerator.Current.Values)
{
changeFeedEvents.Add(changeFeedEvent);
}
// get the change feed cursor. The cursor is not required to get each page of events,
// it is intended to be saved and used to resume iterating at a later date.
string cursor = enumerator.Current.ContinuationToken;
// Resume iterating from the pervious position with the cursor.
foreach (BlobChangeFeedEvent changeFeedEvent in changeFeedClient.GetChanges(
continuationToken: cursor))
{
changeFeedEvents.Add(changeFeedEvent);
}
}
public static void Run([TimerTrigger("0 */30 * * * *")] TimerInfo myTimer, ILogger log)
{
// Create client for Changefeed
BlobServiceClient blobServiceClient = new BlobServiceClient(connectionString);
BlobChangeFeedClient changeFeedClient = blobServiceClient.GetChangeFeedClient();
try
{
BlobContainerClient containerClient = blobServiceClient.CreateBlobContainer("cursorstoragecontainer");
}
catch
{
BlobContainerClient containerClient = blobServiceClient.GetBlobContainerClient("cursorstoragecontainer");
}
BlobClient blobClient = new BlobClient(connectionString, "cursorstoragecontainer", "cursorBlob");
// Create and get cursor for Changefeed
string continuationToken = GetCursor(blobClient, changeFeedClient, log);
// Loop through the events in Changefeed with the continuationToken
foreach (Page <BlobChangeFeedEvent> page in changeFeedClient.GetChanges(continuationToken: continuationToken).AsPages())
{
foreach (BlobChangeFeedEvent changeFeedEvent in page.Values)
{
if (containerCheck.Invoke(changeFeedEvent) && eventTypeCheck.Invoke(changeFeedEvent) && blobCheck.Invoke(changeFeedEvent))
{
log.LogInformation($"event: {changeFeedEvent.EventType} at {changeFeedEvent.Subject.Replace("/blobServices/default","")} on {changeFeedEvent.EventTime}");
}
}
continuationToken = page.ContinuationToken;
}
// Upload new continuationToken to blob
var upStream = new MemoryStream(Encoding.ASCII.GetBytes(continuationToken));
blobClient.Upload(upStream, true);
}
public void ConnectionStringAuth()
{
// Get a connection string to our Azure Storage account. You can
// obtain your connection string from the Azure Portal (click
// Access Keys under Settings in the Portal Storage account blade)
// or using the Azure CLI with:
//
// az storage account show-connection-string --name <account_name> --resource-group <resource_group>
//
// And you can provide the connection string to your application
// using an environment variable.
string connectionString = ConnectionString;
// Create a client that can authenticate with a connection string
BlobChangeFeedClient service = new BlobChangeFeedClient(connectionString);
List <BlobChangeFeedEvent> changeFeedEvents = new List <BlobChangeFeedEvent>();
// Make a service request to verify we've successfully authenticated
foreach (BlobChangeFeedEvent changeFeedEvent in service.GetChanges())
{
changeFeedEvents.Add(changeFeedEvent);
}
}
public async Task ResumeFromTheMiddleOfTheChunkWithManyNonEmptyShards()
{
// This is hardcoded for playback stability. Feel free to modify but make sure recordings match.
DateTimeOffset startTime = new DateTimeOffset(2020, 8, 5, 17, 00, 00, TimeSpan.Zero);
DateTimeOffset endTime = new DateTimeOffset(2020, 8, 5, 17, 15, 00, TimeSpan.Zero);
int expectedShardCount = 3;
BlobServiceClient service = GetServiceClient_SharedKey();
BlobChangeFeedClient blobChangeFeedClient = service.GetChangeFeedClient();
// Collect all events within range
AsyncPageable <BlobChangeFeedEvent> blobChangeFeedAsyncPagable
= blobChangeFeedClient.GetChangesAsync(
start: startTime,
end: endTime);
ISet <string> AllEventIds = new HashSet <string>();
await foreach (BlobChangeFeedEvent e in blobChangeFeedAsyncPagable)
{
AllEventIds.Add(e.Id.ToString());
}
// Iterate over first two pages
ISet <string> EventIdsPart1 = new HashSet <string>();
blobChangeFeedAsyncPagable = blobChangeFeedClient.GetChangesAsync(
start: startTime,
end: endTime);
IAsyncEnumerable <Page <BlobChangeFeedEvent> > asyncEnumerable = blobChangeFeedAsyncPagable.AsPages(pageSizeHint: 50);
Page <BlobChangeFeedEvent> lastPage = null;
int pages = 0;
await foreach (Page <BlobChangeFeedEvent> page in asyncEnumerable)
{
foreach (BlobChangeFeedEvent e in page.Values)
{
EventIdsPart1.Add(e.Id.ToString());
}
pages++;
lastPage = page;
if (pages > 2)
{
break;
}
}
string continuation = lastPage.ContinuationToken;
var currentSegmentCursor = (JsonSerializer.Deserialize(continuation, typeof(ChangeFeedCursor)) as ChangeFeedCursor).CurrentSegmentCursor;
Assert.AreEqual(currentSegmentCursor.ShardCursors.Count, expectedShardCount);
Assert.IsNotNull(currentSegmentCursor.ShardCursors.Find(x => x.BlockOffset > 0), "Making sure we actually finish some shard in the middle of chunk, if this fails play with test data to make it pass");
// Iterate over next two pages
ISet <string> EventIdsPart2 = new HashSet <string>();
blobChangeFeedAsyncPagable = blobChangeFeedClient.GetChangesAsync(continuation);
asyncEnumerable = blobChangeFeedAsyncPagable.AsPages(pageSizeHint: 50);
lastPage = null;
pages = 0;
await foreach (Page <BlobChangeFeedEvent> page in asyncEnumerable)
{
foreach (BlobChangeFeedEvent e in page.Values)
{
EventIdsPart2.Add(e.Id.ToString());
}
pages++;
lastPage = page;
if (pages > 2)
{
break;
}
}
continuation = lastPage.ContinuationToken;
currentSegmentCursor = (JsonSerializer.Deserialize(continuation, typeof(ChangeFeedCursor)) as ChangeFeedCursor).CurrentSegmentCursor;
Assert.AreEqual(currentSegmentCursor.ShardCursors.Count, expectedShardCount);
Assert.IsNotNull(currentSegmentCursor.ShardCursors.Find(x => x.BlockOffset > 0), "Making sure we actually finish some shard in the middle of chunk, if this fails play with test data to make it pass");
// Iterate over remaining
ISet <string> EventIdsTail = new HashSet <string>();
AsyncPageable <BlobChangeFeedEvent> cursorBlobChangeFeedAsyncPagable
= blobChangeFeedClient.GetChangesAsync(continuation);
IList <BlobChangeFeedEvent> list = await cursorBlobChangeFeedAsyncPagable.ToListAsync();
foreach (BlobChangeFeedEvent e in list)
{
EventIdsTail.Add(e.Id.ToString());
}
ISet <string> AllEventIdsFromResumingIteration = new HashSet <string>();
AllEventIdsFromResumingIteration.UnionWith(EventIdsPart1);
AllEventIdsFromResumingIteration.UnionWith(EventIdsPart2);
AllEventIdsFromResumingIteration.UnionWith(EventIdsTail);
Assert.Greater(AllEventIds.Count, 0);
Assert.Greater(EventIdsPart1.Count, 0);
Assert.Greater(EventIdsPart2.Count, 0);
Assert.Greater(EventIdsTail.Count, 0);
Assert.AreEqual(AllEventIds.Count, EventIdsPart1.Count + EventIdsPart2.Count + EventIdsTail.Count);
CollectionAssert.AreEqual(AllEventIds, AllEventIdsFromResumingIteration);
}
public async Task TestTailEvents()
{
// Uncomment when recording.
//DateTimeOffset startTime = DateTimeOffset.Now;
// Update and uncomment after recording.
DateTimeOffset startTime = new DateTimeOffset(2020, 8, 10, 16, 00, 00, TimeSpan.Zero);
TimeSpan pollInterval = Mode == RecordedTestMode.Playback ? TimeSpan.Zero : TimeSpan.FromMinutes(3);
BlobServiceClient service = GetServiceClient_SharedKey();
BlobChangeFeedClient blobChangeFeedClient = service.GetChangeFeedClient();
Page <BlobChangeFeedEvent> lastPage = null;
ISet <string> EventIdsPart1 = new HashSet <string>();
ISet <string> EventIdsPart2 = new HashSet <string>();
ISet <string> EventIdsPart3 = new HashSet <string>();
// Part 1
AsyncPageable <BlobChangeFeedEvent> blobChangeFeedAsyncPagable = blobChangeFeedClient.GetChangesAsync(start: startTime);
IAsyncEnumerable <Page <BlobChangeFeedEvent> > asyncEnumerable = blobChangeFeedAsyncPagable.AsPages();
await foreach (var page in asyncEnumerable)
{
lastPage = page;
foreach (var evt in page.Values)
{
EventIdsPart1.Add(evt.Id.ToString());
}
}
CollectionAssert.IsNotEmpty(EventIdsPart1);
await Task.Delay(pollInterval);
// Part 2
blobChangeFeedAsyncPagable = blobChangeFeedClient.GetChangesAsync(lastPage.ContinuationToken);
asyncEnumerable = blobChangeFeedAsyncPagable.AsPages();
await foreach (var page in asyncEnumerable)
{
lastPage = page;
foreach (var evt in page.Values)
{
EventIdsPart2.Add(evt.Id.ToString());
}
}
CollectionAssert.IsNotEmpty(EventIdsPart2);
await Task.Delay(pollInterval);
// Part 3
blobChangeFeedAsyncPagable = blobChangeFeedClient.GetChangesAsync(lastPage.ContinuationToken);
asyncEnumerable = blobChangeFeedAsyncPagable.AsPages();
await foreach (var page in asyncEnumerable)
{
lastPage = page;
foreach (var evt in page.Values)
{
EventIdsPart3.Add(evt.Id.ToString());
}
}
CollectionAssert.IsNotEmpty(EventIdsPart3);
// Assert events are not duplicated
CollectionAssert.IsEmpty(EventIdsPart1.Intersect(EventIdsPart2));
CollectionAssert.IsEmpty(EventIdsPart1.Intersect(EventIdsPart3));
CollectionAssert.IsEmpty(EventIdsPart2.Intersect(EventIdsPart3));
}
