public async Task CursorTest() { BlobServiceClient service = GetServiceClient_SharedKey(); BlobChangeFeedClient blobChangeFeedClient = service.GetChangeFeedClient(); AsyncPageable <BlobChangeFeedEvent> blobChangeFeedAsyncPagable = blobChangeFeedClient.GetChangesAsync(); IAsyncEnumerable <Page <BlobChangeFeedEvent> > asyncEnumerable = blobChangeFeedAsyncPagable.AsPages(pageSizeHint: 500); Page <BlobChangeFeedEvent> page = await asyncEnumerable.FirstAsync(); foreach (BlobChangeFeedEvent changeFeedEvent in page.Values) { Console.WriteLine(changeFeedEvent); } Console.WriteLine("break"); string continuation = page.ContinuationToken; AsyncPageable <BlobChangeFeedEvent> cursorBlobChangeFeedAsyncPagable = blobChangeFeedClient.GetChangesAsync(continuation); IList <BlobChangeFeedEvent> list = await cursorBlobChangeFeedAsyncPagable.ToListAsync(); foreach (BlobChangeFeedEvent e in list) { Console.WriteLine(e); } }
public async Task <Dictionary <string, List <BlobChangeFeedEvent> > > ReadChangeFeedAsync(string cursor) { // create blob service client BlobServiceClient blobServiceClient = new BlobServiceClient(this.connectionString); // get change feed client BlobChangeFeedClient blobChangeFeedClient = blobServiceClient.GetChangeFeedClient(); List <BlobChangeFeedEvent> blobChangeFeedEvents = new List <BlobChangeFeedEvent>(); IAsyncEnumerator <Page <BlobChangeFeedEvent> > enumerator = blobChangeFeedClient.GetChangesAsync(continuationToken: cursor) .AsPages(pageSizeHint: 5) .GetAsyncEnumerator(); await enumerator.MoveNextAsync(); Dictionary <string, List <BlobChangeFeedEvent> > keyValuePairs = new Dictionary <string, List <BlobChangeFeedEvent> >(); if (enumerator.Current != null && enumerator.Current.Values != null) { foreach (BlobChangeFeedEvent changeFeedEvent in enumerator.Current.Values) { blobChangeFeedEvents.Add(changeFeedEvent); } keyValuePairs.Add(enumerator.Current.ContinuationToken, blobChangeFeedEvents); } return(keyValuePairs); }
// Gets a cursor from container or creates a new cursor if it does not exist private static string GetCursor(BlobClient blobClient, BlobChangeFeedClient changeFeedClient, ILogger log) { string continuationToken = null; if (blobClient.Exists()) { // If the continuationToken exists in blob, download and use it var stream = new MemoryStream(); blobClient.DownloadTo(stream); continuationToken = Encoding.UTF8.GetString(stream.ToArray()); } else { // If the continuationToken does not exist in the blob, get the continuationToken from the first item Page <BlobChangeFeedEvent> page = changeFeedClient.GetChanges().AsPages(pageSizeHint: 1).First <Page <BlobChangeFeedEvent> >(); BlobChangeFeedEvent changeFeedEvent = page.Values.First <BlobChangeFeedEvent>(); if (containerCheck.Invoke(changeFeedEvent) && eventTypeCheck.Invoke(changeFeedEvent) && blobCheck.Invoke(changeFeedEvent)) { log.LogInformation($"event: {changeFeedEvent.EventType} at {changeFeedEvent.Subject.Replace("/blobServices/default", "")} on {changeFeedEvent.EventTime}"); } continuationToken = page.ContinuationToken; } return(continuationToken); }
public async Task TestNonRoundedBoundaries() { // This is hardcoded for playback stability. Feel free to modify but make sure recordings match. DateTimeOffset startTime = new DateTimeOffset(2020, 8, 5, 16, 24, 00, TimeSpan.Zero); DateTimeOffset endTime = new DateTimeOffset(2020, 8, 5, 18, 35, 00, TimeSpan.Zero); DateTimeOffset roundedStartTime = new DateTimeOffset(2020, 8, 5, 16, 00, 00, TimeSpan.Zero); DateTimeOffset roundedEndTime = new DateTimeOffset(2020, 8, 5, 19, 00, 00, TimeSpan.Zero); BlobServiceClient service = GetServiceClient_SharedKey(); BlobChangeFeedClient blobChangeFeedClient = service.GetChangeFeedClient(); // Collect all events within range AsyncPageable <BlobChangeFeedEvent> blobChangeFeedAsyncPagable = blobChangeFeedClient.GetChangesAsync( start: startTime, end: endTime); var eventList = new List <BlobChangeFeedEvent>(await blobChangeFeedAsyncPagable.ToListAsync()); // Assert Assert.Greater(eventList.Count, 1); Assert.IsNull(eventList.Find(e => e.EventTime < roundedStartTime.AddMinutes(-15)), "No event 15 minutes before start is present"); Assert.IsNull(eventList.Find(e => e.EventTime > roundedEndTime.AddMinutes(15)), "No event 15 minutes after end is present"); Assert.IsNotNull(eventList.Find(e => e.EventTime < roundedStartTime.AddMinutes(15)), "There is some event 15 minutes after start"); Assert.IsNotNull(eventList.Find(e => e.EventTime > roundedEndTime.AddMinutes(-15)), "There is some event 15 minutes before end"); }
public void SharedKeyAuth() { // Get a Storage account name, shared key, and endpoint Uri. // // You can obtain both from the Azure Portal by clicking Access // Keys under Settings in the Portal Storage account blade. // // You can also get access to your account keys from the Azure CLI // with: // // az storage account keys list --account-name <account_name> --resource-group <resource_group> // string accountName = StorageAccountName; string accountKey = StorageAccountKey; Uri serviceUri = StorageAccountBlobUri; // Create a SharedKeyCredential that we can use to authenticate StorageSharedKeyCredential credential = new StorageSharedKeyCredential(accountName, accountKey); // Create a client that can authenticate with a connection string BlobChangeFeedClient service = new BlobChangeFeedClient(serviceUri, credential); List <BlobChangeFeedEvent> changeFeedEvents = new List <BlobChangeFeedEvent>(); // Make a service request to verify we've successfully authenticated foreach (BlobChangeFeedEvent changeFeedEvent in service.GetChanges()) { changeFeedEvents.Add(changeFeedEvent); } }
private async Task <List <BlobChangeFeedEvent> > ProcessChangeFeedTask (string BlobStorageAccountName, DateTimeOffset start, DateTimeOffset end) { // Get a new blob service client. string BlobUrl = $"https://{BlobStorageAccountName}.blob.core.windows.net/"; // Get a credential and create a client object for the blob container. Note using new Azure Core credential flow BlobServiceClient blobServiceClient = new BlobServiceClient(new Uri(BlobUrl), new DefaultAzureCredential()); // Get a new change feed client. BlobChangeFeedClient changeFeedClient = blobServiceClient.GetChangeFeedClient(); List <BlobChangeFeedEvent> changeFeedEvents = new List <BlobChangeFeedEvent>(); IAsyncEnumerator <Page <BlobChangeFeedEvent> > enumerator = changeFeedClient .GetChangesAsync(start, end) .AsPages(pageSizeHint: 10) .GetAsyncEnumerator(); await enumerator.MoveNextAsync(); foreach (BlobChangeFeedEvent changeFeedEvent in enumerator.Current.Values) { changeFeedEvents.Add(changeFeedEvent); } return(changeFeedEvents); }
public void ChangeFeedBetweenDates() { // Get a connection string to our Azure Storage account. string connectionString = ConnectionString; // Get a new blob service client. BlobServiceClient blobServiceClient = new BlobServiceClient(connectionString); // Get a new change feed client. BlobChangeFeedClient changeFeedClient = blobServiceClient.GetChangeFeedClient(); List <BlobChangeFeedEvent> changeFeedEvents = new List <BlobChangeFeedEvent>(); // Create the start and end time. The change feed client will round start time down to // the nearest hour, and round endTime up to the next hour if you provide DateTimeOffsets // with minutes and seconds. DateTimeOffset startTime = new DateTimeOffset(2017, 3, 2, 15, 0, 0, TimeSpan.Zero); DateTimeOffset endTime = new DateTimeOffset(2020, 10, 7, 2, 0, 0, TimeSpan.Zero); // You can also provide just a start or end time. foreach (BlobChangeFeedEvent changeFeedEvent in changeFeedClient.GetChanges( start: startTime, end: endTime)) { changeFeedEvents.Add(changeFeedEvent); } }
public void ActiveDirectoryAuth() { // Create a token credential that can use our Azure Active // Directory application to authenticate with Azure Storage TokenCredential credential = new ClientSecretCredential( ActiveDirectoryTenantId, ActiveDirectoryApplicationId, ActiveDirectoryApplicationSecret, new TokenCredentialOptions() { AuthorityHost = ActiveDirectoryAuthEndpoint }); // Create a client that can authenticate using our token credential BlobChangeFeedClient service = new BlobChangeFeedClient(ActiveDirectoryBlobUri, credential); List <BlobChangeFeedEvent> changeFeedEvents = new List <BlobChangeFeedEvent>(); // Make a service request to verify we've successfully authenticated foreach (BlobChangeFeedEvent changeFeedEvent in service.GetChanges()) { changeFeedEvents.Add(changeFeedEvent); } }
public async Task ChangeFeedResumeWithCursorAsync() { // Get a connection string to our Azure Storage account. string connectionString = ConnectionString; // Get a new change feed client. BlobChangeFeedClient changeFeedClient = new BlobChangeFeedClient(connectionString); List <BlobChangeFeedEvent> changeFeedEvents = new List <BlobChangeFeedEvent>(); #region Snippet:SampleSnippetsChangeFeed_ResumeWithCursor string continuationToken = null; await foreach (Page <BlobChangeFeedEvent> page in changeFeedClient.GetChangesAsync().AsPages(pageSizeHint: 10)) { foreach (BlobChangeFeedEvent changeFeedEvent in page.Values) { changeFeedEvents.Add(changeFeedEvent); } // Get the change feed continuation token. The continuation token is not required to get each page of events, // it is intended to be saved and used to resume iterating at a later date. continuationToken = page.ContinuationToken; break; } // Resume iterating from the pervious position with the continuation token. await foreach (BlobChangeFeedEvent changeFeedEvent in changeFeedClient.GetChangesAsync( continuationToken: continuationToken)) { changeFeedEvents.Add(changeFeedEvent); } #endregion }
public async Task ChangeFeedBetweenDatesAsync() { // Get a connection string to our Azure Storage account. string connectionString = ConnectionString; // Get a new change feed client. BlobChangeFeedClient changeFeedClient = new BlobChangeFeedClient(connectionString); List <BlobChangeFeedEvent> changeFeedEvents = new List <BlobChangeFeedEvent>(); #region Snippet:SampleSnippetsChangeFeed_GetEventsBetweenStartAndEndTime // Create the start and end time. The change feed client will round start time down to // the nearest hour, and round endTime up to the next hour if you provide DateTimeOffsets // with minutes and seconds. DateTimeOffset startTime = new DateTimeOffset(2017, 3, 2, 15, 0, 0, TimeSpan.Zero); DateTimeOffset endTime = new DateTimeOffset(2020, 10, 7, 2, 0, 0, TimeSpan.Zero); // You can also provide just a start or end time. await foreach (BlobChangeFeedEvent changeFeedEvent in changeFeedClient.GetChangesAsync( start: startTime, end: endTime)) { changeFeedEvents.Add(changeFeedEvent); } #endregion }
public async Task CursorFormatTest() { // This is hardcoded for playback stability. Feel free to modify but make sure recordings match. DateTimeOffset startTime = new DateTimeOffset(2020, 7, 30, 23, 00, 00, TimeSpan.Zero); DateTimeOffset endTime = new DateTimeOffset(2020, 7, 30, 23, 15, 00, TimeSpan.Zero); BlobServiceClient service = GetServiceClient_SharedKey(); BlobChangeFeedClient blobChangeFeedClient = service.GetChangeFeedClient(); // Iterate over first two pages var blobChangeFeedAsyncPagable = blobChangeFeedClient.GetChangesAsync( start: startTime, end: endTime); IAsyncEnumerable <Page <BlobChangeFeedEvent> > asyncEnumerable = blobChangeFeedAsyncPagable.AsPages(pageSizeHint: 50); Page <BlobChangeFeedEvent> lastPage = null; int pages = 0; await foreach (Page <BlobChangeFeedEvent> page in asyncEnumerable) { foreach (BlobChangeFeedEvent e in page.Values) { Console.WriteLine(e); } pages++; lastPage = page; if (pages > 2) { break; } } // Act string continuation = lastPage.ContinuationToken; // Verify // You may need to update expected values when re-recording var cursor = (JsonSerializer.Deserialize(continuation, typeof(ChangeFeedCursor)) as ChangeFeedCursor); Assert.AreEqual(new DateTimeOffset(2020, 7, 31, 00, 00, 00, TimeSpan.Zero), cursor.EndTime); Assert.AreEqual(1, cursor.CursorVersion); Assert.AreEqual("emilydevtest.blob.core.windows.net", cursor.UrlHost); var currentSegmentCursor = cursor.CurrentSegmentCursor; Assert.AreEqual("idx/segments/2020/07/30/2300/meta.json", currentSegmentCursor.SegmentPath); Assert.AreEqual("log/00/2020/07/30/2300/", currentSegmentCursor.CurrentShardPath); Assert.AreEqual(1, currentSegmentCursor.ShardCursors.Count); var shardCursor = currentSegmentCursor.ShardCursors.First(); Assert.AreEqual("log/00/2020/07/30/2300/00000.avro", shardCursor.CurrentChunkPath); Assert.AreEqual(96253, shardCursor.BlockOffset); Assert.AreEqual(0, shardCursor.EventIndex); }
public void Test() { BlobServiceClient service = GetServiceClient_SharedKey(); BlobChangeFeedClient blobChangeFeedClient = service.GetChangeFeedClient(); Pageable <BlobChangeFeedEvent> blobChangeFeedPagable = blobChangeFeedClient.GetChanges(); IList <BlobChangeFeedEvent> list = blobChangeFeedPagable.ToList(); foreach (BlobChangeFeedEvent e in list) { Console.WriteLine(e); } }
public async Task TestLastHour() { BlobServiceClient service = GetServiceClient_SharedKey(); BlobChangeFeedClient blobChangeFeedClient = service.GetChangeFeedClient(); AsyncPageable <BlobChangeFeedEvent> blobChangeFeedAsyncPagable = blobChangeFeedClient.GetChangesAsync(start: DateTime.Now, end: DateTime.Now); IList <BlobChangeFeedEvent> list = await blobChangeFeedAsyncPagable.ToListAsync(); foreach (BlobChangeFeedEvent e in list) { Console.WriteLine(e); } }
public async Task CanReadTillEnd() { // Uncomment when recording. //DateTimeOffset startTime = DateTimeOffset.Now; // Update and uncomment after recording. DateTimeOffset startTime = new DateTimeOffset(2020, 7, 31, 19, 00, 00, TimeSpan.Zero); BlobServiceClient service = GetServiceClient_SharedKey(); BlobChangeFeedClient blobChangeFeedClient = service.GetChangeFeedClient(); AsyncPageable <BlobChangeFeedEvent> blobChangeFeedAsyncPagable = blobChangeFeedClient.GetChangesAsync(startTime); IList <BlobChangeFeedEvent> list = await blobChangeFeedAsyncPagable.ToListAsync(); CollectionAssert.IsNotEmpty(list); }
public void ChangeFeed() { // Get a connection string to our Azure Storage account. string connectionString = ConnectionString; // Get a new change feed client. BlobChangeFeedClient changeFeedClient = new BlobChangeFeedClient(connectionString); // Get all the events in the change feed. List <BlobChangeFeedEvent> changeFeedEvents = new List <BlobChangeFeedEvent>(); foreach (BlobChangeFeedEvent changeFeedEvent in changeFeedClient.GetChanges()) { changeFeedEvents.Add(changeFeedEvent); } }
public async Task ChangeFeedAsync() { // Get a connection string to our Azure Storage account. string connectionString = ConnectionString; // Get a new change feed client. BlobChangeFeedClient changeFeedClient = new BlobChangeFeedClient(connectionString); #region Snippet:SampleSnippetsChangeFeed_GetAllEvents // Get all the events in the change feed. List <BlobChangeFeedEvent> changeFeedEvents = new List <BlobChangeFeedEvent>(); await foreach (BlobChangeFeedEvent changeFeedEvent in changeFeedClient.GetChangesAsync()) { changeFeedEvents.Add(changeFeedEvent); } #endregion }
public async Task ChangeFeedPollForEventsWithCursor() { // Get a connection string to our Azure Storage account. string connectionString = ConnectionString; // Get a new change feed client. BlobChangeFeedClient changeFeedClient = new BlobChangeFeedClient(connectionString); List <BlobChangeFeedEvent> changeFeedEvents = new List <BlobChangeFeedEvent>(); #region Snippet:SampleSnippetsChangeFeed_PollForEventsWithCursor // Create the start time. The change feed client will round start time down to // the nearest hour if you provide DateTimeOffsets // with minutes and seconds. DateTimeOffset startTime = DateTimeOffset.Now; // Create polling interval. TimeSpan pollingInterval = TimeSpan.FromMinutes(5); // Get initial set of events. IAsyncEnumerable <Page <BlobChangeFeedEvent> > pages = changeFeedClient.GetChangesAsync(start: startTime).AsPages(); string continuationToken = null; while (true) { await foreach (Page <BlobChangeFeedEvent> page in pages) { foreach (BlobChangeFeedEvent changeFeedEvent in page.Values) { changeFeedEvents.Add(changeFeedEvent); } // Get the change feed continuation token. The continuation token is not required to get each page of events, // it is intended to be saved and used to resume iterating at a later date. // For the purpose of actively listening to events the continuation token from last page is used. continuationToken = page.ContinuationToken; } // Wait before processing next batch of events. await Task.Delay(pollingInterval); // Resume from last continuation token and fetch latest set of events. pages = changeFeedClient.GetChangesAsync(continuationToken).AsPages(); } #endregion }
public async Task PageSizeTest() { int pageSize = 100; BlobServiceClient service = GetServiceClient_SharedKey(); BlobChangeFeedClient blobChangeFeedClient = service.GetChangeFeedClient(); IAsyncEnumerator <Page <BlobChangeFeedEvent> > asyncEnumerator = blobChangeFeedClient.GetChangesAsync().AsPages(pageSizeHint: pageSize).GetAsyncEnumerator(); List <int> pageSizes = new List <int>(); while (await asyncEnumerator.MoveNextAsync()) { pageSizes.Add(asyncEnumerator.Current.Values.Count); } // All pages except the last should have a count == pageSize. for (int i = 0; i < pageSizes.Count - 1; i++) { Assert.AreEqual(pageSize, pageSizes[i]); } }
public async Task ImmediateResumeFromEndOfCurrentHourYieldsEmptyResult() { // Uncomment when recording. //DateTimeOffset startTime = DateTimeOffset.Now; // Update and uncomment after recording. DateTimeOffset startTime = new DateTimeOffset(2020, 8, 11, 21, 00, 00, TimeSpan.Zero); BlobServiceClient service = GetServiceClient_SharedKey(); BlobChangeFeedClient blobChangeFeedClient = service.GetChangeFeedClient(); // Collect all events within range AsyncPageable <BlobChangeFeedEvent> blobChangeFeedAsyncPagable = blobChangeFeedClient.GetChangesAsync( start: startTime); ISet <string> AllEventIds = new HashSet <string>(); IAsyncEnumerable <Page <BlobChangeFeedEvent> > asyncEnumerable = blobChangeFeedAsyncPagable.AsPages(pageSizeHint: 50); Page <BlobChangeFeedEvent> lastPage = null; await foreach (Page <BlobChangeFeedEvent> page in asyncEnumerable) { foreach (BlobChangeFeedEvent e in page.Values) { AllEventIds.Add(e.Id.ToString()); } lastPage = page; } string continuation = lastPage.ContinuationToken; // Act blobChangeFeedAsyncPagable = blobChangeFeedClient.GetChangesAsync(continuation); IList <BlobChangeFeedEvent> tail = await blobChangeFeedAsyncPagable.ToListAsync(); // Assert Assert.AreEqual(0, tail.Count); Assert.Greater(AllEventIds.Count, 0); }
public void SharedAccessSignatureAuth() { // Create a blob level SAS that allows reading change events from blob // level APIs AccountSasBuilder sas = new AccountSasBuilder { // Allow access to blobs Services = AccountSasServices.Blobs, // Allow access to the service level APIs ResourceTypes = AccountSasResourceTypes.All, // Access expires in 1 hour! ExpiresOn = DateTimeOffset.UtcNow.AddHours(1) }; // Allow all access sas.SetPermissions(AccountSasPermissions.All); // Create a SharedKeyCredential that we can use to sign the SAS token StorageSharedKeyCredential credential = new StorageSharedKeyCredential(StorageAccountName, StorageAccountKey); // Build a SAS URI UriBuilder sasUri = new UriBuilder(StorageAccountBlobUri); sasUri.Query = sas.ToSasQueryParameters(credential).ToString(); // Create a client that can authenticate with the SAS URI BlobChangeFeedClient service = new BlobChangeFeedClient(sasUri.Uri); List <BlobChangeFeedEvent> changeFeedEvents = new List <BlobChangeFeedEvent>(); // Make a service request to verify we've successfully authenticated foreach (BlobChangeFeedEvent changeFeedEvent in service.GetChanges()) { changeFeedEvents.Add(changeFeedEvent); } }
public async Task ResumeFromEndInThePastYieldsEmptyResult() { // This is hardcoded for playback stability. Feel free to modify but make sure recordings match. DateTimeOffset startTime = new DateTimeOffset(2020, 7, 30, 23, 00, 00, TimeSpan.Zero); DateTimeOffset endTime = new DateTimeOffset(2020, 7, 30, 23, 15, 00, TimeSpan.Zero); BlobServiceClient service = GetServiceClient_SharedKey(); BlobChangeFeedClient blobChangeFeedClient = service.GetChangeFeedClient(); // Collect all events within range AsyncPageable <BlobChangeFeedEvent> blobChangeFeedAsyncPagable = blobChangeFeedClient.GetChangesAsync( start: startTime, end: endTime); ISet <string> AllEventIds = new HashSet <string>(); IAsyncEnumerable <Page <BlobChangeFeedEvent> > asyncEnumerable = blobChangeFeedAsyncPagable.AsPages(pageSizeHint: 50); Page <BlobChangeFeedEvent> lastPage = null; await foreach (Page <BlobChangeFeedEvent> page in asyncEnumerable) { foreach (BlobChangeFeedEvent e in page.Values) { AllEventIds.Add(e.Id.ToString()); } lastPage = page; } string continuation = lastPage.ContinuationToken; // Act blobChangeFeedAsyncPagable = blobChangeFeedClient.GetChangesAsync(continuation); IList <BlobChangeFeedEvent> tail = await blobChangeFeedAsyncPagable.ToListAsync(); // Assert Assert.AreEqual(0, tail.Count); Assert.Greater(AllEventIds.Count, 0); }
public void ChangeFeedResumeWithCursor() { // Get a connection string to our Azure Storage account. string connectionString = ConnectionString; // Get a new blob service client. BlobServiceClient blobServiceClient = new BlobServiceClient(connectionString); // Get a new change feed client. BlobChangeFeedClient changeFeedClient = blobServiceClient.GetChangeFeedClient(); List <BlobChangeFeedEvent> changeFeedEvents = new List <BlobChangeFeedEvent>(); IEnumerator <Page <BlobChangeFeedEvent> > enumerator = changeFeedClient .GetChanges() .AsPages(pageSizeHint: 10) .GetEnumerator(); ; enumerator.MoveNext(); foreach (BlobChangeFeedEvent changeFeedEvent in enumerator.Current.Values) { changeFeedEvents.Add(changeFeedEvent); } // get the change feed cursor. The cursor is not required to get each page of events, // it is intended to be saved and used to resume iterating at a later date. string cursor = enumerator.Current.ContinuationToken; // Resume iterating from the pervious position with the cursor. foreach (BlobChangeFeedEvent changeFeedEvent in changeFeedClient.GetChanges( continuationToken: cursor)) { changeFeedEvents.Add(changeFeedEvent); } }
public static void Run([TimerTrigger("0 */30 * * * *")] TimerInfo myTimer, ILogger log) { // Create client for Changefeed BlobServiceClient blobServiceClient = new BlobServiceClient(connectionString); BlobChangeFeedClient changeFeedClient = blobServiceClient.GetChangeFeedClient(); try { BlobContainerClient containerClient = blobServiceClient.CreateBlobContainer("cursorstoragecontainer"); } catch { BlobContainerClient containerClient = blobServiceClient.GetBlobContainerClient("cursorstoragecontainer"); } BlobClient blobClient = new BlobClient(connectionString, "cursorstoragecontainer", "cursorBlob"); // Create and get cursor for Changefeed string continuationToken = GetCursor(blobClient, changeFeedClient, log); // Loop through the events in Changefeed with the continuationToken foreach (Page <BlobChangeFeedEvent> page in changeFeedClient.GetChanges(continuationToken: continuationToken).AsPages()) { foreach (BlobChangeFeedEvent changeFeedEvent in page.Values) { if (containerCheck.Invoke(changeFeedEvent) && eventTypeCheck.Invoke(changeFeedEvent) && blobCheck.Invoke(changeFeedEvent)) { log.LogInformation($"event: {changeFeedEvent.EventType} at {changeFeedEvent.Subject.Replace("/blobServices/default","")} on {changeFeedEvent.EventTime}"); } } continuationToken = page.ContinuationToken; } // Upload new continuationToken to blob var upStream = new MemoryStream(Encoding.ASCII.GetBytes(continuationToken)); blobClient.Upload(upStream, true); }
public void ConnectionStringAuth() { // Get a connection string to our Azure Storage account. You can // obtain your connection string from the Azure Portal (click // Access Keys under Settings in the Portal Storage account blade) // or using the Azure CLI with: // // az storage account show-connection-string --name <account_name> --resource-group <resource_group> // // And you can provide the connection string to your application // using an environment variable. string connectionString = ConnectionString; // Create a client that can authenticate with a connection string BlobChangeFeedClient service = new BlobChangeFeedClient(connectionString); List <BlobChangeFeedEvent> changeFeedEvents = new List <BlobChangeFeedEvent>(); // Make a service request to verify we've successfully authenticated foreach (BlobChangeFeedEvent changeFeedEvent in service.GetChanges()) { changeFeedEvents.Add(changeFeedEvent); } }
public async Task ResumeFromTheMiddleOfTheChunkWithManyNonEmptyShards() { // This is hardcoded for playback stability. Feel free to modify but make sure recordings match. DateTimeOffset startTime = new DateTimeOffset(2020, 8, 5, 17, 00, 00, TimeSpan.Zero); DateTimeOffset endTime = new DateTimeOffset(2020, 8, 5, 17, 15, 00, TimeSpan.Zero); int expectedShardCount = 3; BlobServiceClient service = GetServiceClient_SharedKey(); BlobChangeFeedClient blobChangeFeedClient = service.GetChangeFeedClient(); // Collect all events within range AsyncPageable <BlobChangeFeedEvent> blobChangeFeedAsyncPagable = blobChangeFeedClient.GetChangesAsync( start: startTime, end: endTime); ISet <string> AllEventIds = new HashSet <string>(); await foreach (BlobChangeFeedEvent e in blobChangeFeedAsyncPagable) { AllEventIds.Add(e.Id.ToString()); } // Iterate over first two pages ISet <string> EventIdsPart1 = new HashSet <string>(); blobChangeFeedAsyncPagable = blobChangeFeedClient.GetChangesAsync( start: startTime, end: endTime); IAsyncEnumerable <Page <BlobChangeFeedEvent> > asyncEnumerable = blobChangeFeedAsyncPagable.AsPages(pageSizeHint: 50); Page <BlobChangeFeedEvent> lastPage = null; int pages = 0; await foreach (Page <BlobChangeFeedEvent> page in asyncEnumerable) { foreach (BlobChangeFeedEvent e in page.Values) { EventIdsPart1.Add(e.Id.ToString()); } pages++; lastPage = page; if (pages > 2) { break; } } string continuation = lastPage.ContinuationToken; var currentSegmentCursor = (JsonSerializer.Deserialize(continuation, typeof(ChangeFeedCursor)) as ChangeFeedCursor).CurrentSegmentCursor; Assert.AreEqual(currentSegmentCursor.ShardCursors.Count, expectedShardCount); Assert.IsNotNull(currentSegmentCursor.ShardCursors.Find(x => x.BlockOffset > 0), "Making sure we actually finish some shard in the middle of chunk, if this fails play with test data to make it pass"); // Iterate over next two pages ISet <string> EventIdsPart2 = new HashSet <string>(); blobChangeFeedAsyncPagable = blobChangeFeedClient.GetChangesAsync(continuation); asyncEnumerable = blobChangeFeedAsyncPagable.AsPages(pageSizeHint: 50); lastPage = null; pages = 0; await foreach (Page <BlobChangeFeedEvent> page in asyncEnumerable) { foreach (BlobChangeFeedEvent e in page.Values) { EventIdsPart2.Add(e.Id.ToString()); } pages++; lastPage = page; if (pages > 2) { break; } } continuation = lastPage.ContinuationToken; currentSegmentCursor = (JsonSerializer.Deserialize(continuation, typeof(ChangeFeedCursor)) as ChangeFeedCursor).CurrentSegmentCursor; Assert.AreEqual(currentSegmentCursor.ShardCursors.Count, expectedShardCount); Assert.IsNotNull(currentSegmentCursor.ShardCursors.Find(x => x.BlockOffset > 0), "Making sure we actually finish some shard in the middle of chunk, if this fails play with test data to make it pass"); // Iterate over remaining ISet <string> EventIdsTail = new HashSet <string>(); AsyncPageable <BlobChangeFeedEvent> cursorBlobChangeFeedAsyncPagable = blobChangeFeedClient.GetChangesAsync(continuation); IList <BlobChangeFeedEvent> list = await cursorBlobChangeFeedAsyncPagable.ToListAsync(); foreach (BlobChangeFeedEvent e in list) { EventIdsTail.Add(e.Id.ToString()); } ISet <string> AllEventIdsFromResumingIteration = new HashSet <string>(); AllEventIdsFromResumingIteration.UnionWith(EventIdsPart1); AllEventIdsFromResumingIteration.UnionWith(EventIdsPart2); AllEventIdsFromResumingIteration.UnionWith(EventIdsTail); Assert.Greater(AllEventIds.Count, 0); Assert.Greater(EventIdsPart1.Count, 0); Assert.Greater(EventIdsPart2.Count, 0); Assert.Greater(EventIdsTail.Count, 0); Assert.AreEqual(AllEventIds.Count, EventIdsPart1.Count + EventIdsPart2.Count + EventIdsTail.Count); CollectionAssert.AreEqual(AllEventIds, AllEventIdsFromResumingIteration); }
public async Task TestTailEvents() { // Uncomment when recording. //DateTimeOffset startTime = DateTimeOffset.Now; // Update and uncomment after recording. DateTimeOffset startTime = new DateTimeOffset(2020, 8, 10, 16, 00, 00, TimeSpan.Zero); TimeSpan pollInterval = Mode == RecordedTestMode.Playback ? TimeSpan.Zero : TimeSpan.FromMinutes(3); BlobServiceClient service = GetServiceClient_SharedKey(); BlobChangeFeedClient blobChangeFeedClient = service.GetChangeFeedClient(); Page <BlobChangeFeedEvent> lastPage = null; ISet <string> EventIdsPart1 = new HashSet <string>(); ISet <string> EventIdsPart2 = new HashSet <string>(); ISet <string> EventIdsPart3 = new HashSet <string>(); // Part 1 AsyncPageable <BlobChangeFeedEvent> blobChangeFeedAsyncPagable = blobChangeFeedClient.GetChangesAsync(start: startTime); IAsyncEnumerable <Page <BlobChangeFeedEvent> > asyncEnumerable = blobChangeFeedAsyncPagable.AsPages(); await foreach (var page in asyncEnumerable) { lastPage = page; foreach (var evt in page.Values) { EventIdsPart1.Add(evt.Id.ToString()); } } CollectionAssert.IsNotEmpty(EventIdsPart1); await Task.Delay(pollInterval); // Part 2 blobChangeFeedAsyncPagable = blobChangeFeedClient.GetChangesAsync(lastPage.ContinuationToken); asyncEnumerable = blobChangeFeedAsyncPagable.AsPages(); await foreach (var page in asyncEnumerable) { lastPage = page; foreach (var evt in page.Values) { EventIdsPart2.Add(evt.Id.ToString()); } } CollectionAssert.IsNotEmpty(EventIdsPart2); await Task.Delay(pollInterval); // Part 3 blobChangeFeedAsyncPagable = blobChangeFeedClient.GetChangesAsync(lastPage.ContinuationToken); asyncEnumerable = blobChangeFeedAsyncPagable.AsPages(); await foreach (var page in asyncEnumerable) { lastPage = page; foreach (var evt in page.Values) { EventIdsPart3.Add(evt.Id.ToString()); } } CollectionAssert.IsNotEmpty(EventIdsPart3); // Assert events are not duplicated CollectionAssert.IsEmpty(EventIdsPart1.Intersect(EventIdsPart2)); CollectionAssert.IsEmpty(EventIdsPart1.Intersect(EventIdsPart3)); CollectionAssert.IsEmpty(EventIdsPart2.Intersect(EventIdsPart3)); }