public async Task OrderInGetReadReceiptsIteratorIsNotAlteredByPaging()
{
//arrange
var threadId = "19:[email protected]";
var uri = new Uri("https://localHostTest");
var baseSenderId = "8:acs:1b5cc06b-f352-4571-b1e6-d9b259b7c776_00000007-0464-274b-b274-5a3a0d00010";
var baseReadOnDate = DateTimeOffset.Parse("2020-12-15T00:00:00Z");
var responseItemsPage1 = new MockResponse(200);
responseItemsPage1.SetContent(Page1ReadReceiptsApiResponsePayload);
var responseItemsPage2 = new MockResponse(200);
responseItemsPage2.SetContent(Page2ReadReceiptsApiResponsePayload);
var chatClientOptions = new ChatClientOptions
{
Transport = new MockTransport(responseItemsPage1, responseItemsPage2)
};
//act
var communicationTokenCredential = new CommunicationTokenCredential(ChatRecordedTestSanitizer.SanitizedUnsignedUserTokenValue);
var chatClient = new ChatClient(uri, communicationTokenCredential, chatClientOptions);
ChatThreadClient chatThreadClient = chatClient.GetChatThreadClient(threadId);
AsyncPageable <ChatMessageReadReceipt> allreadReceipts = chatThreadClient.GetReadReceiptsAsync();
//assert
int pages = 0;
int idCounter = 0;
//string? continuationToken = null;
await foreach (Page <ChatMessageReadReceipt> page in allreadReceipts.AsPages(pageSizeHint: 3))
{
pages++;
foreach (ChatMessageReadReceipt readReceipt in page.Values)
{
idCounter++;
Assert.AreEqual($"{idCounter}", readReceipt.ChatMessageId);
Assert.AreEqual($"{baseSenderId}{idCounter}", readReceipt.SenderId);
Assert.AreEqual(baseReadOnDate.AddSeconds(idCounter), readReceipt.ReadOn);
}
//continuationToken = page.ContinuationToken;
}
Assert.AreEqual(2, pages);
Assert.AreEqual(5, idCounter);
}
public async Task CanGetManifestsWithCustomPageSize(bool anonymous)
{
// Arrange
var client = CreateClient(anonymous);
var repository = client.GetRepository(_repositoryName);
int pageSize = 2;
int minExpectedPages = 2;
// Act
AsyncPageable <ArtifactManifestProperties> artifacts = repository.GetAllManifestPropertiesAsync();
var pages = artifacts.AsPages(pageSizeHint: pageSize);
// Assert
int pageCount = await pages.CountAsync();
Assert.GreaterOrEqual(pageCount, minExpectedPages);
}
// [Test] -- Enable the tests when you're running the samples for the service
public async Task ListResources()
{
#region Snippet:ListResources
var client = GetClient();
AsyncPageable <BinaryData> pageable = client.GetResourcesAsync();
await foreach (var page in pageable.AsPages())
{
foreach (var resourceBinaryData in page.Values)
{
using JsonDocument resourceJson = JsonDocument.Parse(resourceBinaryData.ToMemory());
Console.WriteLine(resourceJson.RootElement.GetProperty("name").ToString());
Console.WriteLine(resourceJson.RootElement.GetProperty("id").ToString());
}
}
#endregion
}
/// <inheritdoc/>
public async Task <TenantCollectionResult> GetChildrenAsync(string tenantId, int limit, string?continuationToken)
{
(ITenant _, BlobContainerClient container) = await this.GetContainerAndTenantForChildTenantsOfAsync(tenantId)
.ConfigureAwait(false);
string?blobContinuationToken = DecodeUrlEncodedContinuationToken(continuationToken);
AsyncPageable <BlobItem> pageable = container.GetBlobsAsync(prefix: LiveTenantsPrefix);
IAsyncEnumerable <Page <BlobItem> > pages = pageable.AsPages(blobContinuationToken, limit);
await using IAsyncEnumerator <Page <BlobItem> > page = pages.GetAsyncEnumerator();
Page <BlobItem>?p = await page.MoveNextAsync()
? page.Current
: null;
IEnumerable <BlobItem> items = p?.Values ?? Enumerable.Empty <BlobItem>();
return(new TenantCollectionResult(
items.Select(s => s.Name[LiveTenantsPrefix.Length..]).ToList(),
// -- or --
public async IAsyncEnumerable <BlobClient> GetAllBlobsAsync(BlobContainerClient container)
{
string token = null;
do
{
AsyncPageable <BlobItem> pageable =
container.GetBlobsAsync(BlobTraits.None, BlobStates.None, string.Empty);
IAsyncEnumerable <Page <BlobItem> > pages = pageable.AsPages(token, 100);
await foreach (Page <BlobItem> page in pages)
{
token = page.ContinuationToken;
foreach (BlobItem blob in page.Values)
{
yield return(container.GetBlobClient(blob.Name));
}
}
} while (!string.IsNullOrEmpty(token));
}
public static async Task <List <SecretProperties> > FindSecrets(AsyncPageable <SecretProperties> secretsPages, Func <SecretProperties, bool> comparison = null)
{
// if no comparison is provided, every item is a match
if (comparison == null)
{
comparison = x => true;
}
var secretsList = new List <SecretProperties>();
await foreach (Page <SecretProperties> page in secretsPages.AsPages())
{
foreach (SecretProperties secret in page.Values.Where(x => comparison(x)))
{
secretsList.Add(secret);
}
}
return(secretsList);
}
public async Task CanGetTagsWithCustomPageSize()
{
// Arrange
var client = CreateClient();
int pageSize = 2;
int minExpectedPages = 2;
// Act
AsyncPageable <TagProperties> tags = client.GetTagsAsync();
var pages = tags.AsPages(pageSizeHint: pageSize);
int pageCount = 0;
await foreach (var page in pages)
{
Assert.IsTrue(page.Values.Count <= pageSize);
pageCount++;
}
// Assert
Assert.IsTrue(pageCount >= minExpectedPages);
}
public static async Task AssertPaginationAsync(AsyncPageable <T> enumerableResource, int expectedPageSize, int expectedTotalResources)
{
string?continuationToken = null;
int expectedRoundTrips = (expectedTotalResources / expectedPageSize) + 1;
int actualPageSize, actualTotalResources = 0, actualRoundTrips = 0;
await foreach (Page <T> page in enumerableResource.AsPages(continuationToken, expectedPageSize))
{
actualRoundTrips++;
actualPageSize = 0;
foreach (T resource in page.Values)
{
actualPageSize++;
actualTotalResources++;
}
continuationToken = page.ContinuationToken;
Assert.GreaterOrEqual(expectedPageSize, actualPageSize);
}
Assert.IsNull(continuationToken);
Assert.AreEqual(expectedTotalResources, actualTotalResources);
Assert.AreEqual(expectedRoundTrips, actualRoundTrips);
}
public async Task AsyncPageableAsPages()
{
// create a client
var client = new SecretClient(new Uri("http://example.com"), new DefaultAzureCredential());
#region Snippet:AsyncPageableAsPages
// call a service method, which returns AsyncPageable<T>
AsyncPageable <SecretProperties> response = client.GetPropertiesOfSecretsAsync();
await foreach (Page <SecretProperties> page in response.AsPages())
{
// enumerate through page items
foreach (SecretProperties secretProperties in page.Values)
{
Console.WriteLine(secretProperties.Name);
}
// get continuation token that can be used in AsPages call to resume enumeration
Console.WriteLine(page.ContinuationToken);
}
#endregion
}
public async Task CachesDifferentResponse(bool isAsync)
{
string getClientRequestId;
if (isAsync)
{
Response <KeyVaultSecret> response = await _fixture.Client.GetSecretAsync(_fixture.SecretName);
getClientRequestId = response.GetRawResponse().ClientRequestId;
}
else
{
Response <KeyVaultSecret> response = _fixture.Client.GetSecret(_fixture.SecretName);
getClientRequestId = response.GetRawResponse().ClientRequestId;
}
string listClientRequestId = null;
if (isAsync)
{
AsyncPageable <SecretProperties> response = _fixture.Client.GetPropertiesOfSecretsAsync();
await foreach (Page <SecretProperties> page in response.AsPages())
{
listClientRequestId = page.GetRawResponse().ClientRequestId;
break;
}
}
else
{
Pageable <SecretProperties> response = _fixture.Client.GetPropertiesOfSecrets();
foreach (Page <SecretProperties> page in response.AsPages())
{
listClientRequestId = page.GetRawResponse().ClientRequestId;
break;
}
}
Assert.NotEqual(getClientRequestId, listClientRequestId);
}
public async Task CanGetRepositoriesWithCustomPageSize()
{
// Arrange
var client = CreateClient();
int pageSize = 2;
int minExpectedPages = 2;
// Act
AsyncPageable <string> repositories = client.GetRepositoryNamesAsync();
var pages = repositories.AsPages(pageSizeHint: pageSize);
int pageCount = 0;
await foreach (var page in pages)
{
Assert.IsTrue(page.Values.Count <= pageSize);
pageCount++;
}
// Assert
Assert.GreaterOrEqual(pageCount, minExpectedPages);
}
public async Task ImmediateResumeFromEndOfCurrentHourYieldsEmptyResult()
{
// Uncomment when recording.
//DateTimeOffset startTime = DateTimeOffset.Now;
// Update and uncomment after recording.
DateTimeOffset startTime = new DateTimeOffset(2020, 8, 11, 21, 00, 00, TimeSpan.Zero);
BlobServiceClient service = GetServiceClient_SharedKey();
BlobChangeFeedClient blobChangeFeedClient = service.GetChangeFeedClient();
// Collect all events within range
AsyncPageable <BlobChangeFeedEvent> blobChangeFeedAsyncPagable
= blobChangeFeedClient.GetChangesAsync(
start: startTime);
ISet <string> AllEventIds = new HashSet <string>();
IAsyncEnumerable <Page <BlobChangeFeedEvent> > asyncEnumerable = blobChangeFeedAsyncPagable.AsPages(pageSizeHint: 50);
Page <BlobChangeFeedEvent> lastPage = null;
await foreach (Page <BlobChangeFeedEvent> page in asyncEnumerable)
{
foreach (BlobChangeFeedEvent e in page.Values)
{
AllEventIds.Add(e.Id.ToString());
}
lastPage = page;
}
string continuation = lastPage.ContinuationToken;
// Act
blobChangeFeedAsyncPagable = blobChangeFeedClient.GetChangesAsync(continuation);
IList <BlobChangeFeedEvent> tail = await blobChangeFeedAsyncPagable.ToListAsync();
// Assert
Assert.AreEqual(0, tail.Count);
Assert.Greater(AllEventIds.Count, 0);
}
public async Task CanGetArtifactsWithCustomPageSize()
{
// Arrange
var client = CreateClient();
int pageSize = 2;
int minExpectedPages = 2;
// Act
AsyncPageable <RegistryArtifactProperties> artifacts = client.GetRegistryArtifactsAsync();
var pages = artifacts.AsPages(pageSizeHint: pageSize);
int pageCount = 0;
await foreach (var page in pages)
{
Assert.GreaterOrEqual(page.Values.Count, pageSize);
pageCount++;
}
// Assert
Assert.IsTrue(pageCount >= minExpectedPages);
}
public async Task ResumeFromEndInThePastYieldsEmptyResult()
{
// This is hardcoded for playback stability. Feel free to modify but make sure recordings match.
DateTimeOffset startTime = new DateTimeOffset(2020, 7, 30, 23, 00, 00, TimeSpan.Zero);
DateTimeOffset endTime = new DateTimeOffset(2020, 7, 30, 23, 15, 00, TimeSpan.Zero);
BlobServiceClient service = GetServiceClient_SharedKey();
BlobChangeFeedClient blobChangeFeedClient = service.GetChangeFeedClient();
// Collect all events within range
AsyncPageable <BlobChangeFeedEvent> blobChangeFeedAsyncPagable
= blobChangeFeedClient.GetChangesAsync(
start: startTime,
end: endTime);
ISet <string> AllEventIds = new HashSet <string>();
IAsyncEnumerable <Page <BlobChangeFeedEvent> > asyncEnumerable = blobChangeFeedAsyncPagable.AsPages(pageSizeHint: 50);
Page <BlobChangeFeedEvent> lastPage = null;
await foreach (Page <BlobChangeFeedEvent> page in asyncEnumerable)
{
foreach (BlobChangeFeedEvent e in page.Values)
{
AllEventIds.Add(e.Id.ToString());
}
lastPage = page;
}
string continuation = lastPage.ContinuationToken;
// Act
blobChangeFeedAsyncPagable = blobChangeFeedClient.GetChangesAsync(continuation);
IList <BlobChangeFeedEvent> tail = await blobChangeFeedAsyncPagable.ToListAsync();
// Assert
Assert.AreEqual(0, tail.Count);
Assert.Greater(AllEventIds.Count, 0);
}
public async Task CanGetManifestsWithCustomPageSize(bool anonymous)
{
// Arrange
var client = CreateClient(anonymous);
var repository = client.GetRepository(_repositoryName);
int pageSize = 2;
int minExpectedPages = 2;
// Act
AsyncPageable <ArtifactManifestProperties> artifacts = repository.GetManifestsAsync();
var pages = artifacts.AsPages(pageSizeHint: pageSize);
int pageCount = 0;
await foreach (var page in pages)
{
Assert.GreaterOrEqual(page.Values.Count, pageSize);
pageCount++;
}
// Assert
Assert.IsTrue(pageCount >= minExpectedPages);
}
public async Task CanGetTagsWithCustomPageSize(bool anonymous)
{
// Arrange
var client = CreateClient(anonymous);
string tagName = "latest";
var artifact = client.GetArtifact(_repositoryName, tagName);
int pageSize = 2;
int minExpectedPages = 2;
// Act
AsyncPageable <ArtifactTagProperties> tags = artifact.GetTagsAsync();
var pages = tags.AsPages(pageSizeHint: pageSize);
int pageCount = 0;
await foreach (var page in pages)
{
Assert.IsTrue(page.Values.Count <= pageSize);
pageCount++;
}
// Assert
Assert.IsTrue(pageCount >= minExpectedPages);
}
public async Task QueryEntitiesAsync()
{
string storageUri = StorageUri;
string accountName = StorageAccountName;
string storageAccountKey = PrimaryStorageAccountKey;
string tableName = "OfficeSupplies4p2" + _random.Next();
string partitionKey = "somePartition";
string rowKey = "1";
string rowKey2 = "2";
var serviceClient = new TableServiceClient(
new Uri(storageUri),
new TableSharedKeyCredential(accountName, storageAccountKey));
await serviceClient.CreateTableAsync(tableName);
var tableClient = serviceClient.GetTableClient(tableName);
var entity = new TableEntity(partitionKey, rowKey)
{
{ "Product", "Markers" },
{ "Price", 5.00 },
};
await tableClient.AddEntityAsync(entity);
var entity2 = new TableEntity(partitionKey, rowKey2)
{
{ "Product", "Chair" },
{ "Price", 7.00 },
};
await tableClient.AddEntityAsync(entity2);
#region Snippet:TablesSample4QueryEntitiesAsync
// Use the <see cref="TableClient"> to query the table. Passing in OData filter strings is optional.
AsyncPageable <TableEntity> queryResults = tableClient.QueryAsync <TableEntity>(filter: $"PartitionKey eq '{partitionKey}'");
int count = 0;
// Iterate the list in order to access individual queried entities.
await foreach (TableEntity qEntity in queryResults)
{
Console.WriteLine($"{qEntity.GetString("Product")}: {qEntity.GetDouble("Price")}");
count++;
}
Console.WriteLine($"The query returned {count} entities.");
#endregion
#region Snippet:TablesSample4QueryEntitiesExpressionAsync
// Use the <see cref="TableClient"> to query the table using a filter expression.
double priceCutOff = 6.00;
AsyncPageable <OfficeSupplyEntity> queryResultsLINQ = tableClient.QueryAsync <OfficeSupplyEntity>(ent => ent.Price >= priceCutOff);
#endregion
#region Snippet:TablesSample4QueryEntitiesSelectAsync
AsyncPageable <TableEntity> queryResultsSelect = tableClient.QueryAsync <TableEntity>(select: new List <string>()
{
"Product", "Price"
});
#endregion
#region Snippet:TablesSample4QueryEntitiesMaxPerPageAsync
AsyncPageable <TableEntity> queryResultsMaxPerPage = tableClient.QueryAsync <TableEntity>(maxPerPage: 10);
// Iterate the <see cref="Pageable"> by page.
await foreach (Page <TableEntity> page in queryResultsMaxPerPage.AsPages())
{
Console.WriteLine("This is a new page!");
foreach (TableEntity qEntity in page.Values)
{
Console.WriteLine($"# of {qEntity.GetString("Product")} inventoried: {qEntity.GetInt32("Quantity")}");
}
}
#endregion
await serviceClient.DeleteTableAsync(tableName);
}
public async Task Query_PaginationWorks()
{
DigitalTwinsClient client = GetClient();
int pageSize = 5;
string floorModelId = await GetUniqueModelIdAsync(client, TestAssetDefaults.FloorModelIdPrefix).ConfigureAwait(false);
string roomModelId = await GetUniqueModelIdAsync(client, TestAssetDefaults.RoomModelIdPrefix).ConfigureAwait(false);
try
{
// Create room model
string roomModel = TestAssetsHelper.GetRoomModelPayload(roomModelId, floorModelId);
await client.CreateModelsAsync(new List <string> {
roomModel
}).ConfigureAwait(false);
// Create a room twin, with property "IsOccupied": true
string roomTwin = TestAssetsHelper.GetRoomTwinPayload(roomModelId);
for (int i = 0; i < pageSize + 1; i++)
{
string roomTwinId = await GetUniqueTwinIdAsync(client, TestAssetDefaults.RoomTwinIdPrefix).ConfigureAwait(false);
await client.CreateDigitalTwinAsync(roomTwinId, roomTwin).ConfigureAwait(false);
}
string queryString = "SELECT * FROM digitaltwins";
// act
var options = new QueryOptions();
options.MaxItemsPerPage = pageSize;
AsyncPageable <string> asyncPageableResponse = client.QueryAsync(queryString, options);
// assert
// Test that page size hint works, and that all returned pages either have the page size hint amount of
// elements, or have no continuation token (signaling that it is the last page)
int pageCount = 0;
await foreach (Page <string> page in asyncPageableResponse.AsPages())
{
pageCount++;
if (page.ContinuationToken != null)
{
page.Values.Count.Should().Be(pageSize, "Unexpected page size for a non-terminal page");
}
}
pageCount.Should().BeGreaterThan(1, "Expected more than one page of query results");
}
finally
{
// clean up
try
{
if (!string.IsNullOrWhiteSpace(roomModelId))
{
await client.DeleteModelAsync(roomModelId).ConfigureAwait(false);
}
}
catch (Exception ex)
{
Assert.Fail($"Test clean up failed: {ex.Message}");
}
}
}
public async Task Relationships_PaginationWorks()
{
DigitalTwinsClient client = GetClient();
string floorModelId = await GetUniqueModelIdAsync(client, TestAssetDefaults.FloorModelIdPrefix).ConfigureAwait(false);
string roomModelId = await GetUniqueModelIdAsync(client, TestAssetDefaults.RoomModelIdPrefix).ConfigureAwait(false);
string hvacModelId = await GetUniqueTwinIdAsync(client, TestAssetDefaults.HvacModelIdPrefix).ConfigureAwait(false);
string floorTwinId = await GetUniqueTwinIdAsync(client, TestAssetDefaults.FloorTwinIdPrefix).ConfigureAwait(false);
string roomTwinId = await GetUniqueTwinIdAsync(client, TestAssetDefaults.RoomTwinIdPrefix).ConfigureAwait(false);
string hvacTwinId = await GetUniqueTwinIdAsync(client, TestAssetDefaults.HvacTwinIdPrefix).ConfigureAwait(false);
try
{
// create floor, room and hvac model
string floorModel = TestAssetsHelper.GetFloorModelPayload(floorModelId, roomModelId, hvacModelId);
string roomModel = TestAssetsHelper.GetRoomModelPayload(roomModelId, floorModelId);
string hvacModel = TestAssetsHelper.GetHvacModelPayload(hvacModelId, floorModelId);
await client.CreateModelsAsync(new List <string> {
floorModel, roomModel, hvacModel
}).ConfigureAwait(false);
// create floor twin
BasicDigitalTwin floorTwin = TestAssetsHelper.GetFloorTwinPayload(floorModelId);
await client.CreateDigitalTwinAsync <BasicDigitalTwin>(floorTwinId, floorTwin).ConfigureAwait(false);
// Create room twin
BasicDigitalTwin roomTwin = TestAssetsHelper.GetRoomTwinPayload(roomModelId);
await client.CreateDigitalTwinAsync <BasicDigitalTwin>(roomTwinId, roomTwin).ConfigureAwait(false);
BasicRelationship floorContainsRoomPayload = TestAssetsHelper.GetRelationshipWithPropertyPayload(roomTwinId, ContainsRelationship, "isAccessRestricted", true);
BasicRelationship floorTwinContainedInRelationshipPayload = TestAssetsHelper.GetRelationshipPayload(floorTwinId, ContainedInRelationship);
// For the sake of test simplicity, we'll just add multiple relationships from the same floor to the same room.
for (int i = 0; i < bulkRelationshipCount; i++)
{
var floorContainsRoomRelationshipId = $"FloorToRoomRelationship-{GetRandom()}";
// create Relationship from Floor -> Room
await client
.CreateRelationshipAsync <BasicRelationship>(
floorTwinId,
floorContainsRoomRelationshipId,
floorContainsRoomPayload)
.ConfigureAwait(false);
}
// For the sake of test simplicity, we'll just add multiple relationships from the same room to the same floor.
for (int i = 0; i < bulkRelationshipCount; i++)
{
var roomContainedInFloorRelationshipId = $"RoomToFloorRelationship-{GetRandom()}";
// create Relationship from Room -> Floor
await client
.CreateRelationshipAsync <BasicRelationship>(
roomTwinId,
roomContainedInFloorRelationshipId,
floorTwinContainedInRelationshipPayload)
.ConfigureAwait(false);
}
// LIST incoming relationships by page
AsyncPageable <IncomingRelationship> incomingRelationships = client.GetIncomingRelationshipsAsync(floorTwinId);
int incomingRelationshipPageCount = 0;
await foreach (Page <IncomingRelationship> incomingRelationshipPage in incomingRelationships.AsPages())
{
incomingRelationshipPageCount++;
if (incomingRelationshipPage.ContinuationToken != null)
{
incomingRelationshipPage.Values.Count.Should().Be(defaultRelationshipPageSize, "Unexpected page size for a non-terminal page");
}
}
incomingRelationshipPageCount.Should().BeGreaterThan(1, "Expected more than one page of incoming relationships");
// LIST outgoing relationships by page
AsyncPageable <string> outgoingRelationships = client.GetRelationshipsAsync(floorTwinId);
int outgoingRelationshipPageCount = 0;
await foreach (Page <string> outgoingRelationshipPage in outgoingRelationships.AsPages())
{
outgoingRelationshipPageCount++;
if (outgoingRelationshipPage.ContinuationToken != null)
{
outgoingRelationshipPage.Values.Count.Should().Be(defaultRelationshipPageSize, "Unexpected page size for a non-terminal page");
}
}
outgoingRelationshipPageCount.Should().BeGreaterThan(1, "Expected more than one page of outgoing relationships");
}
finally
{
// clean up
try
{
await Task
.WhenAll(
client.DeleteDigitalTwinAsync(floorTwinId),
client.DeleteDigitalTwinAsync(roomTwinId),
client.DeleteDigitalTwinAsync(hvacTwinId),
client.DeleteModelAsync(hvacModelId),
client.DeleteModelAsync(floorModelId),
client.DeleteModelAsync(roomModelId))
.ConfigureAwait(false);
}
catch (Exception ex)
{
Assert.Fail($"Test clean up failed: {ex.Message}");
}
}
}
/// <summary>
/// This method is called each polling interval for all containers. The method divides the
/// budget of allocated number of blobs to query, for each container we query a page of
/// that size and we keep the continuation token for the next time. AS a curser, we use
/// the time stamp when the current cycle on the container started. blobs newer than that
/// time will be considered new and registrations will be notified
/// </summary>
/// <param name="container"></param>
/// <param name="containerScanInfo"> Information that includes the last cycle start
/// the continuation token and the current cycle start for a container</param>
/// <param name="clientRequestId"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task <IEnumerable <BlobBaseClient> > PollNewBlobsAsync(
BlobContainerClient container, ContainerScanInfo containerScanInfo, string clientRequestId, CancellationToken cancellationToken)
{
IEnumerable <BlobItem> currentBlobs;
int blobPollLimitPerContainer = _scanBlobLimitPerPoll / _scanInfo.Count;
string continuationToken = containerScanInfo.ContinuationToken;
Page <BlobItem> page;
// if starting the cycle, reset the sweep time
if (continuationToken == null)
{
containerScanInfo.CurrentSweepCycleLatestModified = DateTime.MinValue;
}
Stopwatch sw = Stopwatch.StartNew();
try
{
AsyncPageable <BlobItem> blobsAsyncPageable = container.GetBlobsAsync(cancellationToken: cancellationToken);
IAsyncEnumerable <Page <BlobItem> > pages = blobsAsyncPageable.AsPages(continuationToken: continuationToken, pageSizeHint: blobPollLimitPerContainer);
IAsyncEnumerator <Page <BlobItem> > pagesEnumerator = pages.GetAsyncEnumerator(cancellationToken);
if (await pagesEnumerator.MoveNextAsync().ConfigureAwait(false))
{
page = pagesEnumerator.Current;
currentBlobs = page.Values;
}
else
{
return(Enumerable.Empty <BlobBaseClient>());
}
}
catch (RequestFailedException exception)
{
if (exception.IsNotFound())
{
Logger.ContainerDoesNotExist(_logger, container.Name);
return(Enumerable.Empty <BlobBaseClient>());
}
else
{
throw;
}
}
List <BlobBaseClient> newBlobs = new List <BlobBaseClient>();
// Type cast to IStorageBlob is safe due to useFlatBlobListing: true above.
foreach (BlobItem currentBlob in currentBlobs)
{
cancellationToken.ThrowIfCancellationRequested();
var properties = currentBlob.Properties;
DateTime lastModifiedTimestamp = properties.LastModified.Value.UtcDateTime;
if (lastModifiedTimestamp > containerScanInfo.CurrentSweepCycleLatestModified)
{
containerScanInfo.CurrentSweepCycleLatestModified = lastModifiedTimestamp;
}
// Blob timestamps are rounded to the nearest second, so make sure we continue to check
// the previous timestamp to catch any blobs that came in slightly after our previous poll.
if (lastModifiedTimestamp >= containerScanInfo.LastSweepCycleLatestModified)
{
newBlobs.Add(container.GetBlobClient(currentBlob.Name));
}
}
Logger.PollBlobContainer(_logger, container.Name, containerScanInfo.LastSweepCycleLatestModified, clientRequestId,
newBlobs.Count, sw.ElapsedMilliseconds, !string.IsNullOrWhiteSpace(page.ContinuationToken));
// record continuation token for next chunk retrieval
containerScanInfo.ContinuationToken = page.ContinuationToken;
// if ending a cycle then copy currentSweepCycleStartTime to lastSweepCycleStartTime, if changed
if (page.ContinuationToken == null &&
containerScanInfo.CurrentSweepCycleLatestModified > containerScanInfo.LastSweepCycleLatestModified)
{
containerScanInfo.LastSweepCycleLatestModified = containerScanInfo.CurrentSweepCycleLatestModified;
}
return(newBlobs);
}
public async Task Query_PaginationWorks()
{
DigitalTwinsClient client = GetClient();
int pageSize = 5;
string floorModelId = await GetUniqueModelIdAsync(client, TestAssetDefaults.FloorModelIdPrefix).ConfigureAwait(false);
string roomModelId = await GetUniqueModelIdAsync(client, TestAssetDefaults.RoomModelIdPrefix).ConfigureAwait(false);
TimeSpan QueryWaitTimeout = TimeSpan.FromMinutes(1); // Wait at most one minute for the created twins to become queryable
try
{
// Create room model
string roomModel = TestAssetsHelper.GetRoomModelPayload(roomModelId, floorModelId);
await CreateAndListModelsAsync(client, new List <string> {
roomModel
}).ConfigureAwait(false);
// Create a room twin, with property "IsOccupied": true
BasicDigitalTwin roomTwin = TestAssetsHelper.GetRoomTwinPayload(roomModelId);
for (int i = 0; i < pageSize * 2; i++)
{
string roomTwinId = await GetUniqueTwinIdAsync(client, TestAssetDefaults.RoomTwinIdPrefix).ConfigureAwait(false);
await client.CreateOrReplaceDigitalTwinAsync <BasicDigitalTwin>(roomTwinId, roomTwin).ConfigureAwait(false);
}
string queryString = "SELECT * FROM digitaltwins";
// act
CancellationTokenSource queryTimeoutCancellationToken = new CancellationTokenSource(QueryWaitTimeout);
bool queryHasExpectedCount = false;
while (!queryHasExpectedCount)
{
if (queryTimeoutCancellationToken.IsCancellationRequested)
{
throw new AssertionException($"Timed out waiting for at least {pageSize + 1} twins to be queryable");
}
AsyncPageable <BasicDigitalTwin> asyncPageableResponse = client.QueryAsync <BasicDigitalTwin>(queryString, queryTimeoutCancellationToken.Token);
int count = 0;
await foreach (Page <BasicDigitalTwin> queriedTwinPage in asyncPageableResponse.AsPages(pageSizeHint: pageSize))
{
count += queriedTwinPage.Values.Count;
}
// Once at least (page + 1) twins are query-able, then page size control can be tested.
queryHasExpectedCount = count >= pageSize + 1;
}
// assert
// Test that page size hint works, and that all returned pages either have the page size hint amount of
// elements, or have no continuation token (signaling that it is the last page)
int pageCount = 0;
await foreach (Page <BasicDigitalTwin> page in client.QueryAsync <BasicDigitalTwin>(queryString).AsPages(pageSizeHint: pageSize))
{
pageCount++;
if (page.ContinuationToken != null)
{
page.Values.Count.Should().Be(pageSize, "Unexpected page size for a non-terminal page");
}
}
pageCount.Should().BeGreaterThan(1, "Expected more than one page of query results");
}
catch (Exception ex)
{
Assert.Fail($"Failure in executing a step in the test case: {ex.Message}.");
}
finally
{
// clean up
try
{
if (!string.IsNullOrWhiteSpace(roomModelId))
{
await client.DeleteModelAsync(roomModelId).ConfigureAwait(false);
}
}
catch (Exception ex)
{
Assert.Fail($"Test clean up failed: {ex.Message}");
}
}
}
public async Task QueryTwinsAsync()
{
PrintHeader("QUERY DIGITAL TWINS");
try
{
Console.WriteLine("Making a twin query and iterating over the results.");
#region Snippet:DigitalTwinsSampleQueryTwins
// This code snippet demonstrates the simplest way to iterate over the digital twin results, where paging
// happens under the covers.
AsyncPageable <string> asyncPageableResponse = client.QueryAsync("SELECT * FROM digitaltwins");
// Iterate over the twin instances in the pageable response.
// The "await" keyword here is required because new pages will be fetched when necessary,
// which involves a request to the service.
await foreach (string response in asyncPageableResponse)
{
BasicDigitalTwin twin = JsonSerializer.Deserialize <BasicDigitalTwin>(response);
Console.WriteLine($"Found digital twin: {twin.Id}");
}
#endregion Snippet:DigitalTwinsSampleQueryTwins
Console.WriteLine("Making a twin query, with query-charge header extraction.");
#region Snippet:DigitalTwinsSampleQueryTwinsWithQueryCharge
// This code snippet demonstrates how you could extract the query charges incurred when calling
// the query API. It iterates over the response pages first to access to the query-charge header,
// and then the digital twin results within each page.
AsyncPageable <string> asyncPageableResponseWithCharge = client.QueryAsync("SELECT * FROM digitaltwins");
int pageNum = 0;
// The "await" keyword here is required as a call is made when fetching a new page.
await foreach (Page <string> page in asyncPageableResponseWithCharge.AsPages())
{
Console.WriteLine($"Page {++pageNum} results:");
// Extract the query-charge header from the page
if (QueryChargeHelper.TryGetQueryCharge(page, out float queryCharge))
{
Console.WriteLine($"Query charge was: {queryCharge}");
}
// Iterate over the twin instances.
// The "await" keyword is not required here as the paged response is local.
foreach (string response in page.Values)
{
BasicDigitalTwin twin = JsonSerializer.Deserialize <BasicDigitalTwin>(response);
Console.WriteLine($"Found digital twin: {twin.Id}");
}
}
#endregion Snippet:DigitalTwinsSampleQueryTwinsWithQueryCharge
}
catch (Exception ex)
{
FatalError($"Could not query digital twins due to {ex}");
}
}
public async Task TestTailEvents()
{
// Uncomment when recording.
//DateTimeOffset startTime = DateTimeOffset.Now;
// Update and uncomment after recording.
DateTimeOffset startTime = new DateTimeOffset(2020, 8, 10, 16, 00, 00, TimeSpan.Zero);
TimeSpan pollInterval = Mode == RecordedTestMode.Playback ? TimeSpan.Zero : TimeSpan.FromMinutes(3);
BlobServiceClient service = GetServiceClient_SharedKey();
BlobChangeFeedClient blobChangeFeedClient = service.GetChangeFeedClient();
Page <BlobChangeFeedEvent> lastPage = null;
ISet <string> EventIdsPart1 = new HashSet <string>();
ISet <string> EventIdsPart2 = new HashSet <string>();
ISet <string> EventIdsPart3 = new HashSet <string>();
// Part 1
AsyncPageable <BlobChangeFeedEvent> blobChangeFeedAsyncPagable = blobChangeFeedClient.GetChangesAsync(start: startTime);
IAsyncEnumerable <Page <BlobChangeFeedEvent> > asyncEnumerable = blobChangeFeedAsyncPagable.AsPages();
await foreach (var page in asyncEnumerable)
{
lastPage = page;
foreach (var evt in page.Values)
{
EventIdsPart1.Add(evt.Id.ToString());
}
}
CollectionAssert.IsNotEmpty(EventIdsPart1);
await Task.Delay(pollInterval);
// Part 2
blobChangeFeedAsyncPagable = blobChangeFeedClient.GetChangesAsync(lastPage.ContinuationToken);
asyncEnumerable = blobChangeFeedAsyncPagable.AsPages();
await foreach (var page in asyncEnumerable)
{
lastPage = page;
foreach (var evt in page.Values)
{
EventIdsPart2.Add(evt.Id.ToString());
}
}
CollectionAssert.IsNotEmpty(EventIdsPart2);
await Task.Delay(pollInterval);
// Part 3
blobChangeFeedAsyncPagable = blobChangeFeedClient.GetChangesAsync(lastPage.ContinuationToken);
asyncEnumerable = blobChangeFeedAsyncPagable.AsPages();
await foreach (var page in asyncEnumerable)
{
lastPage = page;
foreach (var evt in page.Values)
{
EventIdsPart3.Add(evt.Id.ToString());
}
}
CollectionAssert.IsNotEmpty(EventIdsPart3);
// Assert events are not duplicated
CollectionAssert.IsEmpty(EventIdsPart1.Intersect(EventIdsPart2));
CollectionAssert.IsEmpty(EventIdsPart1.Intersect(EventIdsPart3));
CollectionAssert.IsEmpty(EventIdsPart2.Intersect(EventIdsPart3));
}
public async Task ResumeFromTheMiddleOfTheChunkWithManyNonEmptyShards()
{
// This is hardcoded for playback stability. Feel free to modify but make sure recordings match.
DateTimeOffset startTime = new DateTimeOffset(2020, 8, 5, 17, 00, 00, TimeSpan.Zero);
DateTimeOffset endTime = new DateTimeOffset(2020, 8, 5, 17, 15, 00, TimeSpan.Zero);
int expectedShardCount = 3;
BlobServiceClient service = GetServiceClient_SharedKey();
BlobChangeFeedClient blobChangeFeedClient = service.GetChangeFeedClient();
// Collect all events within range
AsyncPageable <BlobChangeFeedEvent> blobChangeFeedAsyncPagable
= blobChangeFeedClient.GetChangesAsync(
start: startTime,
end: endTime);
ISet <string> AllEventIds = new HashSet <string>();
await foreach (BlobChangeFeedEvent e in blobChangeFeedAsyncPagable)
{
AllEventIds.Add(e.Id.ToString());
}
// Iterate over first two pages
ISet <string> EventIdsPart1 = new HashSet <string>();
blobChangeFeedAsyncPagable = blobChangeFeedClient.GetChangesAsync(
start: startTime,
end: endTime);
IAsyncEnumerable <Page <BlobChangeFeedEvent> > asyncEnumerable = blobChangeFeedAsyncPagable.AsPages(pageSizeHint: 50);
Page <BlobChangeFeedEvent> lastPage = null;
int pages = 0;
await foreach (Page <BlobChangeFeedEvent> page in asyncEnumerable)
{
foreach (BlobChangeFeedEvent e in page.Values)
{
EventIdsPart1.Add(e.Id.ToString());
}
pages++;
lastPage = page;
if (pages > 2)
{
break;
}
}
string continuation = lastPage.ContinuationToken;
var currentSegmentCursor = (JsonSerializer.Deserialize(continuation, typeof(ChangeFeedCursor)) as ChangeFeedCursor).CurrentSegmentCursor;
Assert.AreEqual(currentSegmentCursor.ShardCursors.Count, expectedShardCount);
Assert.IsNotNull(currentSegmentCursor.ShardCursors.Find(x => x.BlockOffset > 0), "Making sure we actually finish some shard in the middle of chunk, if this fails play with test data to make it pass");
// Iterate over next two pages
ISet <string> EventIdsPart2 = new HashSet <string>();
blobChangeFeedAsyncPagable = blobChangeFeedClient.GetChangesAsync(continuation);
asyncEnumerable = blobChangeFeedAsyncPagable.AsPages(pageSizeHint: 50);
lastPage = null;
pages = 0;
await foreach (Page <BlobChangeFeedEvent> page in asyncEnumerable)
{
foreach (BlobChangeFeedEvent e in page.Values)
{
EventIdsPart2.Add(e.Id.ToString());
}
pages++;
lastPage = page;
if (pages > 2)
{
break;
}
}
continuation = lastPage.ContinuationToken;
currentSegmentCursor = (JsonSerializer.Deserialize(continuation, typeof(ChangeFeedCursor)) as ChangeFeedCursor).CurrentSegmentCursor;
Assert.AreEqual(currentSegmentCursor.ShardCursors.Count, expectedShardCount);
Assert.IsNotNull(currentSegmentCursor.ShardCursors.Find(x => x.BlockOffset > 0), "Making sure we actually finish some shard in the middle of chunk, if this fails play with test data to make it pass");
// Iterate over remaining
ISet <string> EventIdsTail = new HashSet <string>();
AsyncPageable <BlobChangeFeedEvent> cursorBlobChangeFeedAsyncPagable
= blobChangeFeedClient.GetChangesAsync(continuation);
IList <BlobChangeFeedEvent> list = await cursorBlobChangeFeedAsyncPagable.ToListAsync();
foreach (BlobChangeFeedEvent e in list)
{
EventIdsTail.Add(e.Id.ToString());
}
ISet <string> AllEventIdsFromResumingIteration = new HashSet <string>();
AllEventIdsFromResumingIteration.UnionWith(EventIdsPart1);
AllEventIdsFromResumingIteration.UnionWith(EventIdsPart2);
AllEventIdsFromResumingIteration.UnionWith(EventIdsTail);
Assert.Greater(AllEventIds.Count, 0);
Assert.Greater(EventIdsPart1.Count, 0);
Assert.Greater(EventIdsPart2.Count, 0);
Assert.Greater(EventIdsTail.Count, 0);
Assert.AreEqual(AllEventIds.Count, EventIdsPart1.Count + EventIdsPart2.Count + EventIdsTail.Count);
CollectionAssert.AreEqual(AllEventIds, AllEventIdsFromResumingIteration);
}
