private static void ValidateCollection <T>(DocumentFeedResponse <T>[] collectionResponses) where T : Resource, new()
{
for (int i = 0; i < collectionResponses.Length - 1; i++)
{
for (int j = i + 1; j < collectionResponses.Length; j++)
{
Assert.AreEqual(collectionResponses[i].ResponseContinuation, collectionResponses[j].ResponseContinuation, "Continuation(s) between collection response mismatch");
Assert.AreEqual(collectionResponses[i].Count, collectionResponses[j].Count, "Count between collection response mismatch");
}
}
for (int i = 0; i < collectionResponses[0].Count; ++i)
{
List <T> resources = collectionResponses.Select(t => t.ElementAt(i)).ToList();
ReplicationTests.ValidateResourceProperties(resources);
}
}
internal static async Task WaitForReIndexingToFinish(
int maxWaitDurationInSeconds,
CosmosContainerSettings collection)
{
await Task.Delay(TimeSpan.FromSeconds(5));
int currentWaitSeconds = 0;
var lockedClients = ReplicationTests.GetClientsLocked();
for (int index = 0; index < lockedClients.Length; ++index)
{
Logger.LogLine("Client: " + index);
while (true)
{
long reindexerProgress = (await TestCommon.AsyncRetryRateLimiting(
() => lockedClients[index].ReadDocumentCollectionAsync(collection.SelfLink, new RequestOptions {
PopulateQuotaInfo = true
}))).IndexTransformationProgress;
Logger.LogLine("Progress: " + reindexerProgress);
if (reindexerProgress == -1)
{
throw new Exception("Failed to obtain the reindexer progress.");
}
else if (reindexerProgress == 100)
{
Logger.LogLine("ReIndexing finished after: " + currentWaitSeconds + " seconds");
break;
}
else
{
await Task.Delay(TimeSpan.FromSeconds(1));
currentWaitSeconds++;
Logger.LogLine("ReIndexing still running after: " + currentWaitSeconds + " seconds");
if (currentWaitSeconds > maxWaitDurationInSeconds)
{
throw new Exception("ReIndexing did not complete after: " + maxWaitDurationInSeconds + " seconds");
}
}
}
}
}
internal static async Task WaitForLazyIndexingToCompleteAsync(CosmosContainerSettings collection)
{
TimeSpan maxWaitTime = TimeSpan.FromMinutes(10);
TimeSpan sleepTimeBetweenReads = TimeSpan.FromSeconds(1);
// First wait for replication to complete
TestCommon.WaitForServerReplication();
var lockedClients = ReplicationTests.GetClientsLocked();
for (int index = 0; index < lockedClients.Length; ++index)
{
Logger.LogLine("Client: " + index);
while (true)
{
long lazyIndexingProgress = (await lockedClients[index].ReadDocumentCollectionAsync(collection, new RequestOptions {
PopulateQuotaInfo = true
})).LazyIndexingProgress;
if (lazyIndexingProgress == -1)
{
throw new Exception("Failed to obtain the lazy indexing progress.");
}
else if (lazyIndexingProgress == 100)
{
Logger.LogLine("Indexing completed at {0}", DateTime.Now.ToString("HH:mm:ss.f", CultureInfo.InvariantCulture));
break;
}
else
{
Logger.LogLine("Obtained the lazy indexing progress: {0}. Sleep for {1} seconds", lazyIndexingProgress, sleepTimeBetweenReads.TotalSeconds);
await Task.Delay(sleepTimeBetweenReads);
maxWaitTime -= sleepTimeBetweenReads;
if (maxWaitTime.TotalMilliseconds <= 0)
{
throw new Exception("Indexing didn't complete within the allocated time");
}
}
}
}
}
internal static void ValidateCollection <T>(DocumentClient[] replicaClients,
string collectionId,
INameValueCollection headers = null,
bool verifyAddress = true) where T : Resource, new()
{
Assert.IsTrue(replicaClients != null && replicaClients.Length > 1, "Must pass in at least two replica clients");
Task.Delay(3000); // allow previous operations to complete and propagate, bringing more robustness to tests
foreach (DocumentClient client in replicaClients)
{
client.ForceAddressRefresh(true);
if (typeof(T) == typeof(StoredProcedure) ||
typeof(T) == typeof(UserDefinedFunction) ||
typeof(T) == typeof(Trigger))
{
TestCommon.ListAllScriptDirect <T>(client, collectionId, headers ?? new StringKeyValueCollection());
}
else
{
TestCommon.ListAll <T>(client, collectionId, headers ?? new StringKeyValueCollection(), true);
}
}
try
{
BackoffRetryUtility <bool> .ExecuteAsync((bool isInRetry) =>
{
var feeds = new DocumentFeedResponse <T> [replicaClients.Length];
var allHeaders = new StringKeyValueCollection[replicaClients.Length];
for (int i = 0; i < replicaClients.Length; i++)
{
var header = new StringKeyValueCollection();
if (headers != null)
{
foreach (string key in headers)
{
header[key] = headers[key];
}
}
allHeaders[i] = header;
}
var continuations = new string[replicaClients.Length];
var responseHeaders = new INameValueCollection[replicaClients.Length];
do
{
for (int i = 0; i < replicaClients.Length; i++)
{
if (!string.IsNullOrEmpty(continuations[i]))
{
allHeaders[i][HttpConstants.HttpHeaders.Continuation] = continuations[i];
}
feeds[i] = replicaClients[i].ReadFeedWithRetry <T>(collectionId, out responseHeaders[i],
allHeaders[i]);
if (responseHeaders[i] != null)
{
continuations[i] = responseHeaders[i][HttpConstants.HttpHeaders.Continuation];
}
else
{
continuations[i] = null;
}
}
for (int i = 0; i < replicaClients.Length - 1; i++)
{
for (int j = i + 1; j < replicaClients.Length; j++)
{
Assert.AreEqual(continuations[i], continuations[j], "Collection Continuaton mismatch");
if (verifyAddress)
{
var address1 = replicaClients[i].GetAddress();
var address2 = replicaClients[j].GetAddress();
// If the addresses match, we are in mid of reconfiguration, throw GoneException so that we retry on the entire loop
// and take care of intermittent gone at the same time.
if (address1.Equals(address2))
{
throw new GoneException("Addresses matched for multiple replicas " + address1);
}
}
}
}
ReplicationTests.ValidateCollection <T>(feeds);
} while (!string.IsNullOrEmpty(continuations[0]));
return(Task.FromResult <bool>(true));
}, new GoneAndRetryWithRetryPolicy()).Wait();
}
finally
{
//Once address are stable. dont force the cache refresh.
foreach (DocumentClient client in replicaClients)
{
client.ForceAddressRefresh(false);
}
}
}
internal static void ValidateResourceProperties <T>(List <T> resources) where T : Resource, new()
{
for (int i = 0; i < resources.Count - 1; i++)
{
for (int j = i + 1; j < resources.Count; j++)
{
// First validate resource properties
Assert.AreEqual(resources[i].ResourceId, resources[j].ResourceId, "RID mismatched");
Assert.AreEqual(resources[i].ETag, resources[j].ETag, "ETag mismatched");
Assert.AreEqual(resources[i].Id, resources[j].Id, "Name mismatched");
Assert.AreEqual(resources[i].SelfLink, resources[j].SelfLink, "SelfLink mismatched");
Assert.AreEqual(resources[i].Timestamp, resources[j].Timestamp, "Timestamp mismatched");
}
}
if (typeof(T) == typeof(Database))
{
var databases = resources.Cast <Database>().ToArray();
for (int i = 0; i < resources.Count - 1; i++)
{
for (int j = i + 1; j < resources.Count; j++)
{
Assert.AreEqual(databases[i].CollectionsLink, databases[j].CollectionsLink,
"Database CollectionLink don't match");
}
}
}
else if (typeof(T) == typeof(DocumentCollection))
{
var documentCollections = resources.Cast <DocumentCollection>().ToArray();
for (int i = 0; i < documentCollections.Length - 1; i++)
{
for (int j = i + 1; j < documentCollections.Length; j++)
{
Assert.AreEqual(documentCollections[i].DocumentsLink,
documentCollections[j].DocumentsLink,
"DocumentCollection DocumentsLink mismatch");
Assert.AreEqual(documentCollections[i].IndexingPolicy.Automatic,
documentCollections[j].IndexingPolicy.Automatic,
"DocumentCollection IndexingPolicy.Automatic mismatch");
Assert.AreEqual(documentCollections[i].IndexingPolicy.IndexingMode,
documentCollections[j].IndexingPolicy.IndexingMode,
"DocumentCollection IndexingPolicy.IndexingMode mismatch");
// TODO: nemanjam, add other collection properties
}
}
}
else if (typeof(T) == typeof(User))
{
var users = resources.Cast <User>().ToArray();
for (int i = 0; i < users.Length - 1; i++)
{
for (int j = i + 1; j < users.Length; j++)
{
Assert.AreEqual(users[i].PermissionsLink, users[j].PermissionsLink,
"User PermissionsLink mismatch");
}
}
}
else if (typeof(T) == typeof(Permission))
{
var permissions = resources.Cast <Permission>().ToArray();
for (int i = 0; i < permissions.Length - 1; i++)
{
for (int j = i + 1; j < permissions.Length; j++)
{
Assert.AreEqual(permissions[i].PermissionMode, permissions[j].PermissionMode,
"Permission PermissionMode mismatch");
Assert.AreEqual(permissions[i].ResourceLink, permissions[j].ResourceLink,
"Permission ResourceLink mismatch");
}
}
}
else if (typeof(T) == typeof(Document))
{
var documents = resources.Cast <Document>().ToArray();
for (int i = 0; i < documents.Length - 1; i++)
{
for (int j = i + 1; j < documents.Length; j++)
{
Assert.AreEqual(documents[i].AttachmentsLink, documents[j].AttachmentsLink,
"Document AttachmentsLink mismatch");
}
}
//TODO, KRAMAN, ADD validation for document content.
}
else if (typeof(T) == typeof(Attachment))
{
var attachments = resources.Cast <Attachment>().ToArray();
for (int i = 0; i < attachments.Length - 1; i++)
{
for (int j = i + 1; j < attachments.Length; j++)
{
Assert.AreEqual(attachments[i].ContentType, attachments[j].ContentType,
"Attachment ContentType mismatch");
Assert.AreEqual(attachments[i].MediaLink, attachments[j].MediaLink,
"Attachment MediaLink mismatch");
}
}
}
else if (typeof(T) == typeof(StoredProcedure))
{
var storedProcedures = resources.Cast <StoredProcedure>().ToArray();
for (int i = 0; i < storedProcedures.Length - 1; i++)
{
for (int j = i + 1; j < storedProcedures.Length; j++)
{
Assert.AreEqual(storedProcedures[i].Body, storedProcedures[j].Body,
"StoredProcedure Body mismatch");
}
}
}
else if (typeof(T) == typeof(Trigger))
{
var triggers = resources.Cast <Trigger>().ToArray();
for (int i = 0; i < triggers.Length - 1; i++)
{
for (int j = i + 1; j < triggers.Length; j++)
{
Assert.AreEqual(triggers[i].Body, triggers[j].Body, "Trigger Body mismatch");
}
}
}
else if (typeof(T) == typeof(UserDefinedFunction))
{
var userDefinedFunctions = resources.Cast <UserDefinedFunction>().ToArray();
for (int i = 0; i < userDefinedFunctions.Length - 1; i++)
{
for (int j = i + 1; j < userDefinedFunctions.Length; j++)
{
Assert.AreEqual(userDefinedFunctions[i].Body, userDefinedFunctions[j].Body,
"UserDefinedFunction Body mismatch");
}
}
}
else if (typeof(T) == typeof(Conflict))
{
var conflicts = resources.Cast <Conflict>().ToArray();
for (int i = 0; i < conflicts.Length - 1; i++)
{
for (int j = i + 1; j < conflicts.Length; j++)
{
Assert.AreEqual(conflicts[i].ResourceType, conflicts[j].ResourceType,
"Conflict ResourceType mismatch");
Assert.AreEqual(conflicts[i].OperationKind, conflicts[j].OperationKind,
"Conflict OperationKind mismatch");
Assert.AreEqual(conflicts[i].ResourceId, conflicts[j].ResourceId,
"Conflict ResourceId mismatch");
Assert.AreEqual(conflicts[i].OperationKind, conflicts[j].OperationKind,
"Conflict OperationKind mismatch");
if (conflicts[i].OperationKind == Documents.OperationKind.Delete)
{
continue;
}
if (conflicts[i].ResourceType == typeof(Attachment))
{
ReplicationTests.ValidateResourceProperties <Attachment>(
conflicts.Select(x => x.GetResource <Attachment>()).ToList());
}
else if (conflicts[i].ResourceType == typeof(Document))
{
ReplicationTests.ValidateResourceProperties <Document>(
conflicts.Select(x => x.GetResource <Document>()).ToList());
}
else if (conflicts[i].ResourceType == typeof(StoredProcedure))
{
ReplicationTests.ValidateResourceProperties <StoredProcedure>(
conflicts.Select(x => x.GetResource <StoredProcedure>()).ToList());
}
else if (conflicts[i].ResourceType == typeof(Trigger))
{
ReplicationTests.ValidateResourceProperties <Trigger>(
conflicts.Select(x => x.GetResource <Trigger>()).ToList());
}
else if (conflicts[i].ResourceType == typeof(UserDefinedFunction))
{
ReplicationTests.ValidateResourceProperties <UserDefinedFunction>(
conflicts.Select(x => x.GetResource <UserDefinedFunction>()).ToList());
}
else
{
Assert.Fail("Invalid resource type {0}", conflicts[i].ResourceType);
}
}
}
}
}