/// <summary>
/// Helper function to validate whether "retrievedEntity" is correct or not.
/// </summary>
/// <param name="retrievedEntity"></param>
/// <param name="jobType"></param>
/// <param name="jobId"></param>
/// <param name="message"></param>
/// <param name="partition"></param>
/// <param name="row"></param>
/// <param name="checkViewId"></param>
protected void ValidateRetrievedRTableEntity(
SampleRTableEntity retrievedEntity,
string jobType,
string jobId,
string message,
int partition,
int row,
bool checkViewId = true)
{
Assert.AreEqual(
this.GenerateJobType(jobType, partition),
retrievedEntity.JobType,
"JobType of retrievedEntity mismatch");
Assert.AreEqual(
this.GenerateJobId(jobId, partition, row),
retrievedEntity.JobId,
"JobId of retrievedEntity mismatch");
Assert.AreEqual(
this.GenerateMessage(message, partition, row),
retrievedEntity.Message,
"Message of retrievedEntity mismatch");
if (checkViewId)
{
Assert.AreEqual(
this.configurationWrapper.GetReadView().ViewId,
retrievedEntity._rtable_ViewId,
"_rtable_ViewId of retrievedEntity mismatch");
}
}
public void RTableRepairRow()
{
// Insert entity
Assert.IsTrue(this.repTable.Exists(), "RTable does not exist");
View fullView = configurationWrapper.GetWriteView();
List <ReplicaInfo> fullViewReplicas = new List <ReplicaInfo>();
for (int i = 0; i <= fullView.TailIndex; i++)
{
fullViewReplicas.Add(fullView.GetReplicaInfo(i));
}
List <ReplicaInfo> newReplicas = new List <ReplicaInfo>();
for (int i = 1; i <= fullView.TailIndex; i++)
{
newReplicas.Add(fullView.GetReplicaInfo(i));
}
this.UpdateConfiguration(newReplicas, 0);
Assert.IsTrue(configurationWrapper.IsViewStable());
SampleRTableEntity newCustomer = new SampleRTableEntity("firstname1", "lastname1", "*****@*****.**");
TableOperation operation = TableOperation.Insert(newCustomer);
TableResult result = repTable.Execute(operation);
Assert.AreNotEqual(null, result, "result = null");
ReplicatedTableEntity row = (ReplicatedTableEntity)result.Result;
Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch");
Assert.AreNotEqual(null, result.Result, "result.Result = null");
Assert.AreEqual("1", result.Etag, "result.Etag mismatch");
Assert.AreEqual(false, row._rtable_RowLock, "row._rtable_RowLock mismatch");
Assert.AreEqual(1, row._rtable_Version, "row._rtable_Version mismatch");
Assert.AreEqual(false, row._rtable_Tombstone, "row._rtable_Tombstone mismatch");
Assert.AreEqual(configurationWrapper.GetWriteView().ViewId, row._rtable_ViewId, "row._rtable_ViewId mismatch");
ReadFromIndividualAccountsDirectly(newCustomer.PartitionKey, newCustomer.RowKey, true);
//Add replica at head
this.UpdateConfiguration(fullViewReplicas, 1);
// repair row on the new head
Console.WriteLine("Calling TableOperation.Replace(newCustomer)...");
result = repTable.RepairRow(row.PartitionKey, row.RowKey, null);
Assert.AreNotEqual(null, result, "result = null");
// Retrieve Entity
Console.WriteLine("Calling TableOperation.Retrieve<SampleRtableEntity>(firstName, lastName)...");
operation = TableOperation.Retrieve <SampleRTableEntity>("firstname1", "lastname1");
TableResult retrievedResult = repTable.Execute(operation);
Assert.AreNotEqual(null, retrievedResult, "retrievedResult = null");
Assert.AreEqual((int)HttpStatusCode.OK, retrievedResult.HttpStatusCode,
"retrievedResult.HttpStatusCode mismatch");
ReadFromIndividualAccountsDirectly(newCustomer.PartitionKey, newCustomer.RowKey, true);
}
/// <summary>
/// Helper function to retrieve the set of entities for the specified set of jobType and jobId.
/// Validate that entity.Message matches "entityMessage")
/// </summary>
/// <param name="jobType"></param>
/// <param name="jobId"></param>
/// <param name="entityMessage"></param>
/// <param name="checkViewId">True means validate _rtable_ViewId of the entity</param>
protected void PerformRetrieveOperationAndValidate(string jobType, string jobId, string entityMessage, bool checkViewId = true)
{
Console.WriteLine("\nValidating Retrieve operation...");
for (int i = 0; i < this.numberOfPartitions; i++)
{
List <string> rowKeys = new List <string>(); // list of rowKey for the given paritionKey
string partitionKey;
string rowKey;
SampleRTableEntity.GenerateKeys(this.GenerateJobType(jobType, i), "don't care", out partitionKey, out rowKey);
//
// GetAllRows()
//
Console.WriteLine("Calling GetAllRows() for partition {0}", i);
IEnumerable <SampleRTableEntity> allRows = this.rtableWrapper.GetAllRows(partitionKey);
int j = 0; // counting the number of rows per partition
foreach (var retrievedEntity in allRows)
{
this.ValidateRetrievedRTableEntity(retrievedEntity, jobType, jobId, entityMessage, i, j, checkViewId);
rowKeys.Add(retrievedEntity.RowKey);
j++;
}
Assert.AreEqual(this.numberOfRowsPerPartition, j, "Partition {0} only has {1} rows. Expected {2} rows", i, j, this.numberOfRowsPerPartition);
//
// FindRow()
//
Console.WriteLine("Calling FindRow() for partitionKey={0}", partitionKey);
for (j = 0; j < rowKeys.Count; j++)
{
SampleRTableEntity retrievedEntity = this.rtableWrapper.FindRow(partitionKey, rowKeys[j]);
this.ValidateRetrievedRTableEntity(retrievedEntity, jobType, jobId, entityMessage, i, j, checkViewId);
}
}
Console.WriteLine("Passed Retrieve validation.\n");
}
/// <summary>
/// Helper function to validate the specified entries do not exist.
/// </summary>
/// <param name="jobType"></param>
/// <param name="jobId"></param>
protected void ValidateRetrieveNotFound(string jobType, string jobId)
{
Console.WriteLine("\nValidating Retrieve NotFound operation jobType={0} jobId={1}...", jobType, jobId);
for (int i = 0; i < this.numberOfPartitions; i++)
{
List <string> rowKeys = new List <string>(); // list of rowKey for the given paritionKey
string partitionKey;
string rowKey;
SampleRTableEntity.GenerateKeys(this.GenerateJobType(jobType, i), "don't care", out partitionKey, out rowKey);
//
// GetAllRows()
//
Console.WriteLine("Calling GetAllRows() for partition {0}", i);
IEnumerable <SampleRTableEntity> allRows = this.rtableWrapper.GetAllRows(partitionKey);
int count = 0; // counting the number of rows per partition
List <SampleRTableEntity> allRetrievedEntities = new List <SampleRTableEntity>();
foreach (var retrievedEntity in allRows)
{
allRetrievedEntities.Add(retrievedEntity);
count++;
}
Assert.AreEqual(0, count, "GetAllRows() should return 0 entries.");
}
Console.WriteLine("Passed Retrieve validation.\n");
}
public void RandomTableOperationTest()
{
const int partitionIndex = 0;
// Insert some entries.
//Key = RowNumber, Value = RowKey
var currentEntities = new Dictionary <int, String>();
for (int i = 0; i < this.numberOfRowsPerPartition; i++)
{
currentEntities.Add(i, GenerateJobId(JobId, partitionIndex, i));
}
//Key = RowNumber, Value = RowKey
var deletedEntities = new Dictionary <int, String>();
var random = new Random();
for (int i = 0; i < NumberOfOperations; i++)
{
int operation = random.Next(0, Enum.GetNames(typeof(TableOperationType)).Count());
int randomRow = random.Next(0, NumberofEntities);
if ((TableOperationType)operation == TableOperationType.Insert)
{
if (deletedEntities.Count == 0)
{
//Ignore insertion if there is nothing to be inserted
continue;
}
randomRow = deletedEntities.First().Key;
deletedEntities.Remove(randomRow);
}
if (deletedEntities.ContainsKey(randomRow))
{
continue;
}
Console.WriteLine("Operation# {0}, {1} on row {2}", i, (TableOperationType)operation, randomRow);
PerformIndividualOperationAndValidate((TableOperationType)operation, JobType, JobId, partitionIndex, randomRow, OriginalMessage);
if ((TableOperationType)operation == TableOperationType.Delete)
{
deletedEntities.Add(randomRow, GenerateJobId(JobId, partitionIndex, randomRow));
}
}
string rowKey;
string partitionKey;
SampleRTableEntity.GenerateKeys(this.GenerateJobType(JobType, partitionIndex), "don't care", out partitionKey, out rowKey);
//Validations
Console.WriteLine("Performing replica validations");
PerformInvariantChecks(partitionKey, HeadReplicaAccountIndex, TailReplicaAccountIndex);
Console.WriteLine("DONE. Test passed.");
}
/// <summary>
/// Helper function to insert the specified set of jobType and jobId and validate the results.
/// Make sure there are no existing entries for the jobType and jobId before calling this function.
/// </summary>
/// <param name="jobType"></param>
/// <param name="jobId"></param>
/// <param name="entityMessage"></param>
protected void PerformInsertOperationAndValidate(string jobType, string jobId, string entityMessage)
{
Console.WriteLine("\nValidating Insert operation...");
for (int i = 0; i < this.numberOfPartitions; i++)
{
List <string> rowKeys = new List <string>(); // list of rowKey for the given paritionKey
string partitionKey;
string rowKey;
SampleRTableEntity.GenerateKeys(this.GenerateJobType(jobType, i), "don't care", out partitionKey, out rowKey);
//
// GetAllRows() to confirm nothing exists and then call InsertRow()
//
Console.WriteLine("Calling GetAllRows() for partition {0}, expecting 0 rows...", i);
IEnumerable <SampleRTableEntity> allRows = this.rtableWrapper.GetAllRows(partitionKey);
Assert.AreEqual(0, allRows.Count(), "Partition {0} should have 0 rows in order for InsertRow() to work.", i);
for (int j = 0; j < this.numberOfRowsPerPartition; j++)
{
SampleRTableEntity sampleRtableEntity = new SampleRTableEntity(
this.GenerateJobType(jobType, i),
this.GenerateJobId(jobId, i, j),
this.GenerateMessage(entityMessage, i, j));
int attempts = 1;
bool passed = true;
while (attempts < 3)
{
try
{
Console.WriteLine("attempts={0}. partitionKey={1} rowKey={2}. Calling InsertRow API...",
attempts, partitionKey, sampleRtableEntity.RowKey);
this.rtableWrapper.InsertRow(sampleRtableEntity);
passed = true;
break; // get out of while(attempts) if no RTableConflictException
}
catch (RTableConflictException)
{
passed = false;
Console.WriteLine("Got RTableConflictException. attempts={0}", attempts);
attempts++;
System.Threading.Thread.Sleep(1000);
}
}
Assert.IsTrue(passed, "Keep getting RTableConflictException when calling InsertRow API");
}
//
// FindRow()
//
Console.WriteLine("Calling FindRow() for partitionKey={0}", partitionKey);
for (int j = 0; j < rowKeys.Count; j++)
{
SampleRTableEntity retrievedEntity = this.rtableWrapper.FindRow(partitionKey, rowKeys[j]);
this.ValidateRetrievedRTableEntity(retrievedEntity, jobType, jobId, entityMessage, i, j);
}
}
Console.WriteLine("Passed Insert validation.\n");
}
/// <summary>
/// Check whether this instance matches the specified object or not
/// </summary>
/// <param name="obj"></param>
/// <returns></returns>
public override bool Equals(object obj)
{
SampleRTableEntity dst = obj as SampleRTableEntity;
if (dst == null)
{
return(false);
}
else
{
return((this.JobType == dst.JobType && this.JobId == dst.JobId && this.Message == dst.Message) && base.Equals(obj));
}
}
private bool HeadIsLocked(SampleRTableEntity entry)
{
CloudTable table = this.cloudTableClients[0].GetTableReference(this.repTable.TableName);
TableOperation retrieveOperation = TableOperation.Retrieve <SampleRTableEntity>(entry.PartitionKey, entry.RowKey);
TableResult retrieveResult = table.Execute(retrieveOperation);
if (retrieveResult == null ||
retrieveResult.HttpStatusCode != (int)HttpStatusCode.OK ||
retrieveResult.Result == null)
{
return(false);
}
SampleRTableEntity head = (SampleRTableEntity)retrieveResult.Result;
return(head._rtable_RowLock == true);
}
/// <summary>
/// Helper function to insert the specified set of jobType and jobId. It is expected that InsertRow() will fail.
/// </summary>
/// <param name="jobType"></param>
/// <param name="jobId"></param>
/// <param name="entityMessage"></param>
protected void PerformInsertOperationAndExpectToFail(string jobType, string jobId, string entityMessage)
{
Console.WriteLine("\nValidating Insert operation will fail. jobType={0} jobId={1}...", jobType, jobId);
for (int i = 0; i < this.numberOfPartitions; i++)
{
List <string> rowKeys = new List <string>(); // list of rowKey for the given paritionKey
string partitionKey;
string rowKey;
SampleRTableEntity.GenerateKeys(this.GenerateJobType(jobType, i), "don't care", out partitionKey, out rowKey);
for (int j = 0; j < this.numberOfRowsPerPartition; j++)
{
SampleRTableEntity sampleRtableEntity = new SampleRTableEntity(
this.GenerateJobType(jobType, i),
this.GenerateJobId(jobId, i, j),
this.GenerateMessage(entityMessage, i, j));
int attempts = 1;
bool failed = false;
while (attempts < 3)
{
try
{
Console.WriteLine("attempts={0}. partitionKey={1} rowKey={2}. Calling InsertRow API...",
attempts, partitionKey, sampleRtableEntity.RowKey);
this.rtableWrapper.InsertRow(sampleRtableEntity);
failed = false;
break; // get out of while(attempts) if no RTableConflictException
}
catch (RTableConflictException)
{
failed = true;
Console.WriteLine("Got RTableConflictException. attempts={0}", attempts);
attempts++;
System.Threading.Thread.Sleep(1000);
}
}
Assert.IsTrue(failed, "InsertRow() actually passed when we expect it to fail!");
}
}
Console.WriteLine("Passed 'Insert will fail' validation.\n");
}
/// <summary>
/// Read and print the entity of the specified jobType and jobId from individual storage accounts for debugging.
/// </summary>
/// <param name="jobType"></param>
/// <param name="jobId"></param>
/// <param name="checkIndividualAccounts">True means will check the entity in each storage account for consistency</param>
protected void ReadFromIndividualAccountsDirectly(string partitionKey, string rowKey, bool checkIndividualAccounts = false)
{
Console.WriteLine("\nBEGIN ReadFromIndividualAccountsDirectly()...");
partitionKey = partitionKey.ToLower().Replace(" ", "");
rowKey = rowKey.ToLower().Replace(" ", "");
string filter1 = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, partitionKey);
string filter2 = TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.Equal, rowKey);
TableQuery <SampleRTableEntity> query = new TableQuery <SampleRTableEntity>().
Where(TableQuery.CombineFilters(filter1, TableOperators.And, filter2));
SampleRTableEntity[] entities = new SampleRTableEntity[configurationWrapper.GetWriteView().Chain.Count];
for (int i = 0; i < configurationWrapper.GetWriteView().Chain.Count; i++)
{
Console.WriteLine("Executing query for CloudTable #{0}", i);
foreach (var item in ((CloudTableClient)configurationWrapper.GetWriteView()[i]).GetTableReference(repTable.TableName).ExecuteQuery(query))
{
Console.WriteLine("{0}", item.ToString());
entities[i] = item;
}
}
if (checkIndividualAccounts)
{
Console.WriteLine("Checking for consistency...");
for (int i = 1; i < configurationWrapper.GetWriteView().Chain.Count; i++)
{
Assert.IsTrue((entities[0] == null && entities[i] == null) || (entities[0] != null && entities[0].Equals(entities[i])),
"Entities in storage accounts: #0 and #{0} do NOT match", i);
}
Console.WriteLine("Entities in different accounts are consistent.");
}
Console.WriteLine("END ReadFromIndividualAccountsDirectly()\n");
}
public void BatchOperationUsingLargerViewId()
{
long currentViewId = 100;
long futureViewId = currentViewId + 1;
this.UpdateConfiguration(replicas, 0, false, currentViewId);
string jobType = "jobType-BatchOperationUsingLargerViewId";
string jobId = "jobId-BatchOperationUsingLargerViewId";
int count = 2; // number of operations in the batch _rtable_Operation
List <TableOperationType> opTypes = new List <TableOperationType>()
{
TableOperationType.Replace,
TableOperationType.Delete,
};
//
// Insert
//
string jobIdTemplate = jobId + "-{0}";
string messageTemplate = "message-{0}";
string updatedMessageTemplate = "updated-" + messageTemplate;
string partitionKey = string.Empty;
//
// Insert entities
//
for (int i = 0; i < count; i++)
{
SampleRTableEntity originalEntity = new SampleRTableEntity(
jobType,
string.Format(jobIdTemplate, i),
string.Format(messageTemplate, i));
this.repTable.Execute(TableOperation.Insert(originalEntity));
partitionKey = originalEntity.PartitionKey;
}
//
// Retrieve entities and use them to create batchOperation to Replace or Delete
//
IEnumerable <SampleRTableEntity> allEntities = this.rtableWrapper.GetAllRows(partitionKey);
TableBatchOperation batchOperation = new TableBatchOperation();
int m = 0;
foreach (SampleRTableEntity entity in allEntities)
{
Console.WriteLine("{0}", entity.ToString());
Console.WriteLine("---------------------------------------");
if (opTypes[m] == TableOperationType.Replace)
{
SampleRTableEntity replaceEntity = new SampleRTableEntity(
entity.JobType,
entity.JobId,
string.Format(updatedMessageTemplate, m))
{
ETag = entity._rtable_Version.ToString()
};
batchOperation.Replace(replaceEntity);
}
else if (opTypes[m] == TableOperationType.InsertOrReplace)
{
SampleRTableEntity replaceEntity = new SampleRTableEntity(
entity.JobType,
entity.JobId,
string.Format(updatedMessageTemplate, m))
{
ETag = entity._rtable_Version.ToString()
};
batchOperation.InsertOrReplace(replaceEntity);
}
else if (opTypes[m] == TableOperationType.Delete)
{
entity.ETag = entity._rtable_Version.ToString();
batchOperation.Delete(entity);
}
else
{
throw new ArgumentException(
string.Format("opType={0} is NOT supported", opTypes[m]),
"opType");
}
m++;
}
//
// Call ModifyConfigurationBlob to change the viewId of the wrapper to an older value
//
Console.WriteLine("Changing the viewId to larger viewId {0}", futureViewId);
this.UpdateConfiguration(replicas, 0, false, futureViewId);
Console.WriteLine("\nCalling BatchOperation with a larger viewId...");
this.repTable.ExecuteBatch(batchOperation);
this.ExecuteBatchOperationAndValidate(
count,
partitionKey,
jobType,
jobId,
opTypes);
}
/// <summary>
/// Helper function to set up an original entity and then run the specified HttpMangler Behavior.
/// It will return the original entity created.
/// Step (1): Make sure "entity1" with the specified partitionKey and rowKey exists. If not create it. Read "entity1".
/// Step (2): Read "entity1" from individual storage tables directly for debugging.
/// Step (3): Turn on HttpMangler and the specified behavior and call the specified RTable API to operate on "entity1".
/// This function accepts a targetApi with this form: Func<string, string, SampleRTableEntity> targetApi
/// </summary>
/// <param name="entityPartitionKey"></param>
/// <param name="entityRowKey"></param>
/// <param name="behaviors"></param>
/// <param name="targetApi"></param>
/// <param name="targetApiExpectedToFail"></param>
/// <returns></returns>
private SampleRTableEntity SetupAndRunHttpManglerBehaviorHelper(
string entityPartitionKey,
string entityRowKey,
ProxyBehavior[] behaviors,
Func<string, string, SampleRTableEntity> targetApi,
bool targetApiExpectedToFail)
{
DateTime httpManglerStartTime;
SampleRTableEntity originalEntity = new SampleRTableEntity();
// "entity1":
string jobType = entityPartitionKey;
string jobId = entityRowKey;
string referenceMessage = SampleRTableEntity.GenerateRandomMessage();
//
// Arrange
//
// Make sure an entity exists with the specified partitionKey and rowKey
Console.WriteLine("\nMaking sure entity1 exists...");
SampleRTableEntity entity1 = this.rtableWrapper.ReadEntity(jobType, jobId);
if (entity1 == null)
{
SampleRTableEntity createEntity1 = new SampleRTableEntity(jobType, jobId, referenceMessage);
this.rtableWrapper.InsertRow(createEntity1);
entity1 = this.rtableWrapper.ReadEntity(jobType, jobId);
}
else
{
// entity1 already exists. Save the value of its Message for later use
referenceMessage = entity1.Message;
}
Assert.IsNotNull(entity1, "entity1 is null UNEXPECTEDLY.");
Console.WriteLine("entity1:\n{0}", entity1.ToString());
Assert.IsTrue(entity1.JobType == jobType, "entity.JobType is incorrect.");
Assert.IsTrue(entity1.JobId == jobId, "entity.JobId is incorrect.");
originalEntity = entity1.Clone();
// Read from Storage Accounts directly for debugging. Check for consistency of different accounts.
this.ReadFromIndividualAccountsDirectly(jobType, jobId, true);
this.RunHttpManglerBehaviorHelper(
entity1,
behaviors,
targetApi,
targetApiExpectedToFail,
out httpManglerStartTime);
return originalEntity;
}
/// <summary>
/// Helper function to perform Delete, InsertOrReplace, Merge, or Replace operation for the specified set of jobType and jobId.
/// And validate the results.
/// </summary>
/// <param name="opType"></param>
/// <param name="jobType"></param>
/// <param name="jobId"></param>
/// <param name="partition"></param>
/// <param name="row"></param>
/// <param name="originalMessage"></param>
/// <param name="updatedMessage"></param>
protected void PerformIndividualOperationAndValidate(
TableOperationType opType,
string jobType,
string jobId,
int partition,
int row,
string originalMessage,
string updatedMessage = "")
{
Console.WriteLine("\nValidating {0} operation: updatedEntityMessage={1}...", opType, message);
string partitionKey;
string rowKey;
SampleRTableEntity.GenerateKeys(this.GenerateJobType(jobType, partition), this.GenerateJobId(jobId, partition, row), out partitionKey, out rowKey);
Console.WriteLine("PartitionKey = {0}, RowKey = {1}", partitionKey, rowKey);
int attempts = 1;
bool passed = true;
bool resetRow = false;
SampleRTableEntity retrievedEntity = null;
while (attempts < 3)
{
try
{
if (opType != TableOperationType.Insert)
{
retrievedEntity = this.rtableWrapper.FindRow(partitionKey, rowKey);
}
Console.WriteLine("attempts={0}. partitionKey={1} rowKey={2}. Calling {3} API...", attempts, partitionKey, rowKey, opType);
switch (opType)
{
case TableOperationType.Delete:
{
this.rtableWrapper.DeleteRow(retrievedEntity);
//Validate
try
{
retrievedEntity = this.rtableWrapper.FindRow(partitionKey, rowKey);
Assert.Fail("After DeleteRow() was called, FindRow() did not throw RTableResourceNotFoundException");
}
catch (RTableResourceNotFoundException)
{
}
}
break;
case TableOperationType.Insert:
{
SampleRTableEntity sampleRtableEntity = new SampleRTableEntity(
this.GenerateJobType(jobType, partition),
this.GenerateJobId(jobId, partition, row),
this.GenerateMessage(originalMessage, partition, row));
this.rtableWrapper.InsertRow(sampleRtableEntity);
//Validate
try
{
retrievedEntity = this.rtableWrapper.FindRow(partitionKey, rowKey);
}
catch (RTableResourceNotFoundException)
{
Assert.Fail("After InsertRow() was called, FindRow() threw RTableResourceNotFoundException");
}
}
break;
case TableOperationType.InsertOrMerge:
{
retrievedEntity.Message = this.GenerateMessage(updatedMessage, partition, row);
this.rtableWrapper.InsertOrMergeRow(retrievedEntity);
retrievedEntity = this.rtableWrapper.FindRow(partitionKey, rowKey);
this.ValidateRetrievedRTableEntity(retrievedEntity, jobType, jobId, updatedMessage, partition, row);
}
break;
case TableOperationType.InsertOrReplace:
{
retrievedEntity.Message = this.GenerateMessage(updatedMessage, partition, row);
this.rtableWrapper.InsertOrReplaceRow(retrievedEntity);
retrievedEntity = this.rtableWrapper.FindRow(partitionKey, rowKey);
this.ValidateRetrievedRTableEntity(retrievedEntity, jobType, jobId, updatedMessage, partition, row);
resetRow = true;
}
break;
case TableOperationType.Merge:
{
retrievedEntity.Message = this.GenerateMessage(updatedMessage, partition, row);
this.rtableWrapper.MergeRow(retrievedEntity);
retrievedEntity = this.rtableWrapper.FindRow(partitionKey, rowKey);
this.ValidateRetrievedRTableEntity(retrievedEntity, jobType, jobId, updatedMessage, partition, row);
}
break;
case TableOperationType.Replace:
{
retrievedEntity.Message = this.GenerateMessage(updatedMessage, partition, row);
this.rtableWrapper.ReplaceRow(retrievedEntity);
retrievedEntity = this.rtableWrapper.FindRow(partitionKey, rowKey);
this.ValidateRetrievedRTableEntity(retrievedEntity, jobType, jobId, updatedMessage, partition, row);
resetRow = true;
}
break;
case TableOperationType.Retrieve:
{
this.rtableWrapper.FindRow(partitionKey, rowKey);
}
break;
default:
{
throw new InvalidOperationException(string.Format("opType={0} is NOT supported", opType));
}
}
passed = true;
break; // get out of while(attempts) if no RTableConflictException
}
catch (RTableConflictException)
{
passed = false;
Console.WriteLine("Got RTableConflictException. attempts={0}", attempts);
attempts++;
System.Threading.Thread.Sleep(1000);
}
}
Assert.IsTrue(passed, "Keep getting RTableConflictException when calling {0} API", opType);
Console.WriteLine("Passed {0} validation.\n", opType);
//Reset row to original values so subsequent updates on the same row get validated correctly
if (resetRow == true)
{
Console.WriteLine("Resetting row to original values");
retrievedEntity.Message = this.GenerateMessage(originalMessage, partition, row);
this.rtableWrapper.ReplaceRow(retrievedEntity);
}
}
public void A00DelayTwoConflictingInsertOrReplaceCalls()
{
this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);
string entityPartitionKey = "jobType-DelayInsertOrReplaceRowHeadTest";
string entityRowKey = "jobId-DelayInsertOrReplaceRowHeadTest";
this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);
string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0];
Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper);
int delayInMs = 3000;
int insertRequestCount = 0;
int conflictResponseCount = 0;
bool secondUpsertConflicted = false;
int failedCallIndex = -1;
// Delay behavior
ProxyBehavior[] behaviors = new[]
{
// Delay Insert calls so they end up conflicting
TamperBehaviors.TamperAllRequestsIf(
(session =>
{
Interlocked.Increment(ref insertRequestCount);
while (insertRequestCount != 2)
{
Console.WriteLine("insertRequestCount={0}. Waiting on count to reach 2 ...", insertRequestCount);
Thread.Sleep(delayInMs);
}
}),
(session =>
{
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.HTTPMethodIs("POST") &&
session.GetRequestBodyAsString().Contains("\"_rtable_Operation\":\"Insert\""))
{
return(true);
}
return(false);
})),
// Delay conflict response
DelayBehaviors.DelayAllResponsesIf(
delayInMs,
(session =>
{
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.GetRequestBodyAsString().Contains("\"_rtable_Operation\":\"Insert\""))
{
if (session.responseCode == (int)HttpStatusCode.Conflict)
{
Interlocked.Increment(ref conflictResponseCount);
return(true);
}
}
return(false);
})),
};
using (new HttpMangler(false, behaviors))
{
Parallel.For(0, 2, (index) =>
{
var entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, string.Format("upsert message {0}", index));
try
{
this.rtableWrapper.InsertOrReplaceRow(entry);
}
catch (RTableConflictException)
{
if (secondUpsertConflicted)
{
// should never reach here
throw;
}
// That's possible, but that's the Replace step of upsert which conflicted with ongoing write
// can't do anything, client should retry on conflict
secondUpsertConflicted = true;
}
});
}
// got 2 inserts?
Assert.AreEqual(2, insertRequestCount, "Two insert calls expected!");
// got one conflict?
Assert.AreEqual(1, conflictResponseCount, "One conflict response expected!");
// at least one upsert would have succeeded
SampleRTableEntity upsertedEntity = this.rtableWrapper.ReadEntity(entityPartitionKey, entityRowKey);
Assert.NotNull(upsertedEntity, "at least one upsert should have succeeded");
// second upsert failed?
if (secondUpsertConflicted)
{
Assert.AreEqual(1, upsertedEntity._rtable_Version, "one upsert succeeded so version should be = 1");
Assert.AreEqual(upsertedEntity.Message, string.Format("upsert message {0}", (1 - failedCallIndex)));
}
else
{
Assert.AreEqual(2, upsertedEntity._rtable_Version, "both upserts succeeded so version should be = 2");
}
// After recovery from delay, confirm that we can update the row.
//this.ExecuteReplaceRowAndValidate(entityPartitionKey, entityRowKey);
}
/// <summary>
/// Copy the values of JobType, JobId, Message to the specified object
/// </summary>
/// <param name="dst"></param>
public void CopyTo(SampleRTableEntity dst)
{
dst.JobType = this.JobType;
dst.JobId = this.JobId;
dst.Message = this.Message;
}
public void A00InsertOrReplaceCallConflictingWithDeleteOnTheHead()
{
this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);
string entityPartitionKey = "jobType-DelayInsertOrReplaceWhileDelete";
string entityRowKey = "jobType-DelayInsertOrReplaceWhileDelete";
this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);
// Insert one entry
Console.WriteLine("Inserting entry ...");
var entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, "insert message");
this.rtableWrapper.InsertRow(entry);
// 1 - Launch an Upsert task in wait mode ...
bool deleteLockedHead = false;
TableResult upsertResult = null;
var upsertTask = Task.Run(() =>
{
while (!deleteLockedHead)
{
Thread.Sleep(5);
}
try
{
entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, "upsert message");
var table = new ReplicatedTable(this.repTable.TableName, this.configurationService);
TableOperation upserOperation = TableOperation.InsertOrReplace(entry);
Console.WriteLine("Upsert started ...");
upsertResult = table.Execute(upserOperation);
Console.WriteLine("Upsert completed with HttpStatus={0}", upsertResult == null ? "NULL" : upsertResult.HttpStatusCode.ToString());
}
catch (AggregateException ex)
{
Console.WriteLine(ex);
}
});
string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0];
Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper);
int delayInMs = 5;
// Delay behavior
ProxyBehavior[] behaviors = new[]
{
DelayBehaviors.DelayAllResponsesIf(
delayInMs,
(session =>
{
var body = session.GetRequestBodyAsString();
Console.WriteLine(body);
// Delete locking the head response
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.HTTPMethodIs("PUT") &&
body.Contains("\"_rtable_Operation\":\"Replace\"") &&
body.Contains("\"_rtable_RowLock\":true") &&
body.Contains("\"_rtable_Tombstone\":true"))
{
// Signal upsert we locked the head, so it can continue ...
deleteLockedHead = true;
return(true);
}
return(false);
})),
};
// Launch a delete
using (new HttpMangler(false, behaviors))
{
try
{
var table = new ReplicatedTable(this.repTable.TableName, this.configurationService);
// Retrieve entity
TableOperation retrieveOperation = TableOperation.Retrieve <SampleRTableEntity>(entry.PartitionKey, entry.RowKey);
TableResult retrieveResult = table.Execute(retrieveOperation);
Assert.IsNotNull(retrieveResult, "retrieveResult = null");
Assert.AreEqual((int)HttpStatusCode.OK, retrieveResult.HttpStatusCode, "retrieveResult.HttpStatusCode mismatch");
Assert.IsNotNull((SampleRTableEntity)retrieveResult.Result, "Retrieve: customer = null");
// Delete entity
TableOperation deleteOperation = TableOperation.Delete((SampleRTableEntity)retrieveResult.Result);
TableResult deleteResult = table.Execute(deleteOperation);
Assert.IsNotNull(deleteResult, "deleteResult = null");
Assert.AreEqual((int)HttpStatusCode.NoContent, deleteResult.HttpStatusCode, "deleteResult.HttpStatusCode mismatch");
}
catch (AggregateException ex)
{
Console.WriteLine(ex);
}
}
// wait on upsert to finish ...
upsertTask.Wait();
// Upsert suceeded?
Assert.AreEqual((int)HttpStatusCode.NoContent, upsertResult.HttpStatusCode, "Upsert expected to suceed!");
SampleRTableEntity upsertedEntity = this.rtableWrapper.ReadEntity(entityPartitionKey, entityRowKey);
Assert.NotNull(upsertedEntity, "upsert should succeed");
Assert.AreEqual(upsertedEntity.Message, "upsert message", "upsert should succeeded");
}
/// <summary>
/// Helper function to perform Delete, InsertOrReplace, Merge, or Replace operation for the specified set of jobType and jobId.
/// And validate the results.
/// </summary>
/// <param name="opType"></param>
/// <param name="jobType"></param>
/// <param name="jobId"></param>
/// <param name="updatedEntityMessage"></param>
protected void PerformOperationAndValidate(
TableOperationType opType,
string jobType,
string jobId,
string updatedEntityMessage = "")
{
Console.WriteLine("\nValidating {0} operation: updatedEntityMessage={1}...", opType, updatedEntityMessage);
for (int i = 0; i < this.numberOfPartitions; i++)
{
List <string> rowKeys = new List <string>(); // list of rowKey for the given paritionKey
string partitionKey;
string rowKey;
SampleRTableEntity.GenerateKeys(this.GenerateJobType(jobType, i), "don't care", out partitionKey, out rowKey);
//
// GetAllRows()
//
Console.WriteLine("Calling GetAllRows() and then ReplaceRow() for partition {0}", i);
IEnumerable <SampleRTableEntity> allRows = this.rtableWrapper.GetAllRows(partitionKey);
int j = 0; // counting the number of rows per partition
List <SampleRTableEntity> allRetrievedEntities = new List <SampleRTableEntity>();
foreach (var retrievedEntity in allRows)
{
allRetrievedEntities.Add(retrievedEntity);
j++;
}
Assert.AreEqual(this.numberOfRowsPerPartition, j, "Partition {0} only has {1} rows. Expected {2} rows", i, j, this.numberOfRowsPerPartition);
j = 0;
foreach (var oneEntry in allRetrievedEntities)
{
int attempts = 1;
bool passed = true;
while (attempts < 3)
{
try
{
SampleRTableEntity retrievedEntity = oneEntry;
Console.WriteLine("attempts={0}. partitionKey={1} rowKey={2}. Calling {3} API...",
attempts, partitionKey, retrievedEntity.RowKey, opType);
switch (opType)
{
case TableOperationType.Delete:
{
this.rtableWrapper.DeleteRow(retrievedEntity);
}
break;
case TableOperationType.InsertOrReplace:
{
retrievedEntity.Message = this.GenerateMessage(updatedEntityMessage, i, j);
this.rtableWrapper.InsertOrReplaceRow(retrievedEntity);
rowKeys.Add(retrievedEntity.RowKey);
}
break;
case TableOperationType.Merge:
{
retrievedEntity.Message = this.GenerateMessage(updatedEntityMessage, i, j);
this.rtableWrapper.MergeRow(retrievedEntity);
rowKeys.Add(retrievedEntity.RowKey);
}
break;
case TableOperationType.Replace:
{
retrievedEntity.Message = this.GenerateMessage(updatedEntityMessage, i, j);
this.rtableWrapper.ReplaceRow(retrievedEntity);
rowKeys.Add(retrievedEntity.RowKey);
}
break;
default:
{
throw new InvalidOperationException(string.Format("opType={0} is NOT supported", opType));
}
}
passed = true;
break; // get out of while(attempts) if no RTableConflictException
}
catch (RTableConflictException)
{
passed = false;
Console.WriteLine("Got RTableConflictException. attempts={0}", attempts);
attempts++;
System.Threading.Thread.Sleep(1000);
}
}
Assert.IsTrue(passed, "Keep getting RTableConflictException when calling {0} API", opType);
j++;
}
Assert.AreEqual(this.numberOfRowsPerPartition, j, "Partition {0} only has {1} rows. Expected {2} rows", i, j, this.numberOfRowsPerPartition);
//
// FindRow()
//
Console.WriteLine("Calling FindRow() for partitionKey={0}", partitionKey);
for (j = 0; j < rowKeys.Count; j++)
{
if (opType == TableOperationType.Delete)
{
try
{
SampleRTableEntity retrievedEntity = this.rtableWrapper.FindRow(partitionKey, rowKeys[j]);
Assert.Fail("After DeleteRow() was called, FindRow() did not throw RTableResourceNotFoundException");
}
catch (RTableResourceNotFoundException)
{
}
}
else
{
SampleRTableEntity retrievedEntity = this.rtableWrapper.FindRow(partitionKey, rowKeys[j]);
this.ValidateRetrievedRTableEntity(retrievedEntity, jobType, jobId, updatedEntityMessage, i, j);
}
}
}
Console.WriteLine("Passed {0} validation.\n", opType);
}
public void RTableRepairRowDelete()
{
// Insert entity
Assert.IsTrue(this.repTable.Exists(), "RTable does not exist");
View fullView = configurationWrapper.GetWriteView();
List<ReplicaInfo> fullViewReplicas = new List<ReplicaInfo>();
for (int i = 0; i <= fullView.TailIndex; i++)
{
fullViewReplicas.Add(fullView.GetReplicaInfo(i));
}
List<ReplicaInfo> newReplicas = new List<ReplicaInfo>();
for (int i = 1; i <= fullView.TailIndex; i++)
{
newReplicas.Add(fullView.GetReplicaInfo(i));
}
SampleRTableEntity newCustomer = new SampleRTableEntity("firstName1", "lastName1", "*****@*****.**");
TableOperation operation = TableOperation.Insert(newCustomer);
TableResult result = repTable.Execute(operation);
Assert.AreNotEqual(null, result, "result = null");
ReplicatedTableEntity row = (ReplicatedTableEntity)result.Result;
Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch");
Assert.AreNotEqual(null, result.Result, "result.Result = null");
Assert.AreEqual("1", result.Etag, "result.Etag mismatch");
Assert.AreEqual(false, row._rtable_RowLock, "row._rtable_RowLock mismatch");
Assert.AreEqual(1, row._rtable_Version, "row._rtable_Version mismatch");
Assert.AreEqual(false, row._rtable_Tombstone, "row._rtable_Tombstone mismatch");
Assert.AreEqual(configurationWrapper.GetWriteView().ViewId, row._rtable_ViewId, "row._rtable_ViewId mismatch");
ReadFromIndividualAccountsDirectly(newCustomer.PartitionKey, newCustomer.RowKey, true);
// remove replica from the head
this.UpdateConfiguration(newReplicas, 0);
Assert.IsTrue(configurationWrapper.IsViewStable());
// delete row
TableOperation deleteOperation = TableOperation.Delete(newCustomer);
result = repTable.Execute(deleteOperation);
Assert.AreNotEqual(null, result, "result = null");
Assert.AreEqual((int) HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch");
//Add replica at head
this.UpdateConfiguration(fullViewReplicas, 1);
// repair row on the new head
Console.WriteLine("Calling repair row");
result = repTable.RepairRow(row.PartitionKey, row.RowKey, null);
Assert.AreNotEqual(null, result, "result = null");
// Retrieve Entity
Console.WriteLine("Calling TableOperation.Retrieve<SampleRtableEntity>(firstName, lastName)...");
operation = TableOperation.Retrieve<SampleRTableEntity>("firstname1", "lastName1");
TableResult retrievedResult = repTable.Execute(operation);
Assert.AreNotEqual(null, retrievedResult, "retrievedResult = null");
Assert.AreEqual((int)HttpStatusCode.NotFound, retrievedResult.HttpStatusCode,
"retrievedResult.HttpStatusCode mismatch");
ReadFromIndividualAccountsDirectly(newCustomer.PartitionKey, newCustomer.RowKey, true);
}
public void BatchOperationExceptionWhenUsingSmallerViewId()
{
long currentViewId = 100;
long badViewId = currentViewId - 1;
this.UpdateConfiguration(replicas, 0, false, currentViewId);
string jobType = "jobType-BatchOperationExceptionWhenUsingSmallerViewId";
string jobId = "jobId-BatchOperationExceptionWhenUsingSmallerViewId";
int count = 3; // number of operations in the batch _rtable_Operation
List <TableOperationType> opTypes = new List <TableOperationType>()
{
TableOperationType.Replace,
TableOperationType.InsertOrReplace,
TableOperationType.Delete,
};
//
// Insert
//
string jobIdTemplate = jobId + "-{0}";
string messageTemplate = "message-{0}";
string updatedMessageTemplate = "updated-" + messageTemplate;
string partitionKey = string.Empty;
//
// Insert entities
//
for (int i = 0; i < count; i++)
{
SampleRTableEntity originalEntity = new SampleRTableEntity(
jobType,
string.Format(jobIdTemplate, i),
string.Format(messageTemplate, i));
this.repTable.Execute(TableOperation.Insert(originalEntity));
partitionKey = originalEntity.PartitionKey;
}
//
// Retrieve entities and use them to create batchOperation to Replace or Delete
//
IEnumerable <SampleRTableEntity> allEntities = this.rtableWrapper.GetAllRows(partitionKey);
TableBatchOperation batchOperation = new TableBatchOperation();
int m = 0;
foreach (SampleRTableEntity entity in allEntities)
{
Console.WriteLine("{0}", entity.ToString());
Console.WriteLine("---------------------------------------");
if (opTypes[m] == TableOperationType.Replace)
{
SampleRTableEntity replaceEntity = new SampleRTableEntity(
entity.JobType,
entity.JobId,
string.Format(updatedMessageTemplate, m))
{
ETag = entity._rtable_Version.ToString()
};
batchOperation.Replace(replaceEntity);
}
else if (opTypes[m] == TableOperationType.InsertOrReplace)
{
SampleRTableEntity replaceEntity = new SampleRTableEntity(
entity.JobType,
entity.JobId,
string.Format(updatedMessageTemplate, m))
{
ETag = entity._rtable_Version.ToString()
};
batchOperation.InsertOrReplace(replaceEntity);
}
else if (opTypes[m] == TableOperationType.Delete)
{
entity.ETag = entity._rtable_Version.ToString();
batchOperation.Delete(entity);
}
else
{
throw new ArgumentException(
string.Format("opType={0} is NOT supported", opTypes[m]),
"opType");
}
m++;
}
//
// Call ModifyConfigurationBlob to change the viewId of the wrapper to an older value
//
Console.WriteLine("Changing the viewId to badViewId {0}", badViewId);
this.UpdateConfiguration(replicas, 0, false, badViewId);
//
// Execute Batch _rtable_Operation with bad viewId
//
Console.WriteLine("\nCalling BatchOperation with badViewId...");
try
{
this.repTable.ExecuteBatch(batchOperation);
}
catch (ReplicatedTableStaleViewException ex)
{
Console.WriteLine("Get this RTableStaleViewException: {0}", ex.Message);
Assert.IsTrue(ex.ErrorCode == ReplicatedTableViewErrorCodes.ViewIdSmallerThanEntryViewId);
Assert.IsTrue(ex.Message.Contains(string.Format("current _rtable_ViewId {0} is smaller than", badViewId)), "Got unexpected exception message");
}
}
/// <summary>
/// Helper function to create a new entity and validate the creation was successful.
/// </summary>
/// <param name="entityPartitionKey"></param>
/// <param name="entityRowKey"></param>
protected void ExecuteCreateRowAndValidate(
string entityPartitionKey,
string entityRowKey)
{
Console.WriteLine("\nExecuteCreateRowAndValidate(): Create entity and validate...");
string jobType = entityPartitionKey;
string jobId = entityRowKey;
string referenceMessage = SampleRTableEntity.GenerateRandomMessage();
SampleRTableEntity createEntity = new SampleRTableEntity(jobType, jobId, referenceMessage);
bool gotExceptionInLastAttempt = true;
int retries = 0;
while (retries < MaxRetries)
{
try
{
Console.WriteLine("\nretries={0}. Calling InsertRow(). referenceMessage={1}", retries, referenceMessage);
this.rtableWrapper.InsertRow(createEntity);
gotExceptionInLastAttempt = false;
break;
}
catch (RTableConflictException ex)
{
Console.WriteLine("retries={0}. InsertRow() got an RTableConflictException: {1}",
retries, ex.ToString());
retries++;
gotExceptionInLastAttempt = true;
Thread.Sleep(this.configurationWrapper.GetLockTimeout());
// For debug purposes: read from the Head and Tail accounts:
this.ReadFromIndividualAccountsDirectly(entityPartitionKey, entityRowKey);
}
catch (RTableRetriableException ex)
{
Console.WriteLine("retries={0}. InsertRow() got an RTableRetriableException: {1}",
retries, ex.ToString());
retries++;
gotExceptionInLastAttempt = true;
Thread.Sleep(ConflictExceptionSleepTimeInMsec);
// For debug purposes: read from the Head and Tail accounts:
this.ReadFromIndividualAccountsDirectly(entityPartitionKey, entityRowKey);
}
}
Console.WriteLine("gotExceptionInLastAttempt = {0}", gotExceptionInLastAttempt);
Console.WriteLine("\nRead entity back and validate...");
SampleRTableEntity retrievedEntity = this.rtableWrapper.ReadEntity(jobType, jobId);
Assert.IsNotNull(retrievedEntity, "retrievedEntity is null UNEXPECTEDLY.");
Console.WriteLine("retrievedEntity:\n{0}", retrievedEntity.ToString());
Assert.IsTrue(retrievedEntity.JobType == jobType, "retrievedEntity.JobType is incorrect.");
Assert.IsTrue(retrievedEntity.JobId == jobId, "retrievedEntity.JobId is incorrect.");
Assert.IsTrue(retrievedEntity.Message == referenceMessage, "retrievedEntity.Message is incorrect.");
Console.WriteLine("Passed validation");
}
/// <summary>
/// This help function is for getting the reference behavior of the Storage DLL when HttpMangler is enabled.
/// Edit HttpMangler.cs to use HandleFiddlerEvent_DEBUG().
/// </summary>
/// <param name="entityPartitionKey"></param>
/// <param name="entityRowKey"></param>
/// <param name="targetStorageAccount"></param>
/// <param name="targetApiExpectedToFail"></param>
/// <param name="behaviors"></param>
protected void SetupAndRunXStoreHttpManglerTest(
string entityPartitionKey,
string entityRowKey,
int targetStorageAccount,
bool targetApiExpectedToFail,
ProxyBehavior[] behaviors)
{
Assert.IsTrue(0 <= targetStorageAccount && targetStorageAccount < this.actualStorageAccountsUsed.Count,
"targetStorageAccount={0} is out-of-range", targetStorageAccount);
int index = this.actualStorageAccountsUsed[targetStorageAccount];
string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[index];
Console.WriteLine("accountNameToTamper={0}", accountNameToTamper);
CloudTableClient tableClient = this.cloudTableClients[targetStorageAccount];
CloudTable table = tableClient.GetTableReference(this.repTable.TableName);
//
// Insert
//
string jobType = entityPartitionKey;
string jobId = entityRowKey;
string referenceMessage = SampleRTableEntity.GenerateRandomMessage();
SampleRTableEntity originalEntity = new SampleRTableEntity(jobType, jobId, referenceMessage);
Console.WriteLine("\nCalling XStore Insert...");
TableOperation insertOperation = TableOperation.Insert(originalEntity);
TableResult insertResult = table.Execute(insertOperation);
Assert.IsNotNull(insertResult, "insertResult = null");
Console.WriteLine("insertResult.HttpStatusCode = {0}", insertResult.HttpStatusCode);
Console.WriteLine("insertResult.ETag = {0}", insertResult.Etag);
Assert.AreEqual((int)HttpStatusCode.NoContent, insertResult.HttpStatusCode, "insertResult.HttpStatusCode mismatch");
Assert.IsFalse(string.IsNullOrEmpty(insertResult.Etag), "insertResult.ETag = null or empty");
ITableEntity row = (ITableEntity)insertResult.Result;
//
// Retrieve
//
Console.WriteLine("Calling XStore Retrieve...");
TableOperation retrieveOperation = TableOperation.Retrieve<SampleRTableEntity>(row.PartitionKey, row.RowKey);
TableResult retrieveResult = table.Execute(retrieveOperation);
Assert.IsNotNull(retrieveResult, "retrieveResult = null");
Console.WriteLine("retrieveResult.HttpStatusCode = {0}", retrieveResult.HttpStatusCode);
Assert.AreEqual((int)HttpStatusCode.OK, retrieveResult.HttpStatusCode, "retrieveResult.HttpStatusCode mismatch");
SampleRTableEntity retrievedEntity = (SampleRTableEntity)retrieveResult.Result;
Console.WriteLine("retrieveEntity:\n{0}", retrievedEntity);
Assert.IsTrue(originalEntity.Equals(retrievedEntity), "originalEntity != retrievedEntity");
//
// Replace with HttpMangler enabled
//
Console.WriteLine("Calling XStore TableOperation.Replace with HttpMangler enabled...");
referenceMessage = SampleRTableEntity.GenerateRandomMessage();
retrievedEntity.Message = referenceMessage;
TableOperation updateOperation = TableOperation.Replace(retrievedEntity);
bool abortTest = false;
try
{
using (HttpMangler proxy = new HttpMangler(false, behaviors))
{
Console.WriteLine("Calling table.Execute(updateOperation)");
TableResult updateResult = table.Execute(updateOperation);
if (targetApiExpectedToFail)
{
// if targetApi is expected to fail, and we are here, that means something is wrong.
abortTest = true;
throw new Exception("SetupAndRunXStoreHttpManglerTest(): Should not reach here. HttpMangler allowed an targetApi() to go through UNEXPECTEDLY.");
}
}
}
catch (Exception ex)
{
if (abortTest)
{
throw;
}
else
{
Console.WriteLine("\nException is Expected. targetApi(entity1) threw an exception: {0}\n", ex.ToString());
}
}
//
// Retrieve again
//
Console.WriteLine("After HttpMangler is disabled, calling XStore Retrieve again...");
retrieveOperation = TableOperation.Retrieve<SampleRTableEntity>(row.PartitionKey, row.RowKey);
retrieveResult = table.Execute(retrieveOperation);
Assert.IsNotNull(retrieveResult, "retrieveResult = null");
Console.WriteLine("retrieveResult.HttpStatusCode = {0}", retrieveResult.HttpStatusCode);
Assert.AreEqual((int)HttpStatusCode.OK, retrieveResult.HttpStatusCode, "retrieveResult.HttpStatusCode mismatch");
SampleRTableEntity retrievedEntity2 = (SampleRTableEntity)retrieveResult.Result;
Console.WriteLine("retrieveEntity2:\n{0}", retrievedEntity2);
Assert.IsTrue(originalEntity.Equals(retrievedEntity2), "originalEntity != retrievedEntity2");
}
/// <summary>
/// Helper function to make ReadEntity() API call and validate the retrieve contents match the specified originalEntity.
/// Call this helper after you have finished tampered some operation on the same entity.
/// </summary>
/// <param name="entityPartitionKey"></param>
/// <param name="entityRowKey"></param>
/// <param name="originalEntity">Set it to null to by-pass validation against originalEntity.</param>
/// <param name="checkIndividualAccounts">True means will check the entity in each storage account for consistency</param>
protected void ExecuteReadRowAndValidate(
string entityPartitionKey,
string entityRowKey,
SampleRTableEntity originalEntity,
bool checkIndividualAccounts = false)
{
Console.WriteLine("\nExecuteReadRowAndValidate(): Read entity back and validate...");
string jobType = entityPartitionKey;
string jobId = entityRowKey;
SampleRTableEntity retrievedEntity = this.rtableWrapper.ReadEntity(jobType, jobId);
if (retrievedEntity == null)
{
Console.WriteLine("ERROR: retrievedEntity = null");
}
else
{
Console.WriteLine("retrievedEntity = \n{0}", retrievedEntity.ToString());
}
if (originalEntity != null)
{
Console.WriteLine("originalEntity = \n{0}", originalEntity.ToString());
}
// For debug purposes: read from the Head and Tail accounts:
this.ReadFromIndividualAccountsDirectly(jobType, jobId, checkIndividualAccounts);
Assert.IsNotNull(retrievedEntity, "retrievedEntity is null UNEXPECTEDLY");
if (originalEntity != null)
{
Assert.IsTrue(retrievedEntity.Equals(originalEntity), "retrievedEntity NOT equal to originalEntity");
Console.WriteLine("Passed validation");
}
}
/// <summary>
/// Helper function to create some initial entities and then call the specified batchOperation with HttpMangler enabled.
/// </summary>
/// <param name="count"></param>
/// <param name="jobType"></param>
/// <param name="jobId"></param>
/// <param name="targetStorageAccount"></param>
/// <param name="opTypes"></param>
/// <param name="targetApiExpectedToFail"></param>
/// <param name="checkOriginalEntityUnchanged"></param>
/// <param name="checkStorageAccountsConsistent"></param>
/// <param name="httpManglerStartTime"></param>
/// <param name="skipInitialSessions"></param>
/// <returns>ParitionKey of the initial entities created</returns>
protected string SetupAndRunTemperBatchOperation(
int count,
string jobType,
string jobId,
int targetStorageAccount,
List<TableOperationType> opTypes,
bool targetApiExpectedToFail,
bool checkOriginalEntityUnchanged,
bool checkStorageAccountsConsistent,
out DateTime httpManglerStartTime,
int skipInitialSessions = 0)
{
Assert.IsTrue(0 <= targetStorageAccount && targetStorageAccount < this.actualStorageAccountsUsed.Count,
"SetupAndRunTemperBatchOperation() is called with out-of-range targetStorageAccount={0}", targetStorageAccount);
int index = this.actualStorageAccountsUsed[targetStorageAccount];
string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[index];
Console.WriteLine("SetupAndRunTemperBatchOperation(): accountNameToTamper={0} skipInitialSessions={1}",
accountNameToTamper, skipInitialSessions);
Assert.AreEqual(count, opTypes.Count, "count and opTypes.Count should be the same");
//
// Tamper behavior
//
ProxyBehavior[] behaviors = new[]
{
TamperBehaviors.TamperAllRequestsIf(
(session => { session.Abort();}),
skipInitialSessions,
AzureStorageSelectors.TableTraffic().IfHostNameContains(accountNameToTamper + "."))
};
string jobIdTemplate = jobId + "-{0}";
string messageTemplate = "message-{0}";
string updatedMessageTemplate = "updated-" + messageTemplate;
string partitionKey = string.Empty;
//
// Insert entities
//
for (int i = 0; i < count; i++)
{
SampleRTableEntity originalEntity = new SampleRTableEntity(
jobType,
string.Format(jobIdTemplate, i),
string.Format(messageTemplate, i));
this.repTable.Execute(TableOperation.Insert(originalEntity));
partitionKey = originalEntity.PartitionKey;
}
//
// Retrieve entities and use them to create batchOperation to Replace or Delete
//
IEnumerable<SampleRTableEntity> allEntities = this.rtableWrapper.GetAllRows(partitionKey);
TableBatchOperation batchOperation = new TableBatchOperation();
int m = 0;
foreach (SampleRTableEntity entity in allEntities)
{
Console.WriteLine("{0}", entity.ToString());
Console.WriteLine("---------------------------------------");
if (opTypes[m] == TableOperationType.Replace)
{
SampleRTableEntity replaceEntity = new SampleRTableEntity(
entity.JobType,
entity.JobId,
string.Format(updatedMessageTemplate, m))
{
ETag = entity._rtable_Version.ToString()
};
batchOperation.Replace(replaceEntity);
}
else if (opTypes[m] == TableOperationType.Delete)
{
entity.ETag = entity._rtable_Version.ToString();
batchOperation.Delete(entity);
}
else
{
throw new ArgumentException(
string.Format("opType={0} is NOT supported", opTypes[m]),
"opType");
}
m++;
}
//
// Enable HttpMangler
// Call this.repTable.ExecuteBatch(batchOperation)
//
this.RunHttpManglerBehaviorHelper(
batchOperation,
behaviors,
targetApiExpectedToFail,
out httpManglerStartTime);
if (checkOriginalEntityUnchanged)
{
Console.WriteLine("Validate originalEntity remain unchanged.");
allEntities = this.rtableWrapper.GetAllRows(partitionKey);
batchOperation = new TableBatchOperation();
m = 0;
foreach (SampleRTableEntity entity in allEntities)
{
Console.WriteLine("{0}", entity.ToString());
Console.WriteLine("---------------------------------------");
Assert.AreEqual(string.Format(jobType, m), entity.JobType, "JobType does not match");
Assert.AreEqual(string.Format(jobIdTemplate, m), entity.JobId, "JobId does not match");
Assert.AreEqual(string.Format(messageTemplate, m), entity.Message, "Message does not match");
m++;
}
Console.WriteLine("Passed validation");
}
//
// After httpMangler is turned off, read from individual accounts...
//
Console.WriteLine("\nAfter httpMangler is turned off, read from individual accounts...");
for (int i = 0; i < count; i++)
{
this.ReadFromIndividualAccountsDirectly(
jobType,
string.Format(jobIdTemplate, i),
checkStorageAccountsConsistent);
}
return partitionKey;
}
/// <summary>
/// Run the specified HttpMangler proxy behaviors for the specified Api, which takes this form:
/// Func<string, string, SampleRTableEntity> targetApi
/// </summary>
/// <param name="originalEntity"></param>
/// <param name="behaviors"></param>
/// <param name="targetApi"></param>
/// <param name="targetApiExpectedToFail"></param>
/// <param name="httpManglerStartTime"></param>
protected void RunHttpManglerBehaviorHelper(
SampleRTableEntity originalEntity,
ProxyBehavior[] behaviors,
Func<string, string, SampleRTableEntity> targetApi,
bool targetApiExpectedToFail,
out DateTime httpManglerStartTime)
{
//
// Act
//
// Call targetApi(entity1) and tamper the request
httpManglerStartTime = DateTime.UtcNow;
Console.WriteLine("\nRunHttpManglerBehaviorHelper(): Call targetApi(entity1) with HttpMangler enabled...");
originalEntity.Message = SampleRTableEntity.GenerateRandomMessage();
bool abortTest = false;
try
{
using (HttpMangler proxy = new HttpMangler(false, behaviors))
{
Console.WriteLine("Calling targetApi(entity1)");
targetApi(originalEntity.PartitionKey, originalEntity.RowKey);
if (targetApiExpectedToFail)
{
// if targetApi is expected to fail, and we are here, that means something is wrong.
abortTest = true;
throw new Exception("RunHttpManglerBehaviorHelper(): Should not reach here. HttpMangler allowed an targetApi() to go through UNEXPECTEDLY.");
}
}
}
catch (Exception ex)
{
if (targetApiExpectedToFail == false)
{
// if targetApi is NOT expected to fail, and we are here, that means something is wrong.
throw new Exception(
string.Format("RunHttpManglerBehaviorHelper(): targetApi() is NOT unexpected to throw an exception. Got this exception: {0}",
ex.ToString()));
}
if (abortTest)
{
throw;
}
else
{
Console.WriteLine("\nException is Expected. targetApi(entity1) threw an exception: {0}\n", ex.ToString());
}
}
}
public void RepairMidAnUpdate()
{
this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);
// Not-Stable chain
// Reconfigure RTable so Head is WriteOnly.
View view = this.configurationService.GetTableView(this.repTable.TableName);
ReplicatedTableConfiguration config;
ReplicatedTableQuorumReadResult readStatus = this.configurationService.RetrieveConfiguration(out config);
Assert.IsTrue(readStatus.Code == ReplicatedTableQuorumReadCode.Success);
// Set Head as WriteOnly mode
ReplicatedTableConfigurationStore viewConfg = config.GetView(view.Name);
viewConfg.ReplicaChain[0].Status = ReplicaStatus.WriteOnly;
config.SetView(view.Name, viewConfg);
// Upload RTable config back
this.configurationService.UpdateConfiguration(config);
// Sanity: Replicated mode and chain Not-Stable
view = this.configurationService.GetTableView(this.repTable.TableName);
Assert.IsTrue(view != null && view.Chain.Count > 1, "Two replicas should be used.");
Assert.IsFalse(view.IsStable);
// Insert one entry
Console.WriteLine("Inserting entry ...");
string entityPartitionKey = "jobType-RepairMidAnUpdate-Replace";
string entityRowKey = "jobId-RepairMidAnUpdate-Replace";
var entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, "message");
this.rtableWrapper.InsertRow(entry);
// 1 - Launch a RepairRow task in wait mode ...
bool triggerRepair = false;
bool repaireDone = false;
bool headAfterRepairWasLocked = false;
TableResult repairResult = null;
Task.Run(() =>
{
ReplicatedTable repairTable = new ReplicatedTable(this.repTable.TableName, this.configurationService);
while (!triggerRepair)
{
Thread.Sleep(5);
}
Console.WriteLine("RepairRow started ...");
repairResult = repairTable.RepairRow(entry.PartitionKey, entry.RowKey, null);
Console.WriteLine("RepairRow completed with HttpStatus={0}", repairResult == null ? "NULL" : repairResult.HttpStatusCode.ToString());
// Check the entry at the Head is still locked i.e. RepairRow was NOP
headAfterRepairWasLocked = HeadIsLocked(entry);
Console.WriteLine("Signal the commit to Head job");
repaireDone = true;
});
// 2 - Configure Mangler ...
string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0];
Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper);
ProxyBehavior[] behaviors = new[]
{
TamperBehaviors.TamperAllRequestsIf(
(session =>
{
Console.WriteLine("Delaying commit to the Head ... => signal RepairRow job");
// Let RepairRow task go through
triggerRepair = true;
int iter = 0;
while (!repaireDone)
{
// TODO: break the loop after couple of iteration ...
Thread.Sleep(100);
Console.WriteLine("Waiting on RepairRow to finish ({0}) ...", ++iter);
}
Console.WriteLine("Request a commit to the head");
}),
(session =>
{
// Commit on head i.e. a PUT with RowLock == false
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.HTTPMethodIs("PUT") &&
session.GetRequestBodyAsString().Contains("\"_rtable_RowLock\":false"))
{
return(true);
}
return(false);
}))
};
using (new HttpMangler(false, behaviors))
{
Console.WriteLine("Updating entry ...");
entry = this.rtableWrapper.FindRow(entry.PartitionKey, entry.RowKey);
entry.Message = "updated message";
this.rtableWrapper.ReplaceRow(entry);
}
Assert.IsTrue(triggerRepair);
Assert.IsTrue(repairResult != null && repairResult.HttpStatusCode == (int)HttpStatusCode.OK, "Repair failed.");
Assert.IsTrue(repaireDone);
Assert.IsTrue(headAfterRepairWasLocked);
Console.WriteLine("DONE. Test passed.");
}
public void TxnViewExpiresAndViewIdHasChanged()
{
// Insert an entry
string entityPartitionKey = "jobType-ReplaceWhenTxnViewExpires";
string entityRowKey = "jobType-ReplaceWhenTxnViewExpires";
this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);
Console.WriteLine("Inserting entry ...");
var entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, "insert message");
this.rtableWrapper.InsertRow(entry);
// Retrieve the config ...
ReplicatedTableConfiguration config;
ReplicatedTableQuorumReadResult readStatus = this.configurationService.RetrieveConfiguration(out config);
Assert.IsTrue(readStatus.Code == ReplicatedTableQuorumReadCode.Success);
// Reconfigure RTable config with a short LeaseDuration
int leaseDurationInSec = 3;
config.LeaseDuration = leaseDurationInSec;
this.configurationService.UpdateConfiguration(config);
string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0];
Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper);
// Delay behavior
ProxyBehavior[] behaviors = new[]
{
DelayBehaviors.DelayAllResponsesIf(
1,
(session =>
{
var body = session.GetRequestBodyAsString();
// Delay lock of the Head response enough so the txView expires ...
// And, upload a new config with a new viewId
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.HTTPMethodIs("PUT") &&
body.Contains("\"_rtable_Operation\":\"Replace\"") &&
body.Contains("\"_rtable_RowLock\":true"))
{
// Upload a new config with a new ViewId
View view = this.configurationService.GetTableView(this.repTable.TableName);
config.GetView(view.Name).ViewId++;
this.configurationService.UpdateConfiguration(config, false);
// ensure txView expires
Thread.Sleep(2 * leaseDurationInSec * 1000);
return(true);
}
return(false);
})),
};
// Launch an Update
using (new HttpMangler(false, behaviors))
{
try
{
entry = this.rtableWrapper.ReadEntity(entityPartitionKey, entityRowKey);
entry.Message = "update message";
var table = new ReplicatedTable(this.repTable.TableName, this.configurationService);
TableOperation replaceOperation = TableOperation.Replace(entry);
// Get a snapshot of txView before transaction starts ...
View viewBeforeTx = this.configurationService.GetTableView(this.repTable.TableName);
// Transaction will get delayed 2*LeaseDuration sec. and Config/ViewId will be changed
TableResult replaceResult = table.Execute(replaceOperation);
Assert.IsTrue(viewBeforeTx.IsExpired(), "txView expected to expire!");
Assert.AreNotEqual(this.configurationService.GetTableView(this.repTable.TableName).ViewId, viewBeforeTx.ViewId, "ViewId should have changed!");
Assert.IsNotNull(replaceResult, "upsertResult = null");
Assert.AreNotEqual((int)HttpStatusCode.NoContent, replaceResult.HttpStatusCode, "upsertResult.HttpStatusCode mismatch");
}
catch (ReplicatedTableStaleViewException ex)
{
// ValidateTxnView is called form everywhere ... we may endup here too
switch (ex.ErrorCode)
{
case ReplicatedTableViewErrorCodes.ViewIdChanged:
break;
default:
Console.WriteLine(ex);
Assert.IsTrue(false);
break;
}
}
}
}
public void RTableRepairTable()
{
// Insert entity
Assert.IsTrue(this.repTable.Exists(), "RTable does not exist");
View fullView = configurationWrapper.GetWriteView();
List <ReplicaInfo> fullViewReplicas = new List <ReplicaInfo>();
for (int i = 0; i <= fullView.TailIndex; i++)
{
fullViewReplicas.Add(fullView.GetReplicaInfo(i));
}
List <ReplicaInfo> newReplicas = new List <ReplicaInfo>();
for (int i = 1; i <= fullView.TailIndex; i++)
{
newReplicas.Add(fullView.GetReplicaInfo(i));
}
SampleRTableEntity customer1 = new SampleRTableEntity("firstName1", "lastName1", "*****@*****.**");
TableOperation operation = TableOperation.Insert(customer1);
TableResult result = repTable.Execute(operation);
Assert.AreNotEqual(null, result, "result = null");
Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch");
SampleRTableEntity customer2 = new SampleRTableEntity("firstName2", "lastName2", "*****@*****.**");
operation = TableOperation.Insert(customer2);
result = repTable.Execute(operation);
Assert.AreNotEqual(null, result, "result = null");
Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch");
customer2 = (SampleRTableEntity)result.Result;
SampleRTableEntity customer3 = new SampleRTableEntity("firstName3", "lastName3", "*****@*****.**");
operation = TableOperation.Insert(customer3);
result = repTable.Execute(operation);
Assert.AreNotEqual(null, result, "result = null");
Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch");
ReadFromIndividualAccountsDirectly(customer1.PartitionKey, customer1.RowKey, true);
ReadFromIndividualAccountsDirectly(customer2.PartitionKey, customer2.RowKey, true);
ReadFromIndividualAccountsDirectly(customer3.PartitionKey, customer3.RowKey, true);
// remove replica from the head
this.UpdateConfiguration(newReplicas, 0);
Assert.IsTrue(configurationWrapper.IsViewStable());
// delete a row
TableOperation deleteOperation = TableOperation.Delete(customer1);
result = repTable.Execute(deleteOperation);
Assert.AreNotEqual(null, result, "result = null");
Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch");
// add a row
SampleRTableEntity customer4 = new SampleRTableEntity("firstName4", "lastName4", "*****@*****.**");
operation = TableOperation.Insert(customer4);
result = repTable.Execute(operation);
Assert.AreNotEqual(null, result, "result = null");
Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch");
// replace a row
customer2.Message = "updated after view update";
operation = TableOperation.Replace(customer2);
result = repTable.Execute(operation);
Assert.AreNotEqual(null, result, "result = null");
Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch");
//Add replica at head
this.UpdateConfiguration(fullViewReplicas, 1);
// repair table on the new head
Console.WriteLine("Calling repair table");
ReconfigurationStatus status = repTable.RepairTable(0, null);
Assert.AreEqual(ReconfigurationStatus.SUCCESS, status, "RepairTable status is not success: {0}", status);
ReadFromIndividualAccountsDirectly(customer1.PartitionKey, customer1.RowKey, true);
ReadFromIndividualAccountsDirectly(customer2.PartitionKey, customer2.RowKey, true);
ReadFromIndividualAccountsDirectly(customer3.PartitionKey, customer3.RowKey, true);
ReadFromIndividualAccountsDirectly(customer4.PartitionKey, customer4.RowKey, true);
}
public void BatchOperationExceptionWhenUsingSmallerViewId()
{
long currentViewId = 100;
long badViewId = currentViewId - 1;
this.UpdateConfiguration(replicas, 0, false, currentViewId);
string jobType = "jobType-BatchOperationExceptionWhenUsingSmallerViewId";
string jobId = "jobId-BatchOperationExceptionWhenUsingSmallerViewId";
int count = 3; // number of operations in the batch _rtable_Operation
List<TableOperationType> opTypes = new List<TableOperationType>()
{
TableOperationType.Replace,
TableOperationType.InsertOrReplace,
TableOperationType.Delete,
};
//
// Insert
//
string jobIdTemplate = jobId + "-{0}";
string messageTemplate = "message-{0}";
string updatedMessageTemplate = "updated-" + messageTemplate;
string partitionKey = string.Empty;
//
// Insert entities
//
for (int i = 0; i < count; i++)
{
SampleRTableEntity originalEntity = new SampleRTableEntity(
jobType,
string.Format(jobIdTemplate, i),
string.Format(messageTemplate, i));
this.repTable.Execute(TableOperation.Insert(originalEntity));
partitionKey = originalEntity.PartitionKey;
}
//
// Retrieve entities and use them to create batchOperation to Replace or Delete
//
IEnumerable<SampleRTableEntity> allEntities = this.rtableWrapper.GetAllRows(partitionKey);
TableBatchOperation batchOperation = new TableBatchOperation();
int m = 0;
foreach (SampleRTableEntity entity in allEntities)
{
Console.WriteLine("{0}", entity.ToString());
Console.WriteLine("---------------------------------------");
if (opTypes[m] == TableOperationType.Replace)
{
SampleRTableEntity replaceEntity = new SampleRTableEntity(
entity.JobType,
entity.JobId,
string.Format(updatedMessageTemplate, m))
{
ETag = entity._rtable_Version.ToString()
};
batchOperation.Replace(replaceEntity);
}
else if (opTypes[m] == TableOperationType.InsertOrReplace)
{
SampleRTableEntity replaceEntity = new SampleRTableEntity(
entity.JobType,
entity.JobId,
string.Format(updatedMessageTemplate, m))
{
ETag = entity._rtable_Version.ToString()
};
batchOperation.InsertOrReplace(replaceEntity);
}
else if (opTypes[m] == TableOperationType.Delete)
{
entity.ETag = entity._rtable_Version.ToString();
batchOperation.Delete(entity);
}
else
{
throw new ArgumentException(
string.Format("opType={0} is NOT supported", opTypes[m]),
"opType");
}
m++;
}
//
// Call ModifyConfigurationBlob to change the viewId of the wrapper to an older value
//
Console.WriteLine("Changing the viewId to badViewId {0}", badViewId);
this.UpdateConfiguration(replicas, 0, false, badViewId);
//
// Execute Batch _rtable_Operation with bad viewId
//
Console.WriteLine("\nCalling BatchOperation with badViewId...");
try
{
this.repTable.ExecuteBatch(batchOperation);
}
catch (ReplicatedTableStaleViewException ex)
{
Console.WriteLine("Get this RTableStaleViewException: {0}", ex.Message);
Assert.IsTrue(ex.Message.Contains(string.Format("current _rtable_ViewId {0} is smaller than", badViewId)), "Got unexpected exception message");
}
}
/// <summary>
/// Copy the values of JobType, JobId, Message to the specified object
/// </summary>
/// <param name="dst"></param>
public void CopyTo(SampleRTableEntity dst)
{
dst.JobType = this.JobType;
dst.JobId = this.JobId;
dst.Message = this.Message;
}
/// <summary>
/// Helper function to insert the specified set of jobType and jobId. It is expected that InsertRow() will fail.
/// </summary>
/// <param name="jobType"></param>
/// <param name="jobId"></param>
/// <param name="entityMessage"></param>
protected void PerformInsertOperationAndExpectToFail(string jobType, string jobId, string entityMessage)
{
Console.WriteLine("\nValidating Insert operation will fail. jobType={0} jobId={1}...", jobType, jobId);
for (int i = 0; i < this.numberOfPartitions; i++)
{
List<string> rowKeys = new List<string>(); // list of rowKey for the given paritionKey
string partitionKey;
string rowKey;
SampleRTableEntity.GenerateKeys(this.GenerateJobType(jobType, i), "don't care", out partitionKey, out rowKey);
for (int j = 0; j < this.numberOfRowsPerPartition; j++)
{
SampleRTableEntity sampleRtableEntity = new SampleRTableEntity(
this.GenerateJobType(jobType, i),
this.GenerateJobId(jobId, i, j),
this.GenerateMessage(entityMessage, i, j));
int attempts = 1;
bool failed = false;
while (attempts < 3)
{
try
{
Console.WriteLine("attempts={0}. partitionKey={1} rowKey={2}. Calling InsertRow API...",
attempts, partitionKey, sampleRtableEntity.RowKey);
this.rtableWrapper.InsertRow(sampleRtableEntity);
failed = false;
break; // get out of while(attempts) if no RTableConflictException
}
catch (RTableConflictException)
{
failed = true;
Console.WriteLine("Got RTableConflictException. attempts={0}", attempts);
attempts++;
System.Threading.Thread.Sleep(1000);
}
}
Assert.IsTrue(failed, "InsertRow() actually passed when we expect it to fail!");
}
}
Console.WriteLine("Passed 'Insert will fail' validation.\n");
}
public void A00TwoInsertOrReplaceCallsConflictingOnTheHead()
{
this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);
string entityPartitionKey = "jobType-DelayInsertOrReplaceRowHeadTest";
string entityRowKey = "jobId-DelayInsertOrReplaceRowHeadTest";
this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);
// Insert one entry
Console.WriteLine("Inserting entry ...");
var entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, "insert message");
this.rtableWrapper.InsertRow(entry);
string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0];
Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper);
int delayInMs = 3000;
bool firstWritterInitiatingCommit = false;
bool secondWritterTriedLockingHead = false;
// Delay behavior
ProxyBehavior[] behaviors = new[]
{
// Writter-1 tampering
TamperBehaviors.TamperAllRequestsIf(
(session =>
{
int iter = 0;
// Signal Writter-2
firstWritterInitiatingCommit = true;
// Blobk commit to head ... until Writter-2 try to lock the head
while (!secondWritterTriedLockingHead)
{
Console.WriteLine("Writter-1 waiting on Writter-2 to try to lock the Head (#{0})", iter);
Thread.Sleep(delayInMs);
if (++iter > 10)
{
break;
}
}
}),
(session =>
{
var body = session.GetRequestBodyAsString();
// Writter-1 committing to head
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.HTTPMethodIs("PUT") &&
body.Contains("\"_rtable_Operation\":\"Replace\"") &&
body.Contains("\"_rtable_RowLock\":false") &&
body.Contains("\"Message\":\"upsert message 0\"")
)
{
return(true);
}
return(false);
})),
// Writter-2 tampering
TamperBehaviors.TamperAllRequestsIf(
(session =>
{
// Block till Writter-1 issues a commit to head
while (!firstWritterInitiatingCommit)
{
Console.WriteLine("Writter-2 waiting on Writter-1 to issue a commit to head");
Thread.Sleep(delayInMs);
}
}),
(session =>
{
var body = session.GetRequestBodyAsString();
// Writter-2 locking the head
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.HTTPMethodIs("PUT") &&
body.Contains("\"_rtable_Operation\":\"Replace\"") &&
body.Contains("\"_rtable_RowLock\":true") &&
body.Contains("\"Message\":\"upsert message 1\"")
)
{
return(true);
}
return(false);
})),
// Delay Writter-2 lock-to-the-head's response, so Writter-1 can continue with its commit.
DelayBehaviors.DelayAllResponsesIf(
delayInMs,
(session =>
{
var body = session.GetRequestBodyAsString();
// Writter-2 locking the head response
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.HTTPMethodIs("PUT") &&
body.Contains("\"_rtable_Operation\":\"Replace\"") &&
body.Contains("\"_rtable_RowLock\":true") &&
body.Contains("\"Message\":\"upsert message 1\"")
)
{
// Signal Writter-1 so it can continue with commit to head
secondWritterTriedLockingHead = true;
return(true);
}
return(false);
})),
};
// Launch 2 concurrent Upserts
var results = new TableResult[2];
using (new HttpMangler(false, behaviors))
{
Parallel.For(0, 2, (index) =>
{
entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, string.Format("upsert message {0}", index));
try
{
var table = new ReplicatedTable(this.repTable.TableName, this.configurationService);
TableOperation operation = TableOperation.InsertOrReplace(entry);
results[index] = table.Execute(operation);
}
catch (AggregateException ex)
{
Console.WriteLine(ex);
}
});
}
// Writter-1 suceed?
Assert.AreEqual((int)HttpStatusCode.NoContent, results[0].HttpStatusCode, "Writter-1 expected to suceed!");
// Writter-2 suceeded?
Assert.AreEqual((int)HttpStatusCode.NoContent, results[1].HttpStatusCode, "Writter-2 expected to suceed!");
// Writter-2 upsert succeeded
SampleRTableEntity upsertedEntity = this.rtableWrapper.ReadEntity(entityPartitionKey, entityRowKey);
Assert.NotNull(upsertedEntity, "Writter-2 upsert succeeded");
Assert.AreEqual(upsertedEntity.Message, string.Format("upsert message {0}", 1), "Writter-2 upsert succeeded");
}
/// <summary>
/// Helper function to insert the specified set of jobType and jobId and validate the results.
/// Make sure there are no existing entries for the jobType and jobId before calling this function.
/// </summary>
/// <param name="jobType"></param>
/// <param name="jobId"></param>
/// <param name="entityMessage"></param>
protected void PerformInsertOperationAndValidate(string jobType, string jobId, string entityMessage)
{
Console.WriteLine("\nValidating Insert operation...");
for (int i = 0; i < this.numberOfPartitions; i++)
{
List<string> rowKeys = new List<string>(); // list of rowKey for the given paritionKey
string partitionKey;
string rowKey;
SampleRTableEntity.GenerateKeys(this.GenerateJobType(jobType, i), "don't care", out partitionKey, out rowKey);
//
// GetAllRows() to confirm nothing exists and then call InsertRow()
//
Console.WriteLine("Calling GetAllRows() for partition {0}, expecting 0 rows...", i);
IEnumerable<SampleRTableEntity> allRows = this.rtableWrapper.GetAllRows(partitionKey);
Assert.AreEqual(0, allRows.Count(), "Partition {0} should have 0 rows in order for InsertRow() to work.", i);
for (int j = 0; j < this.numberOfRowsPerPartition; j++)
{
SampleRTableEntity sampleRtableEntity = new SampleRTableEntity(
this.GenerateJobType(jobType, i),
this.GenerateJobId(jobId, i, j),
this.GenerateMessage(entityMessage, i, j));
int attempts = 1;
bool passed = true;
while (attempts < 3)
{
try
{
Console.WriteLine("attempts={0}. partitionKey={1} rowKey={2}. Calling InsertRow API...",
attempts, partitionKey, sampleRtableEntity.RowKey);
this.rtableWrapper.InsertRow(sampleRtableEntity);
passed = true;
break; // get out of while(attempts) if no RTableConflictException
}
catch (RTableConflictException)
{
passed = false;
Console.WriteLine("Got RTableConflictException. attempts={0}", attempts);
attempts++;
System.Threading.Thread.Sleep(1000);
}
}
Assert.IsTrue(passed, "Keep getting RTableConflictException when calling InsertRow API");
}
//
// FindRow()
//
Console.WriteLine("Calling FindRow() for partitionKey={0}", partitionKey);
for (int j = 0; j < rowKeys.Count; j++)
{
SampleRTableEntity retrievedEntity = this.rtableWrapper.FindRow(partitionKey, rowKeys[j]);
this.ValidateRetrievedRTableEntity(retrievedEntity, jobType, jobId, entityMessage, i, j);
}
}
Console.WriteLine("Passed Insert validation.\n");
}
/// <summary>
/// Helper function to perform Delete, InsertOrReplace, Merge, or Replace operation for the specified set of jobType and jobId.
/// And validate the results.
/// </summary>
/// <param name="opType"></param>
/// <param name="jobType"></param>
/// <param name="jobId"></param>
/// <param name="partition"></param>
/// <param name="row"></param>
/// <param name="originalMessage"></param>
/// <param name="updatedMessage"></param>
protected void PerformIndividualOperationAndValidate(
TableOperationType opType,
string jobType,
string jobId,
int partition,
int row,
string originalMessage,
string updatedMessage = "")
{
Console.WriteLine("\nValidating {0} operation: updatedEntityMessage={1}...", opType, message);
string partitionKey;
string rowKey;
SampleRTableEntity.GenerateKeys(this.GenerateJobType(jobType, partition), this.GenerateJobId(jobId, partition, row), out partitionKey, out rowKey);
Console.WriteLine("PartitionKey = {0}, RowKey = {1}", partitionKey, rowKey);
int attempts = 1;
bool passed = true;
bool resetRow = false;
SampleRTableEntity retrievedEntity = null;
while (attempts < 3)
{
try
{
if (opType != TableOperationType.Insert)
{
retrievedEntity = this.rtableWrapper.FindRow(partitionKey, rowKey);
}
Console.WriteLine("attempts={0}. partitionKey={1} rowKey={2}. Calling {3} API...", attempts, partitionKey, rowKey, opType);
switch (opType)
{
case TableOperationType.Delete:
{
this.rtableWrapper.DeleteRow(retrievedEntity);
//Validate
try
{
retrievedEntity = this.rtableWrapper.FindRow(partitionKey, rowKey);
Assert.Fail("After DeleteRow() was called, FindRow() did not throw RTableResourceNotFoundException");
}
catch (RTableResourceNotFoundException)
{
}
}
break;
case TableOperationType.Insert:
{
SampleRTableEntity sampleRtableEntity = new SampleRTableEntity(
this.GenerateJobType(jobType, partition),
this.GenerateJobId(jobId, partition, row),
this.GenerateMessage(originalMessage, partition, row));
this.rtableWrapper.InsertRow(sampleRtableEntity);
//Validate
try
{
retrievedEntity = this.rtableWrapper.FindRow(partitionKey, rowKey);
}
catch (RTableResourceNotFoundException)
{
Assert.Fail("After InsertRow() was called, FindRow() threw RTableResourceNotFoundException");
}
}
break;
case TableOperationType.InsertOrMerge:
{
retrievedEntity.Message = this.GenerateMessage(updatedMessage, partition, row);
this.rtableWrapper.InsertOrMergeRow(retrievedEntity);
retrievedEntity = this.rtableWrapper.FindRow(partitionKey, rowKey);
this.ValidateRetrievedRTableEntity(retrievedEntity, jobType, jobId, updatedMessage, partition, row);
}
break;
case TableOperationType.InsertOrReplace:
{
retrievedEntity.Message = this.GenerateMessage(updatedMessage, partition, row);
this.rtableWrapper.InsertOrReplaceRow(retrievedEntity);
retrievedEntity = this.rtableWrapper.FindRow(partitionKey, rowKey);
this.ValidateRetrievedRTableEntity(retrievedEntity, jobType, jobId, updatedMessage, partition, row);
resetRow = true;
}
break;
case TableOperationType.Merge:
{
retrievedEntity.Message = this.GenerateMessage(updatedMessage, partition, row);
this.rtableWrapper.MergeRow(retrievedEntity);
retrievedEntity = this.rtableWrapper.FindRow(partitionKey, rowKey);
this.ValidateRetrievedRTableEntity(retrievedEntity, jobType, jobId, updatedMessage, partition, row);
}
break;
case TableOperationType.Replace:
{
retrievedEntity.Message = this.GenerateMessage(updatedMessage, partition, row);
this.rtableWrapper.ReplaceRow(retrievedEntity);
retrievedEntity = this.rtableWrapper.FindRow(partitionKey, rowKey);
this.ValidateRetrievedRTableEntity(retrievedEntity, jobType, jobId, updatedMessage, partition, row);
resetRow = true;
}
break;
case TableOperationType.Retrieve:
{
this.rtableWrapper.FindRow(partitionKey, rowKey);
}
break;
default:
{
throw new InvalidOperationException(string.Format("opType={0} is NOT supported", opType));
}
}
passed = true;
break; // get out of while(attempts) if no RTableConflictException
}
catch (RTableConflictException)
{
passed = false;
Console.WriteLine("Got RTableConflictException. attempts={0}", attempts);
attempts++;
System.Threading.Thread.Sleep(1000);
}
}
Assert.IsTrue(passed, "Keep getting RTableConflictException when calling {0} API", opType);
Console.WriteLine("Passed {0} validation.\n", opType);
//Reset row to original values so subsequent updates on the same row get validated correctly
if (resetRow == true)
{
Console.WriteLine("Resetting row to original values");
retrievedEntity.Message = this.GenerateMessage(originalMessage, partition, row);
this.rtableWrapper.ReplaceRow(retrievedEntity);
}
}
/// <summary>
/// Helper function to validate whether "retrievedEntity" is correct or not.
/// </summary>
/// <param name="retrievedEntity"></param>
/// <param name="jobType"></param>
/// <param name="jobId"></param>
/// <param name="message"></param>
/// <param name="partition"></param>
/// <param name="row"></param>
/// <param name="checkViewId"></param>
protected void ValidateRetrievedRTableEntity(
SampleRTableEntity retrievedEntity,
string jobType,
string jobId,
string message,
int partition,
int row,
bool checkViewId = true)
{
Assert.AreEqual(
this.GenerateJobType(jobType, partition),
retrievedEntity.JobType,
"JobType of retrievedEntity mismatch");
Assert.AreEqual(
this.GenerateJobId(jobId, partition, row),
retrievedEntity.JobId,
"JobId of retrievedEntity mismatch");
Assert.AreEqual(
this.GenerateMessage(message, partition, row),
retrievedEntity.Message,
"Message of retrievedEntity mismatch");
if (checkViewId)
{
Assert.AreEqual(
this.configurationWrapper.GetReadView().ViewId,
retrievedEntity._rtable_ViewId,
"_rtable_ViewId of retrievedEntity mismatch");
}
}
public void RTableRepairTable()
{
// Insert entity
Assert.IsTrue(this.repTable.Exists(), "RTable does not exist");
View fullView = configurationWrapper.GetWriteView();
List<ReplicaInfo> fullViewReplicas = new List<ReplicaInfo>();
for (int i = 0; i <= fullView.TailIndex; i++)
{
fullViewReplicas.Add(fullView.GetReplicaInfo(i));
}
List<ReplicaInfo> newReplicas = new List<ReplicaInfo>();
for (int i = 1; i <= fullView.TailIndex; i++)
{
newReplicas.Add(fullView.GetReplicaInfo(i));
}
SampleRTableEntity customer1 = new SampleRTableEntity("firstName1", "lastName1", "*****@*****.**");
TableOperation operation = TableOperation.Insert(customer1);
TableResult result = repTable.Execute(operation);
Assert.AreNotEqual(null, result, "result = null");
Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch");
SampleRTableEntity customer2 = new SampleRTableEntity("firstName2", "lastName2", "*****@*****.**");
operation = TableOperation.Insert(customer2);
result = repTable.Execute(operation);
Assert.AreNotEqual(null, result, "result = null");
Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch");
customer2 = (SampleRTableEntity) result.Result;
SampleRTableEntity customer3 = new SampleRTableEntity("firstName3", "lastName3", "*****@*****.**");
operation = TableOperation.Insert(customer3);
result = repTable.Execute(operation);
Assert.AreNotEqual(null, result, "result = null");
Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch");
ReadFromIndividualAccountsDirectly(customer1.PartitionKey, customer1.RowKey, true);
ReadFromIndividualAccountsDirectly(customer2.PartitionKey, customer2.RowKey, true);
ReadFromIndividualAccountsDirectly(customer3.PartitionKey, customer3.RowKey, true);
// remove replica from the head
this.UpdateConfiguration(newReplicas, 0);
Assert.IsTrue(configurationWrapper.IsViewStable());
// delete a row
TableOperation deleteOperation = TableOperation.Delete(customer1);
result = repTable.Execute(deleteOperation);
Assert.AreNotEqual(null, result, "result = null");
Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch");
// add a row
SampleRTableEntity customer4 = new SampleRTableEntity("firstName4", "lastName4", "*****@*****.**");
operation = TableOperation.Insert(customer4);
result = repTable.Execute(operation);
Assert.AreNotEqual(null, result, "result = null");
Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch");
// replace a row
customer2.Message = "updated after view update";
operation = TableOperation.Replace(customer2);
result = repTable.Execute(operation);
Assert.AreNotEqual(null, result, "result = null");
Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch");
//Add replica at head
this.UpdateConfiguration(fullViewReplicas, 1);
// repair table on the new head
Console.WriteLine("Calling repair table");
ReconfigurationStatus status = repTable.RepairTable(0, null);
Assert.AreEqual(ReconfigurationStatus.SUCCESS, status, "RepairTable status is not success: {0}", status);
ReadFromIndividualAccountsDirectly(customer1.PartitionKey, customer1.RowKey, true);
ReadFromIndividualAccountsDirectly(customer2.PartitionKey, customer2.RowKey, true);
ReadFromIndividualAccountsDirectly(customer3.PartitionKey, customer3.RowKey, true);
ReadFromIndividualAccountsDirectly(customer4.PartitionKey, customer4.RowKey, true);
}
/// <summary>
/// Helper function to execute the specified BatchOperation after HttpMangler is turned off and validate correctness.
/// </summary>
/// <param name="count">Number of operations in the batch</param>
/// <param name="partitionKey">partitionKey to operate on</param>
/// <param name="jobType">partitionKey is generated from jobType</param>
/// <param name="jobId">RowKey is generated from jobId. JobIdTemplate = jobId-{0}</param>
/// <param name="opTypes">Specifies the batch operation to be performed</param>
protected void ExecuteBatchOperationAndValidate(
int count,
string partitionKey,
string jobType,
string jobId,
List<TableOperationType> opTypes)
{
Console.WriteLine("\nExecuteBatchOperationAndValidate(): Trying to batch update {0} entities...", count);
Assert.IsNotNull(opTypes, "opTypes = null");
Assert.AreEqual(count, opTypes.Count, "count and opTypes.Count should be the same");
string jobIdTemplate = jobId + "-{0}";
string replaceMessageTemplate = "updated-after-httpMangler-{0}";
IEnumerable<SampleRTableEntity> allEntities = null;
TableBatchOperation batchOperation = null;
int m = 0;
bool gotExceptionInLastAttempt = true;
int retries = 0;
while (retries < MaxRetries)
{
try
{
//
// GetAllRows()
//
allEntities = this.rtableWrapper.GetAllRows(partitionKey);
//
// Create a batchOperation to perform the specified opTypes
//
batchOperation = new TableBatchOperation();
m = 0;
foreach (SampleRTableEntity entity in allEntities)
{
if (opTypes[m] == TableOperationType.Replace)
{
// set up the new entity to be used in the batch operation
SampleRTableEntity replaceEntity = new SampleRTableEntity(
entity.JobType,
entity.JobId,
string.Format(replaceMessageTemplate, m))
{
ETag = entity._rtable_Version.ToString()
};
// add the operation to the batch operation
batchOperation.Replace(replaceEntity);
}
else if (opTypes[m] == TableOperationType.Delete)
{
batchOperation.Delete(entity);
}
else
{
throw new ArgumentException(
string.Format("opType={0} is NOT supported", opTypes[m]),
"opType");
}
m++;
}
//
// Call this.repTable.ExecuteBatch(batchOperation);
//
if (batchOperation.Count == 0)
{
Console.WriteLine("retries={0}. Done. batchOperation.Count == 0", retries);
}
else
{
this.repTable.ExecuteBatch(batchOperation);
Console.WriteLine("retries={0}. Done ExecuteBatch()", retries);
}
gotExceptionInLastAttempt = false;
break;
}
catch (RTableConflictException ex)
{
Console.WriteLine("retries={0}. ExecuteBatch() got an RTableConflictException: {1}",
retries, ex.ToString());
retries++;
gotExceptionInLastAttempt = true;
Thread.Sleep(ConflictExceptionSleepTimeInMsec);
}
}
Console.WriteLine("gotExceptionInLastAttempt={0} retries={1} MaxRetries={2}",
gotExceptionInLastAttempt, retries, MaxRetries);
Assert.IsFalse(gotExceptionInLastAttempt, "The last API call should not throw an exception.");
//
// Final validation
//
Console.WriteLine("Final validation...");
allEntities = this.rtableWrapper.GetAllRows(partitionKey);
m = 0;
int opTypesCounter = 0;
foreach (SampleRTableEntity entity in allEntities)
{
Console.WriteLine("{0}", entity.ToString());
Console.WriteLine("---------------------------------------");
// If the operation is Delete, then skip it. No need to validate.
while (opTypesCounter < count && opTypes[m] == TableOperationType.Delete)
{
m++;
}
Assert.IsTrue(m < count, "m={0} count={1}: m shoud be < count, but it is not.", m, count);
if (opTypes[m] == TableOperationType.Replace)
{
Assert.AreEqual(string.Format(jobType, m), entity.JobType, "JobType does not match");
Assert.AreEqual(string.Format(jobIdTemplate, m), entity.JobId, "JobId does not match");
Assert.AreEqual(string.Format(replaceMessageTemplate, m), entity.Message, "Message does not match");
m++;
}
else
{
throw new ArgumentException(
string.Format("opType={0} is NOT supported", opTypes[opTypesCounter]),
"opType");
}
}
for (int i = 0; i < count; i++)
{
this.ReadFromIndividualAccountsDirectly(
jobType,
string.Format(jobIdTemplate, i),
true);
}
Console.WriteLine("Passed final validation.");
}
