public void RTableRepairTable()
{
// Insert entity
Assert.IsTrue(this.repTable.Exists(), "RTable does not exist");
View fullView = configurationWrapper.GetWriteView();
List <ReplicaInfo> fullViewReplicas = new List <ReplicaInfo>();
for (int i = 0; i <= fullView.TailIndex; i++)
{
fullViewReplicas.Add(fullView.GetReplicaInfo(i));
}
List <ReplicaInfo> newReplicas = new List <ReplicaInfo>();
for (int i = 1; i <= fullView.TailIndex; i++)
{
newReplicas.Add(fullView.GetReplicaInfo(i));
}
SampleRTableEntity customer1 = new SampleRTableEntity("firstName1", "lastName1", "*****@*****.**");
TableOperation operation = TableOperation.Insert(customer1);
TableResult result = repTable.Execute(operation);
Assert.AreNotEqual(null, result, "result = null");
Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch");
SampleRTableEntity customer2 = new SampleRTableEntity("firstName2", "lastName2", "*****@*****.**");
operation = TableOperation.Insert(customer2);
result = repTable.Execute(operation);
Assert.AreNotEqual(null, result, "result = null");
Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch");
customer2 = (SampleRTableEntity)result.Result;
SampleRTableEntity customer3 = new SampleRTableEntity("firstName3", "lastName3", "*****@*****.**");
operation = TableOperation.Insert(customer3);
result = repTable.Execute(operation);
Assert.AreNotEqual(null, result, "result = null");
Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch");
ReadFromIndividualAccountsDirectly(customer1.PartitionKey, customer1.RowKey, true);
ReadFromIndividualAccountsDirectly(customer2.PartitionKey, customer2.RowKey, true);
ReadFromIndividualAccountsDirectly(customer3.PartitionKey, customer3.RowKey, true);
// remove replica from the head
this.UpdateConfiguration(newReplicas, 0);
Assert.IsTrue(configurationWrapper.IsViewStable());
// delete a row
TableOperation deleteOperation = TableOperation.Delete(customer1);
result = repTable.Execute(deleteOperation);
Assert.AreNotEqual(null, result, "result = null");
Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch");
// add a row
SampleRTableEntity customer4 = new SampleRTableEntity("firstName4", "lastName4", "*****@*****.**");
operation = TableOperation.Insert(customer4);
result = repTable.Execute(operation);
Assert.AreNotEqual(null, result, "result = null");
Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch");
// replace a row
customer2.Message = "updated after view update";
operation = TableOperation.Replace(customer2);
result = repTable.Execute(operation);
Assert.AreNotEqual(null, result, "result = null");
Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch");
//Add replica at head
this.UpdateConfiguration(fullViewReplicas, 1);
// repair table on the new head
Console.WriteLine("Calling repair table");
ReconfigurationStatus status = repTable.RepairTable(0, null);
Assert.AreEqual(ReconfigurationStatus.SUCCESS, status, "RepairTable status is not success: {0}", status);
ReadFromIndividualAccountsDirectly(customer1.PartitionKey, customer1.RowKey, true);
ReadFromIndividualAccountsDirectly(customer2.PartitionKey, customer2.RowKey, true);
ReadFromIndividualAccountsDirectly(customer3.PartitionKey, customer3.RowKey, true);
ReadFromIndividualAccountsDirectly(customer4.PartitionKey, customer4.RowKey, true);
}
public void RemoveAndReAddHeadReplicaTest()
{
try
{
int retrieveAttemtps = 2;
this.LoadTestConfiguration();
string tableName = this.GenerateRandomTableName();
bool useHttps = true;
long viewId = 1;
string viewIdString = viewId.ToString();
int headReplicaAccountIndex = 0;
int tailReplicaAccountIndex = 1;
List <int> actualStorageAccountsToBeUsed = new List <int>()
{
headReplicaAccountIndex, tailReplicaAccountIndex
};
bool convertXStoreTableMode = false;
Console.WriteLine("Setting up RTable that has a Head Replica and a Tail Replica...");
this.SetupRTableEnv(
tableName,
useHttps,
viewIdString,
actualStorageAccountsToBeUsed,
convertXStoreTableMode,
1);
Assert.AreEqual(2, this.actualStorageAccountsUsed.Count, "Two stoarge accounts should be used at this point.");
Console.WriteLine("Inserting entities to the RTable...");
this.numberOfPartitions = 1;
this.numberOfRowsPerPartition = 1;
string jobTypeInsert = "jobType-RemoveAndReAddHeadReplicaTest-Insert"; // will call Insert() in Phase One, Two (expect to fail), Three (expect to fail)
string jobIdInsert = "jobId-RemoveAndReAddHeadReplicaTest-Insert";
string jobTypeStale = "jobType-RemoveAndReAddHeadReplicaTest-Stale"; // will call Replace() in Phase One, Two, Three
string jobIdStale = "jobId-RemoveAndReAddHeadReplicaTest-Stale";
string jobTypeStatic = "jobType-RemoveAndReAddHeadReplicaTest-Static"; // will NOT call Replace() in Phase One. Let RepairTable() fix this row.
string jobIdStatic = "jobId-RemoveAndReAddHeadReplicaTest-Static";
string jobTypeDelete = "jobType-RemoveAndReAddHeadReplicaTest-Delete"; // will call Delete() in Phase One, Insert() in Phase Two.
string jobIdDelete = "jobId-RemoveAndReAddHeadReplicaTest-Delete";
string jobTypeReplaceDelete = "jobType-RemoveAndReAddHeadReplicaTest-ReplaceDelete"; // will call Replace() in Phase One, Delete() in Phase Two, Insert() in Phase Three.
string jobIdReplaceDelete = "jobId-RemoveAndReAddHeadReplicaTest-ReplaceDelete";
string originalMessage = "message";
string updatedMessage = "updatedMessage";
string updatedMessage2 = "updatedMessageTwo";
string staticMessage = "staticMessage";
// Insert some entries.
this.PerformInsertOperationAndValidate(jobTypeInsert,
jobIdInsert,
originalMessage);
this.PerformInsertOperationAndValidate(jobTypeStale,
jobIdStale,
originalMessage);
this.PerformInsertOperationAndValidate(jobTypeStatic,
jobIdStatic,
staticMessage);
this.PerformInsertOperationAndValidate(jobTypeDelete,
jobIdDelete,
originalMessage);
this.PerformInsertOperationAndValidate(jobTypeReplaceDelete,
jobIdReplaceDelete,
originalMessage);
Console.WriteLine("\n\nRemoving Head Replica from the chain. Will sleep some time...");
int readViewHeadIndex = 0;
this.actualStorageAccountsUsed = new List <int> {
tailReplicaAccountIndex
};
viewId++;
this.RefreshRTableEnvJsonConfigBlob(viewId, convertXStoreTableMode, readViewHeadIndex);
//
// Phase One:
// Using Tail Replica only. readViewHeadIndex = 0.
//
Console.WriteLine("\n\nPhase One: Head Replica has been removed. Using Tail Replica only. Verifying Replace() and Delete() API...");
for (int i = 0; i < retrieveAttemtps; i++)
{
this.PerformRetrieveOperationAndValidate(jobTypeStale, jobIdStale, originalMessage, false);
}
for (int i = 0; i < retrieveAttemtps; i++)
{
this.PerformRetrieveOperationAndValidate(jobTypeStatic, jobIdStatic, staticMessage, false);
}
for (int i = 0; i < retrieveAttemtps; i++)
{
this.PerformRetrieveOperationAndValidate(jobTypeDelete, jobIdDelete, originalMessage, false);
}
for (int i = 0; i < retrieveAttemtps; i++)
{
this.PerformRetrieveOperationAndValidate(jobTypeReplaceDelete, jobIdReplaceDelete, originalMessage, false);
}
// Replace() API
this.PerformOperationAndValidate(TableOperationType.Replace, jobTypeStale, jobIdStale, updatedMessage);
this.PerformOperationAndValidate(TableOperationType.Replace, jobTypeReplaceDelete, jobIdReplaceDelete, updatedMessage);
// Delete() API
this.PerformOperationAndValidate(TableOperationType.Delete, jobTypeDelete, jobIdDelete);
Console.WriteLine("\n\nAdding the Head Replica back to the chain. readViewHeadIndex = 1. Will sleep some time...");
readViewHeadIndex = 1;
this.actualStorageAccountsUsed = new List <int> {
headReplicaAccountIndex, tailReplicaAccountIndex
};
viewId++;
this.RefreshRTableEnvJsonConfigBlob(viewId, convertXStoreTableMode, readViewHeadIndex);
//
// Phase Two:
// Using Head Replica and Tail Replica. readViewHeadIndex = 1.
// Just added back the Head Replica, which contains stale entries.
//
Console.WriteLine("\n\nPhase Two: Head Replica has been added back. readViewHeadIndex = 1. Verifying Replace(), Delete() and Insert() API...");
// Insert() existing row will fail
Console.WriteLine("Verifying that Insert() an existing row will fail...");
this.PerformInsertOperationAndExpectToFail(jobTypeInsert, jobIdInsert, updatedMessage);
for (int i = 0; i < retrieveAttemtps; i++)
{
this.PerformRetrieveOperationAndValidate(jobTypeStale, jobIdStale, updatedMessage, false);
}
for (int i = 0; i < retrieveAttemtps; i++)
{
this.PerformRetrieveOperationAndValidate(jobTypeStatic, jobIdStatic, staticMessage, false);
}
for (int i = 0; i < retrieveAttemtps; i++)
{
this.PerformRetrieveOperationAndValidate(jobTypeReplaceDelete, jobIdReplaceDelete, updatedMessage, false);
}
// Replace():
this.PerformOperationAndValidate(TableOperationType.Replace, jobTypeStale, jobIdStale, updatedMessage2);
// Verify NotFound:
Console.WriteLine("Verifying jobIdDelete does not exist in Phase Two (Head and Tail Replica; readViewHeadIndex = 1)...");
for (int i = 0; i < retrieveAttemtps; i++)
{
this.ValidateRetrieveNotFound(jobTypeDelete, jobIdDelete);
}
// Insert()
Console.WriteLine("Should be able to Insert() jobIdDelete in Phase Two (Head and Tail Replica; readViewHeadIndex = 1)...");
this.PerformInsertOperationAndValidate(jobTypeDelete,
jobIdDelete,
updatedMessage);
// Delete():
Console.WriteLine("Verifying Delete() jobTypeReplaceDelete in Phase Two (Head and Tail Replica; readViewHeadIndex = 1)...");
this.PerformOperationAndValidate(TableOperationType.Delete, jobTypeReplaceDelete, jobIdReplaceDelete);
Console.WriteLine("\n\nRepairing RTable...");
ReconfigurationStatus repairStatus = this.repTable.RepairTable(1, null);
Assert.AreEqual(ReconfigurationStatus.SUCCESS, repairStatus, "Repair failed.");
Console.WriteLine("\n\nDone repairing. Changing RTable config to use ReadViewHeadIndex=0, will sleep for some time...");
readViewHeadIndex = 0;
viewId++;
this.RefreshRTableEnvJsonConfigBlob(viewId, convertXStoreTableMode, readViewHeadIndex);
//
// Phase Three:
// Using Head Replica and Tail Replica. readViewHeadIndex = 0
//
Console.WriteLine("\n\nPhase Three: RepairTable() completed successfully. readViewHeadIndex = 0. Verifying Replace(), Delete() and Insert() API...");
// Insert() existing row will fail
Console.WriteLine("Verifying that Insert() an existing row will fail...");
this.PerformInsertOperationAndExpectToFail(jobTypeInsert, jobIdInsert, updatedMessage);
for (int i = 0; i < retrieveAttemtps; i++)
{
this.PerformRetrieveOperationAndValidate(jobTypeStale, jobIdStale, updatedMessage2, false);
}
for (int i = 0; i < retrieveAttemtps; i++)
{
this.PerformRetrieveOperationAndValidate(jobTypeStatic, jobIdStatic, staticMessage, false);
}
for (int i = 0; i < retrieveAttemtps; i++)
{
this.PerformRetrieveOperationAndValidate(jobTypeDelete, jobIdDelete, updatedMessage, false);
}
// Verify NotFound:
Console.WriteLine("Verifying jobTypeReplaceDelete does not exist in Phase Three (Head and Tail Replica; readViewHeadIndex = 0)...");
for (int i = 0; i < retrieveAttemtps; i++)
{
this.ValidateRetrieveNotFound(jobTypeReplaceDelete, jobIdReplaceDelete);
}
// Insert()
Console.WriteLine("Should be able to Insert() jobTypeReplaceDelete in Phase Three (Head and Tail Replica; readViewHeadIndex = 0)...");
this.PerformInsertOperationAndValidate(jobTypeReplaceDelete,
jobIdReplaceDelete,
updatedMessage2);
Console.WriteLine("Checking Replace() API...");
this.PerformOperationAndValidate(TableOperationType.Replace, jobTypeStale, jobIdStale, updatedMessage);
this.PerformOperationAndValidate(TableOperationType.Replace, jobTypeStatic, jobIdStatic, updatedMessage);
this.PerformOperationAndValidate(TableOperationType.Replace, jobTypeDelete, jobIdDelete, updatedMessage2);
Console.WriteLine("DONE. Test passed.");
}
finally
{
base.DeleteAllRtableResources();
}
}
public void AddNewHeadReplicaTest()
{
try
{
int retrieveAttemtps = 2;
this.LoadTestConfiguration();
string tableName = this.GenerateRandomTableName();
bool useHttps = true;
long viewId = 1;
string viewIdString = viewId.ToString();
int headReplicaAccountIndex = 0;
int tailReplicaAccountIndex = 1;
List <int> actualStorageAccountsToBeUsed = new List <int>()
{
tailReplicaAccountIndex
};
bool convertXStoreTableMode = false;
Console.WriteLine("Setting up RTable that only has a Tail Replica...");
this.SetupRTableEnv(
tableName,
useHttps,
viewIdString,
actualStorageAccountsToBeUsed,
convertXStoreTableMode,
1);
Assert.AreEqual(1, this.actualStorageAccountsUsed.Count, "Only one stoarge account should be used at this point.");
Console.WriteLine("Inserting entities to the Tail Replica...");
this.numberOfPartitions = 1;
this.numberOfRowsPerPartition = 1;
string jobTypeReplace = "jobType-AddNewHeadReplicaTest-Replace"; // will call Replace in Phase One and Two
string jobIdReplace = "jobId-AddNewHeadReplicaTest-Replace";
string jobTypeDelete = "jobType-AddNewHeadReplicaTest-Delete"; // will call Delete in Phase One, then try to Insert again in Phase Two
string jobIdDelete = "jobId-AddNewHeadReplicaTest-Delete";
string jobTypeStatic = "jobType-AddNewHeadReplicaTest-Static"; // will NOT call Replace in Phase One. Let RepairTable() fix this row. Call Replace in Phase Two.
string jobIdStatic = "jobId-AddNewHeadReplicaTest-Static";
string originalMessage = "message";
string updatedMessage = "updatedMessage";
string updatedMessage2 = "updatedMessageTwo";
string staticMessage = "staticMessage";
// Insert some entries.
this.PerformInsertOperationAndValidate(jobTypeReplace,
jobIdReplace,
originalMessage);
this.PerformInsertOperationAndValidate(jobTypeDelete,
jobIdDelete,
originalMessage);
this.PerformInsertOperationAndValidate(jobTypeStatic,
jobIdStatic,
staticMessage);
Console.WriteLine("\n\nAdding new a new Head Replica to the chain. Will sleep some time...");
int readViewHeadIndex = 1;
this.actualStorageAccountsUsed = new List <int> {
headReplicaAccountIndex, tailReplicaAccountIndex
};
viewId++;
this.RefreshRTableEnvJsonConfigBlob(viewId, convertXStoreTableMode, readViewHeadIndex);
this.repTable.CreateIfNotExists(); // since we are adding a new Replica, need to create RTable (if needed).
//
// Phase One:
// Using Head Replica and Tail Replica. readViewHeadIndex = 1
//
Console.WriteLine("\n\nPhase One: New Head Replica has been added. readViewHeadIndex = 1. Verifying Replace(), Delete() and Insert() API...");
for (int i = 0; i < retrieveAttemtps; i++)
{
this.PerformRetrieveOperationAndValidate(jobTypeReplace, jobIdReplace, originalMessage, false);
}
for (int i = 0; i < retrieveAttemtps; i++)
{
this.PerformRetrieveOperationAndValidate(jobTypeDelete, jobIdDelete, originalMessage, false);
}
// Replace() API
this.PerformOperationAndValidate(TableOperationType.Replace, jobTypeReplace, jobIdReplace, updatedMessage);
// Delete() API:
this.PerformOperationAndValidate(TableOperationType.Delete, jobTypeDelete, jobIdDelete);
string jobTypeInsert = "jobType-AddNewHeadReplicaTest-Insert"; // Will call Insert in Phase One, and Replace in Phase Two
string jobIdInsert = "jobId-AddNewHeadReplicaTest-Insert";
// Insert() API:
this.PerformInsertOperationAndValidate(jobTypeInsert,
jobIdInsert,
originalMessage);
for (int i = 0; i < retrieveAttemtps; i++)
{
this.PerformRetrieveOperationAndValidate(jobTypeInsert, jobIdInsert, originalMessage, false);
}
Console.WriteLine("\n\nRepairing RTable...");
ReconfigurationStatus repairStatus = this.repTable.RepairTable(1, null);
Assert.AreEqual(ReconfigurationStatus.SUCCESS, repairStatus, "Repair failed.");
Console.WriteLine("\n\nDone repairing. Changing RTable config to use ReadViewHeadIndex=0, will sleep for some time...");
readViewHeadIndex = 0;
viewId++;
this.RefreshRTableEnvJsonConfigBlob(viewId, convertXStoreTableMode, readViewHeadIndex);
//
// Phase Two:
// Using Head Replica and Tail Replica. readViewHeadIndex = 0
//
Console.WriteLine("\n\nPhase Two: RepairTable() completed successfully. readViewHeadIndex = 0. Verifying Replace(), Delete() and Insert() API...");
Console.WriteLine("Checking entries after Table is repaired...");
for (int i = 0; i < retrieveAttemtps; i++)
{
this.PerformRetrieveOperationAndValidate(jobTypeReplace, jobIdReplace, updatedMessage, false);
}
// Replace() API:
this.PerformOperationAndValidate(TableOperationType.Replace, jobTypeReplace, jobIdReplace, updatedMessage2);
for (int i = 0; i < retrieveAttemtps; i++)
{
this.PerformRetrieveOperationAndValidate(jobTypeStatic, jobIdStatic, staticMessage, false);
}
// Replace() API:
this.PerformOperationAndValidate(TableOperationType.Replace, jobTypeStatic, jobIdStatic, updatedMessage);
for (int i = 0; i < retrieveAttemtps; i++)
{
this.PerformRetrieveOperationAndValidate(jobTypeInsert, jobIdInsert, originalMessage, false);
}
// Replace() API:
this.PerformOperationAndValidate(TableOperationType.Replace, jobTypeInsert, jobIdInsert, updatedMessage);
// Verify NotFound
Console.WriteLine("Verifying jobIdDelete does not exist after RepairTable...");
for (int i = 0; i < retrieveAttemtps; i++)
{
this.ValidateRetrieveNotFound(jobTypeDelete, jobIdDelete);
}
// Insert() API:
Console.WriteLine("Should be able to Insert() jobIdDelete again after RepairTable...");
this.PerformInsertOperationAndValidate(jobTypeDelete,
jobIdDelete,
updatedMessage);
Console.WriteLine("DONE. Test passed.");
}
finally
{
base.DeleteAllRtableResources();
}
}
public void RepairTableWontRepairRowsWithHigherViewIdWhenIgnoreHigherViewIdRowsFlagIsSet()
{
TableOperation operation;
TableResult result;
CustomerEntity customer;
// - View has 2 replicas ?
View view = this.configurationWrapper.GetWriteView();
Assert.IsTrue(view.Chain.Count == 2, "expects 2 replicas only!");
// - Remove the Head, [None]->[RW]
RemoveHeadFromView(view.Name, 600, this.configurationService);
// 1 - Insert entries in old viewId
var rtable = new ReplicatedTable(this.repTable.TableName, this.configurationService);
string firstName = "FirstName";
string lastName = "LastName";
for (int i = 0; i < 10; i++)
{
customer = new CustomerEntity(firstName + i, lastName + i);
operation = TableOperation.Insert(customer);
rtable.Execute(operation);
}
// 2 - Increase viewId => so we can create rows with higher viewId
// - Update entry #5 and #8 in new viewId
ReplicatedTableConfiguration config;
ReplicatedTableQuorumReadResult readStatus = this.configurationService.RetrieveConfiguration(out config);
Assert.IsTrue(readStatus.Code == ReplicatedTableQuorumReadCode.Success);
ReplicatedTableConfigurationStore viewConfg = config.GetView(view.Name);
viewConfg.ViewId += 100;
this.configurationService.UpdateConfiguration(config);
foreach (int entryId in new int[] { 5, 8 })
{
operation = TableOperation.Retrieve <CustomerEntity>(firstName + entryId, lastName + entryId);
result = rtable.Execute(operation);
Assert.IsTrue(result != null && result.HttpStatusCode == (int)HttpStatusCode.OK && (CustomerEntity)result.Result != null, "Retrieve customer failed");
customer = (CustomerEntity)result.Result;
customer.Email = "*****@*****.**";
operation = TableOperation.Replace(customer);
result = rtable.Execute(operation);
Assert.IsTrue(result != null && result.HttpStatusCode == (int)HttpStatusCode.NoContent, "Update customer failed");
}
// 3 - Restore previous viewId, and,
// - Set 'IgnoreHigherViewIdRows' flag so we ignore rows with higher viewIds
readStatus = this.configurationService.RetrieveConfiguration(out config);
Assert.IsTrue(readStatus.Code == ReplicatedTableQuorumReadCode.Success);
viewConfg = config.GetView(view.Name);
viewConfg.ViewId -= 100;
config.SetIgnoreHigherViewIdRowsFlag(true);
this.configurationService.UpdateConfiguration(config);
try
{
// Check Retrieve of row #5 and #8 returns NotFound
foreach (int entryId in new int[] { 5, 8 })
{
operation = TableOperation.Retrieve <CustomerEntity>(firstName + entryId, lastName + entryId);
var retrievedResult = rtable.Execute(operation);
Assert.AreNotEqual(null, retrievedResult, "retrievedResult = null");
Assert.AreEqual((int)HttpStatusCode.NotFound, retrievedResult.HttpStatusCode, "retrievedResult.HttpStatusCode mismatch");
}
}
catch (ReplicatedTableStaleViewException)
{
Assert.Fail("Retrieve() is expected to NotFound the row, but got RTableStaleViewException !");
}
// 4 - Now insert a Head [WO]->[RW]
// - Then, call RepairTable ...
InsertHeadInView(view.Name, 600, this.configurationService);
ReconfigurationStatus status = rtable.RepairTable(0, null);
Assert.AreEqual(status, ReconfigurationStatus.PARTIAL_FAILURE, "rows with higher viewId should not be repaired");
// 5 - Check rows with higher viewId still NotFound, even after repair ...
try
{
// Check Retrieve of row #5 and #8 returns NotFound
foreach (int entryId in new int[] { 5, 8 })
{
operation = TableOperation.Retrieve <CustomerEntity>(firstName + entryId, lastName + entryId);
var retrievedResult = rtable.Execute(operation);
Assert.AreNotEqual(null, retrievedResult, "retrievedResult = null");
Assert.AreEqual((int)HttpStatusCode.NotFound, retrievedResult.HttpStatusCode, "retrievedResult.HttpStatusCode mismatch");
}
}
catch (ReplicatedTableStaleViewException)
{
Assert.Fail("Retrieve() is expected to NotFound the row, but got RTableStaleViewException !");
}
}
