public void RTableRepairTable() { // Insert entity Assert.IsTrue(this.repTable.Exists(), "RTable does not exist"); View fullView = configurationWrapper.GetWriteView(); List <ReplicaInfo> fullViewReplicas = new List <ReplicaInfo>(); for (int i = 0; i <= fullView.TailIndex; i++) { fullViewReplicas.Add(fullView.GetReplicaInfo(i)); } List <ReplicaInfo> newReplicas = new List <ReplicaInfo>(); for (int i = 1; i <= fullView.TailIndex; i++) { newReplicas.Add(fullView.GetReplicaInfo(i)); } SampleRTableEntity customer1 = new SampleRTableEntity("firstName1", "lastName1", "*****@*****.**"); TableOperation operation = TableOperation.Insert(customer1); TableResult result = repTable.Execute(operation); Assert.AreNotEqual(null, result, "result = null"); Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch"); SampleRTableEntity customer2 = new SampleRTableEntity("firstName2", "lastName2", "*****@*****.**"); operation = TableOperation.Insert(customer2); result = repTable.Execute(operation); Assert.AreNotEqual(null, result, "result = null"); Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch"); customer2 = (SampleRTableEntity)result.Result; SampleRTableEntity customer3 = new SampleRTableEntity("firstName3", "lastName3", "*****@*****.**"); operation = TableOperation.Insert(customer3); result = repTable.Execute(operation); Assert.AreNotEqual(null, result, "result = null"); Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch"); ReadFromIndividualAccountsDirectly(customer1.PartitionKey, customer1.RowKey, true); ReadFromIndividualAccountsDirectly(customer2.PartitionKey, customer2.RowKey, true); ReadFromIndividualAccountsDirectly(customer3.PartitionKey, customer3.RowKey, true); // remove replica from the head this.UpdateConfiguration(newReplicas, 0); Assert.IsTrue(configurationWrapper.IsViewStable()); // delete a row TableOperation deleteOperation = TableOperation.Delete(customer1); result = repTable.Execute(deleteOperation); Assert.AreNotEqual(null, result, "result = null"); Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch"); // add a row SampleRTableEntity customer4 = new SampleRTableEntity("firstName4", "lastName4", "*****@*****.**"); operation = TableOperation.Insert(customer4); result = repTable.Execute(operation); Assert.AreNotEqual(null, result, "result = null"); Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch"); // replace a row customer2.Message = "updated after view update"; operation = TableOperation.Replace(customer2); result = repTable.Execute(operation); Assert.AreNotEqual(null, result, "result = null"); Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch"); //Add replica at head this.UpdateConfiguration(fullViewReplicas, 1); // repair table on the new head Console.WriteLine("Calling repair table"); ReconfigurationStatus status = repTable.RepairTable(0, null); Assert.AreEqual(ReconfigurationStatus.SUCCESS, status, "RepairTable status is not success: {0}", status); ReadFromIndividualAccountsDirectly(customer1.PartitionKey, customer1.RowKey, true); ReadFromIndividualAccountsDirectly(customer2.PartitionKey, customer2.RowKey, true); ReadFromIndividualAccountsDirectly(customer3.PartitionKey, customer3.RowKey, true); ReadFromIndividualAccountsDirectly(customer4.PartitionKey, customer4.RowKey, true); }
public void RemoveAndReAddHeadReplicaTest() { try { int retrieveAttemtps = 2; this.LoadTestConfiguration(); string tableName = this.GenerateRandomTableName(); bool useHttps = true; long viewId = 1; string viewIdString = viewId.ToString(); int headReplicaAccountIndex = 0; int tailReplicaAccountIndex = 1; List <int> actualStorageAccountsToBeUsed = new List <int>() { headReplicaAccountIndex, tailReplicaAccountIndex }; bool convertXStoreTableMode = false; Console.WriteLine("Setting up RTable that has a Head Replica and a Tail Replica..."); this.SetupRTableEnv( tableName, useHttps, viewIdString, actualStorageAccountsToBeUsed, convertXStoreTableMode, 1); Assert.AreEqual(2, this.actualStorageAccountsUsed.Count, "Two stoarge accounts should be used at this point."); Console.WriteLine("Inserting entities to the RTable..."); this.numberOfPartitions = 1; this.numberOfRowsPerPartition = 1; string jobTypeInsert = "jobType-RemoveAndReAddHeadReplicaTest-Insert"; // will call Insert() in Phase One, Two (expect to fail), Three (expect to fail) string jobIdInsert = "jobId-RemoveAndReAddHeadReplicaTest-Insert"; string jobTypeStale = "jobType-RemoveAndReAddHeadReplicaTest-Stale"; // will call Replace() in Phase One, Two, Three string jobIdStale = "jobId-RemoveAndReAddHeadReplicaTest-Stale"; string jobTypeStatic = "jobType-RemoveAndReAddHeadReplicaTest-Static"; // will NOT call Replace() in Phase One. Let RepairTable() fix this row. string jobIdStatic = "jobId-RemoveAndReAddHeadReplicaTest-Static"; string jobTypeDelete = "jobType-RemoveAndReAddHeadReplicaTest-Delete"; // will call Delete() in Phase One, Insert() in Phase Two. string jobIdDelete = "jobId-RemoveAndReAddHeadReplicaTest-Delete"; string jobTypeReplaceDelete = "jobType-RemoveAndReAddHeadReplicaTest-ReplaceDelete"; // will call Replace() in Phase One, Delete() in Phase Two, Insert() in Phase Three. string jobIdReplaceDelete = "jobId-RemoveAndReAddHeadReplicaTest-ReplaceDelete"; string originalMessage = "message"; string updatedMessage = "updatedMessage"; string updatedMessage2 = "updatedMessageTwo"; string staticMessage = "staticMessage"; // Insert some entries. this.PerformInsertOperationAndValidate(jobTypeInsert, jobIdInsert, originalMessage); this.PerformInsertOperationAndValidate(jobTypeStale, jobIdStale, originalMessage); this.PerformInsertOperationAndValidate(jobTypeStatic, jobIdStatic, staticMessage); this.PerformInsertOperationAndValidate(jobTypeDelete, jobIdDelete, originalMessage); this.PerformInsertOperationAndValidate(jobTypeReplaceDelete, jobIdReplaceDelete, originalMessage); Console.WriteLine("\n\nRemoving Head Replica from the chain. Will sleep some time..."); int readViewHeadIndex = 0; this.actualStorageAccountsUsed = new List <int> { tailReplicaAccountIndex }; viewId++; this.RefreshRTableEnvJsonConfigBlob(viewId, convertXStoreTableMode, readViewHeadIndex); // // Phase One: // Using Tail Replica only. readViewHeadIndex = 0. // Console.WriteLine("\n\nPhase One: Head Replica has been removed. Using Tail Replica only. Verifying Replace() and Delete() API..."); for (int i = 0; i < retrieveAttemtps; i++) { this.PerformRetrieveOperationAndValidate(jobTypeStale, jobIdStale, originalMessage, false); } for (int i = 0; i < retrieveAttemtps; i++) { this.PerformRetrieveOperationAndValidate(jobTypeStatic, jobIdStatic, staticMessage, false); } for (int i = 0; i < retrieveAttemtps; i++) { this.PerformRetrieveOperationAndValidate(jobTypeDelete, jobIdDelete, originalMessage, false); } for (int i = 0; i < retrieveAttemtps; i++) { this.PerformRetrieveOperationAndValidate(jobTypeReplaceDelete, jobIdReplaceDelete, originalMessage, false); } // Replace() API this.PerformOperationAndValidate(TableOperationType.Replace, jobTypeStale, jobIdStale, updatedMessage); this.PerformOperationAndValidate(TableOperationType.Replace, jobTypeReplaceDelete, jobIdReplaceDelete, updatedMessage); // Delete() API this.PerformOperationAndValidate(TableOperationType.Delete, jobTypeDelete, jobIdDelete); Console.WriteLine("\n\nAdding the Head Replica back to the chain. readViewHeadIndex = 1. Will sleep some time..."); readViewHeadIndex = 1; this.actualStorageAccountsUsed = new List <int> { headReplicaAccountIndex, tailReplicaAccountIndex }; viewId++; this.RefreshRTableEnvJsonConfigBlob(viewId, convertXStoreTableMode, readViewHeadIndex); // // Phase Two: // Using Head Replica and Tail Replica. readViewHeadIndex = 1. // Just added back the Head Replica, which contains stale entries. // Console.WriteLine("\n\nPhase Two: Head Replica has been added back. readViewHeadIndex = 1. Verifying Replace(), Delete() and Insert() API..."); // Insert() existing row will fail Console.WriteLine("Verifying that Insert() an existing row will fail..."); this.PerformInsertOperationAndExpectToFail(jobTypeInsert, jobIdInsert, updatedMessage); for (int i = 0; i < retrieveAttemtps; i++) { this.PerformRetrieveOperationAndValidate(jobTypeStale, jobIdStale, updatedMessage, false); } for (int i = 0; i < retrieveAttemtps; i++) { this.PerformRetrieveOperationAndValidate(jobTypeStatic, jobIdStatic, staticMessage, false); } for (int i = 0; i < retrieveAttemtps; i++) { this.PerformRetrieveOperationAndValidate(jobTypeReplaceDelete, jobIdReplaceDelete, updatedMessage, false); } // Replace(): this.PerformOperationAndValidate(TableOperationType.Replace, jobTypeStale, jobIdStale, updatedMessage2); // Verify NotFound: Console.WriteLine("Verifying jobIdDelete does not exist in Phase Two (Head and Tail Replica; readViewHeadIndex = 1)..."); for (int i = 0; i < retrieveAttemtps; i++) { this.ValidateRetrieveNotFound(jobTypeDelete, jobIdDelete); } // Insert() Console.WriteLine("Should be able to Insert() jobIdDelete in Phase Two (Head and Tail Replica; readViewHeadIndex = 1)..."); this.PerformInsertOperationAndValidate(jobTypeDelete, jobIdDelete, updatedMessage); // Delete(): Console.WriteLine("Verifying Delete() jobTypeReplaceDelete in Phase Two (Head and Tail Replica; readViewHeadIndex = 1)..."); this.PerformOperationAndValidate(TableOperationType.Delete, jobTypeReplaceDelete, jobIdReplaceDelete); Console.WriteLine("\n\nRepairing RTable..."); ReconfigurationStatus repairStatus = this.repTable.RepairTable(1, null); Assert.AreEqual(ReconfigurationStatus.SUCCESS, repairStatus, "Repair failed."); Console.WriteLine("\n\nDone repairing. Changing RTable config to use ReadViewHeadIndex=0, will sleep for some time..."); readViewHeadIndex = 0; viewId++; this.RefreshRTableEnvJsonConfigBlob(viewId, convertXStoreTableMode, readViewHeadIndex); // // Phase Three: // Using Head Replica and Tail Replica. readViewHeadIndex = 0 // Console.WriteLine("\n\nPhase Three: RepairTable() completed successfully. readViewHeadIndex = 0. Verifying Replace(), Delete() and Insert() API..."); // Insert() existing row will fail Console.WriteLine("Verifying that Insert() an existing row will fail..."); this.PerformInsertOperationAndExpectToFail(jobTypeInsert, jobIdInsert, updatedMessage); for (int i = 0; i < retrieveAttemtps; i++) { this.PerformRetrieveOperationAndValidate(jobTypeStale, jobIdStale, updatedMessage2, false); } for (int i = 0; i < retrieveAttemtps; i++) { this.PerformRetrieveOperationAndValidate(jobTypeStatic, jobIdStatic, staticMessage, false); } for (int i = 0; i < retrieveAttemtps; i++) { this.PerformRetrieveOperationAndValidate(jobTypeDelete, jobIdDelete, updatedMessage, false); } // Verify NotFound: Console.WriteLine("Verifying jobTypeReplaceDelete does not exist in Phase Three (Head and Tail Replica; readViewHeadIndex = 0)..."); for (int i = 0; i < retrieveAttemtps; i++) { this.ValidateRetrieveNotFound(jobTypeReplaceDelete, jobIdReplaceDelete); } // Insert() Console.WriteLine("Should be able to Insert() jobTypeReplaceDelete in Phase Three (Head and Tail Replica; readViewHeadIndex = 0)..."); this.PerformInsertOperationAndValidate(jobTypeReplaceDelete, jobIdReplaceDelete, updatedMessage2); Console.WriteLine("Checking Replace() API..."); this.PerformOperationAndValidate(TableOperationType.Replace, jobTypeStale, jobIdStale, updatedMessage); this.PerformOperationAndValidate(TableOperationType.Replace, jobTypeStatic, jobIdStatic, updatedMessage); this.PerformOperationAndValidate(TableOperationType.Replace, jobTypeDelete, jobIdDelete, updatedMessage2); Console.WriteLine("DONE. Test passed."); } finally { base.DeleteAllRtableResources(); } }
public void AddNewHeadReplicaTest() { try { int retrieveAttemtps = 2; this.LoadTestConfiguration(); string tableName = this.GenerateRandomTableName(); bool useHttps = true; long viewId = 1; string viewIdString = viewId.ToString(); int headReplicaAccountIndex = 0; int tailReplicaAccountIndex = 1; List <int> actualStorageAccountsToBeUsed = new List <int>() { tailReplicaAccountIndex }; bool convertXStoreTableMode = false; Console.WriteLine("Setting up RTable that only has a Tail Replica..."); this.SetupRTableEnv( tableName, useHttps, viewIdString, actualStorageAccountsToBeUsed, convertXStoreTableMode, 1); Assert.AreEqual(1, this.actualStorageAccountsUsed.Count, "Only one stoarge account should be used at this point."); Console.WriteLine("Inserting entities to the Tail Replica..."); this.numberOfPartitions = 1; this.numberOfRowsPerPartition = 1; string jobTypeReplace = "jobType-AddNewHeadReplicaTest-Replace"; // will call Replace in Phase One and Two string jobIdReplace = "jobId-AddNewHeadReplicaTest-Replace"; string jobTypeDelete = "jobType-AddNewHeadReplicaTest-Delete"; // will call Delete in Phase One, then try to Insert again in Phase Two string jobIdDelete = "jobId-AddNewHeadReplicaTest-Delete"; string jobTypeStatic = "jobType-AddNewHeadReplicaTest-Static"; // will NOT call Replace in Phase One. Let RepairTable() fix this row. Call Replace in Phase Two. string jobIdStatic = "jobId-AddNewHeadReplicaTest-Static"; string originalMessage = "message"; string updatedMessage = "updatedMessage"; string updatedMessage2 = "updatedMessageTwo"; string staticMessage = "staticMessage"; // Insert some entries. this.PerformInsertOperationAndValidate(jobTypeReplace, jobIdReplace, originalMessage); this.PerformInsertOperationAndValidate(jobTypeDelete, jobIdDelete, originalMessage); this.PerformInsertOperationAndValidate(jobTypeStatic, jobIdStatic, staticMessage); Console.WriteLine("\n\nAdding new a new Head Replica to the chain. Will sleep some time..."); int readViewHeadIndex = 1; this.actualStorageAccountsUsed = new List <int> { headReplicaAccountIndex, tailReplicaAccountIndex }; viewId++; this.RefreshRTableEnvJsonConfigBlob(viewId, convertXStoreTableMode, readViewHeadIndex); this.repTable.CreateIfNotExists(); // since we are adding a new Replica, need to create RTable (if needed). // // Phase One: // Using Head Replica and Tail Replica. readViewHeadIndex = 1 // Console.WriteLine("\n\nPhase One: New Head Replica has been added. readViewHeadIndex = 1. Verifying Replace(), Delete() and Insert() API..."); for (int i = 0; i < retrieveAttemtps; i++) { this.PerformRetrieveOperationAndValidate(jobTypeReplace, jobIdReplace, originalMessage, false); } for (int i = 0; i < retrieveAttemtps; i++) { this.PerformRetrieveOperationAndValidate(jobTypeDelete, jobIdDelete, originalMessage, false); } // Replace() API this.PerformOperationAndValidate(TableOperationType.Replace, jobTypeReplace, jobIdReplace, updatedMessage); // Delete() API: this.PerformOperationAndValidate(TableOperationType.Delete, jobTypeDelete, jobIdDelete); string jobTypeInsert = "jobType-AddNewHeadReplicaTest-Insert"; // Will call Insert in Phase One, and Replace in Phase Two string jobIdInsert = "jobId-AddNewHeadReplicaTest-Insert"; // Insert() API: this.PerformInsertOperationAndValidate(jobTypeInsert, jobIdInsert, originalMessage); for (int i = 0; i < retrieveAttemtps; i++) { this.PerformRetrieveOperationAndValidate(jobTypeInsert, jobIdInsert, originalMessage, false); } Console.WriteLine("\n\nRepairing RTable..."); ReconfigurationStatus repairStatus = this.repTable.RepairTable(1, null); Assert.AreEqual(ReconfigurationStatus.SUCCESS, repairStatus, "Repair failed."); Console.WriteLine("\n\nDone repairing. Changing RTable config to use ReadViewHeadIndex=0, will sleep for some time..."); readViewHeadIndex = 0; viewId++; this.RefreshRTableEnvJsonConfigBlob(viewId, convertXStoreTableMode, readViewHeadIndex); // // Phase Two: // Using Head Replica and Tail Replica. readViewHeadIndex = 0 // Console.WriteLine("\n\nPhase Two: RepairTable() completed successfully. readViewHeadIndex = 0. Verifying Replace(), Delete() and Insert() API..."); Console.WriteLine("Checking entries after Table is repaired..."); for (int i = 0; i < retrieveAttemtps; i++) { this.PerformRetrieveOperationAndValidate(jobTypeReplace, jobIdReplace, updatedMessage, false); } // Replace() API: this.PerformOperationAndValidate(TableOperationType.Replace, jobTypeReplace, jobIdReplace, updatedMessage2); for (int i = 0; i < retrieveAttemtps; i++) { this.PerformRetrieveOperationAndValidate(jobTypeStatic, jobIdStatic, staticMessage, false); } // Replace() API: this.PerformOperationAndValidate(TableOperationType.Replace, jobTypeStatic, jobIdStatic, updatedMessage); for (int i = 0; i < retrieveAttemtps; i++) { this.PerformRetrieveOperationAndValidate(jobTypeInsert, jobIdInsert, originalMessage, false); } // Replace() API: this.PerformOperationAndValidate(TableOperationType.Replace, jobTypeInsert, jobIdInsert, updatedMessage); // Verify NotFound Console.WriteLine("Verifying jobIdDelete does not exist after RepairTable..."); for (int i = 0; i < retrieveAttemtps; i++) { this.ValidateRetrieveNotFound(jobTypeDelete, jobIdDelete); } // Insert() API: Console.WriteLine("Should be able to Insert() jobIdDelete again after RepairTable..."); this.PerformInsertOperationAndValidate(jobTypeDelete, jobIdDelete, updatedMessage); Console.WriteLine("DONE. Test passed."); } finally { base.DeleteAllRtableResources(); } }
public void RepairTableWontRepairRowsWithHigherViewIdWhenIgnoreHigherViewIdRowsFlagIsSet() { TableOperation operation; TableResult result; CustomerEntity customer; // - View has 2 replicas ? View view = this.configurationWrapper.GetWriteView(); Assert.IsTrue(view.Chain.Count == 2, "expects 2 replicas only!"); // - Remove the Head, [None]->[RW] RemoveHeadFromView(view.Name, 600, this.configurationService); // 1 - Insert entries in old viewId var rtable = new ReplicatedTable(this.repTable.TableName, this.configurationService); string firstName = "FirstName"; string lastName = "LastName"; for (int i = 0; i < 10; i++) { customer = new CustomerEntity(firstName + i, lastName + i); operation = TableOperation.Insert(customer); rtable.Execute(operation); } // 2 - Increase viewId => so we can create rows with higher viewId // - Update entry #5 and #8 in new viewId ReplicatedTableConfiguration config; ReplicatedTableQuorumReadResult readStatus = this.configurationService.RetrieveConfiguration(out config); Assert.IsTrue(readStatus.Code == ReplicatedTableQuorumReadCode.Success); ReplicatedTableConfigurationStore viewConfg = config.GetView(view.Name); viewConfg.ViewId += 100; this.configurationService.UpdateConfiguration(config); foreach (int entryId in new int[] { 5, 8 }) { operation = TableOperation.Retrieve <CustomerEntity>(firstName + entryId, lastName + entryId); result = rtable.Execute(operation); Assert.IsTrue(result != null && result.HttpStatusCode == (int)HttpStatusCode.OK && (CustomerEntity)result.Result != null, "Retrieve customer failed"); customer = (CustomerEntity)result.Result; customer.Email = "*****@*****.**"; operation = TableOperation.Replace(customer); result = rtable.Execute(operation); Assert.IsTrue(result != null && result.HttpStatusCode == (int)HttpStatusCode.NoContent, "Update customer failed"); } // 3 - Restore previous viewId, and, // - Set 'IgnoreHigherViewIdRows' flag so we ignore rows with higher viewIds readStatus = this.configurationService.RetrieveConfiguration(out config); Assert.IsTrue(readStatus.Code == ReplicatedTableQuorumReadCode.Success); viewConfg = config.GetView(view.Name); viewConfg.ViewId -= 100; config.SetIgnoreHigherViewIdRowsFlag(true); this.configurationService.UpdateConfiguration(config); try { // Check Retrieve of row #5 and #8 returns NotFound foreach (int entryId in new int[] { 5, 8 }) { operation = TableOperation.Retrieve <CustomerEntity>(firstName + entryId, lastName + entryId); var retrievedResult = rtable.Execute(operation); Assert.AreNotEqual(null, retrievedResult, "retrievedResult = null"); Assert.AreEqual((int)HttpStatusCode.NotFound, retrievedResult.HttpStatusCode, "retrievedResult.HttpStatusCode mismatch"); } } catch (ReplicatedTableStaleViewException) { Assert.Fail("Retrieve() is expected to NotFound the row, but got RTableStaleViewException !"); } // 4 - Now insert a Head [WO]->[RW] // - Then, call RepairTable ... InsertHeadInView(view.Name, 600, this.configurationService); ReconfigurationStatus status = rtable.RepairTable(0, null); Assert.AreEqual(status, ReconfigurationStatus.PARTIAL_FAILURE, "rows with higher viewId should not be repaired"); // 5 - Check rows with higher viewId still NotFound, even after repair ... try { // Check Retrieve of row #5 and #8 returns NotFound foreach (int entryId in new int[] { 5, 8 }) { operation = TableOperation.Retrieve <CustomerEntity>(firstName + entryId, lastName + entryId); var retrievedResult = rtable.Execute(operation); Assert.AreNotEqual(null, retrievedResult, "retrievedResult = null"); Assert.AreEqual((int)HttpStatusCode.NotFound, retrievedResult.HttpStatusCode, "retrievedResult.HttpStatusCode mismatch"); } } catch (ReplicatedTableStaleViewException) { Assert.Fail("Retrieve() is expected to NotFound the row, but got RTableStaleViewException !"); } }