/// <summary> /// Helper function to validate whether "retrievedEntity" is correct or not. /// </summary> /// <param name="retrievedEntity"></param> /// <param name="jobType"></param> /// <param name="jobId"></param> /// <param name="message"></param> /// <param name="partition"></param> /// <param name="row"></param> /// <param name="checkViewId"></param> protected void ValidateRetrievedRTableEntity( SampleRTableEntity retrievedEntity, string jobType, string jobId, string message, int partition, int row, bool checkViewId = true) { Assert.AreEqual( this.GenerateJobType(jobType, partition), retrievedEntity.JobType, "JobType of retrievedEntity mismatch"); Assert.AreEqual( this.GenerateJobId(jobId, partition, row), retrievedEntity.JobId, "JobId of retrievedEntity mismatch"); Assert.AreEqual( this.GenerateMessage(message, partition, row), retrievedEntity.Message, "Message of retrievedEntity mismatch"); if (checkViewId) { Assert.AreEqual( this.configurationWrapper.GetReadView().ViewId, retrievedEntity._rtable_ViewId, "_rtable_ViewId of retrievedEntity mismatch"); } }
public void RTableRepairRow() { // Insert entity Assert.IsTrue(this.repTable.Exists(), "RTable does not exist"); View fullView = configurationWrapper.GetWriteView(); List <ReplicaInfo> fullViewReplicas = new List <ReplicaInfo>(); for (int i = 0; i <= fullView.TailIndex; i++) { fullViewReplicas.Add(fullView.GetReplicaInfo(i)); } List <ReplicaInfo> newReplicas = new List <ReplicaInfo>(); for (int i = 1; i <= fullView.TailIndex; i++) { newReplicas.Add(fullView.GetReplicaInfo(i)); } this.UpdateConfiguration(newReplicas, 0); Assert.IsTrue(configurationWrapper.IsViewStable()); SampleRTableEntity newCustomer = new SampleRTableEntity("firstname1", "lastname1", "*****@*****.**"); TableOperation operation = TableOperation.Insert(newCustomer); TableResult result = repTable.Execute(operation); Assert.AreNotEqual(null, result, "result = null"); ReplicatedTableEntity row = (ReplicatedTableEntity)result.Result; Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch"); Assert.AreNotEqual(null, result.Result, "result.Result = null"); Assert.AreEqual("1", result.Etag, "result.Etag mismatch"); Assert.AreEqual(false, row._rtable_RowLock, "row._rtable_RowLock mismatch"); Assert.AreEqual(1, row._rtable_Version, "row._rtable_Version mismatch"); Assert.AreEqual(false, row._rtable_Tombstone, "row._rtable_Tombstone mismatch"); Assert.AreEqual(configurationWrapper.GetWriteView().ViewId, row._rtable_ViewId, "row._rtable_ViewId mismatch"); ReadFromIndividualAccountsDirectly(newCustomer.PartitionKey, newCustomer.RowKey, true); //Add replica at head this.UpdateConfiguration(fullViewReplicas, 1); // repair row on the new head Console.WriteLine("Calling TableOperation.Replace(newCustomer)..."); result = repTable.RepairRow(row.PartitionKey, row.RowKey, null); Assert.AreNotEqual(null, result, "result = null"); // Retrieve Entity Console.WriteLine("Calling TableOperation.Retrieve<SampleRtableEntity>(firstName, lastName)..."); operation = TableOperation.Retrieve <SampleRTableEntity>("firstname1", "lastname1"); TableResult retrievedResult = repTable.Execute(operation); Assert.AreNotEqual(null, retrievedResult, "retrievedResult = null"); Assert.AreEqual((int)HttpStatusCode.OK, retrievedResult.HttpStatusCode, "retrievedResult.HttpStatusCode mismatch"); ReadFromIndividualAccountsDirectly(newCustomer.PartitionKey, newCustomer.RowKey, true); }
/// <summary> /// Helper function to retrieve the set of entities for the specified set of jobType and jobId. /// Validate that entity.Message matches "entityMessage") /// </summary> /// <param name="jobType"></param> /// <param name="jobId"></param> /// <param name="entityMessage"></param> /// <param name="checkViewId">True means validate _rtable_ViewId of the entity</param> protected void PerformRetrieveOperationAndValidate(string jobType, string jobId, string entityMessage, bool checkViewId = true) { Console.WriteLine("\nValidating Retrieve operation..."); for (int i = 0; i < this.numberOfPartitions; i++) { List <string> rowKeys = new List <string>(); // list of rowKey for the given paritionKey string partitionKey; string rowKey; SampleRTableEntity.GenerateKeys(this.GenerateJobType(jobType, i), "don't care", out partitionKey, out rowKey); // // GetAllRows() // Console.WriteLine("Calling GetAllRows() for partition {0}", i); IEnumerable <SampleRTableEntity> allRows = this.rtableWrapper.GetAllRows(partitionKey); int j = 0; // counting the number of rows per partition foreach (var retrievedEntity in allRows) { this.ValidateRetrievedRTableEntity(retrievedEntity, jobType, jobId, entityMessage, i, j, checkViewId); rowKeys.Add(retrievedEntity.RowKey); j++; } Assert.AreEqual(this.numberOfRowsPerPartition, j, "Partition {0} only has {1} rows. Expected {2} rows", i, j, this.numberOfRowsPerPartition); // // FindRow() // Console.WriteLine("Calling FindRow() for partitionKey={0}", partitionKey); for (j = 0; j < rowKeys.Count; j++) { SampleRTableEntity retrievedEntity = this.rtableWrapper.FindRow(partitionKey, rowKeys[j]); this.ValidateRetrievedRTableEntity(retrievedEntity, jobType, jobId, entityMessage, i, j, checkViewId); } } Console.WriteLine("Passed Retrieve validation.\n"); }
/// <summary> /// Helper function to validate the specified entries do not exist. /// </summary> /// <param name="jobType"></param> /// <param name="jobId"></param> protected void ValidateRetrieveNotFound(string jobType, string jobId) { Console.WriteLine("\nValidating Retrieve NotFound operation jobType={0} jobId={1}...", jobType, jobId); for (int i = 0; i < this.numberOfPartitions; i++) { List <string> rowKeys = new List <string>(); // list of rowKey for the given paritionKey string partitionKey; string rowKey; SampleRTableEntity.GenerateKeys(this.GenerateJobType(jobType, i), "don't care", out partitionKey, out rowKey); // // GetAllRows() // Console.WriteLine("Calling GetAllRows() for partition {0}", i); IEnumerable <SampleRTableEntity> allRows = this.rtableWrapper.GetAllRows(partitionKey); int count = 0; // counting the number of rows per partition List <SampleRTableEntity> allRetrievedEntities = new List <SampleRTableEntity>(); foreach (var retrievedEntity in allRows) { allRetrievedEntities.Add(retrievedEntity); count++; } Assert.AreEqual(0, count, "GetAllRows() should return 0 entries."); } Console.WriteLine("Passed Retrieve validation.\n"); }
public void RandomTableOperationTest() { const int partitionIndex = 0; // Insert some entries. //Key = RowNumber, Value = RowKey var currentEntities = new Dictionary <int, String>(); for (int i = 0; i < this.numberOfRowsPerPartition; i++) { currentEntities.Add(i, GenerateJobId(JobId, partitionIndex, i)); } //Key = RowNumber, Value = RowKey var deletedEntities = new Dictionary <int, String>(); var random = new Random(); for (int i = 0; i < NumberOfOperations; i++) { int operation = random.Next(0, Enum.GetNames(typeof(TableOperationType)).Count()); int randomRow = random.Next(0, NumberofEntities); if ((TableOperationType)operation == TableOperationType.Insert) { if (deletedEntities.Count == 0) { //Ignore insertion if there is nothing to be inserted continue; } randomRow = deletedEntities.First().Key; deletedEntities.Remove(randomRow); } if (deletedEntities.ContainsKey(randomRow)) { continue; } Console.WriteLine("Operation# {0}, {1} on row {2}", i, (TableOperationType)operation, randomRow); PerformIndividualOperationAndValidate((TableOperationType)operation, JobType, JobId, partitionIndex, randomRow, OriginalMessage); if ((TableOperationType)operation == TableOperationType.Delete) { deletedEntities.Add(randomRow, GenerateJobId(JobId, partitionIndex, randomRow)); } } string rowKey; string partitionKey; SampleRTableEntity.GenerateKeys(this.GenerateJobType(JobType, partitionIndex), "don't care", out partitionKey, out rowKey); //Validations Console.WriteLine("Performing replica validations"); PerformInvariantChecks(partitionKey, HeadReplicaAccountIndex, TailReplicaAccountIndex); Console.WriteLine("DONE. Test passed."); }
/// <summary> /// Helper function to insert the specified set of jobType and jobId and validate the results. /// Make sure there are no existing entries for the jobType and jobId before calling this function. /// </summary> /// <param name="jobType"></param> /// <param name="jobId"></param> /// <param name="entityMessage"></param> protected void PerformInsertOperationAndValidate(string jobType, string jobId, string entityMessage) { Console.WriteLine("\nValidating Insert operation..."); for (int i = 0; i < this.numberOfPartitions; i++) { List <string> rowKeys = new List <string>(); // list of rowKey for the given paritionKey string partitionKey; string rowKey; SampleRTableEntity.GenerateKeys(this.GenerateJobType(jobType, i), "don't care", out partitionKey, out rowKey); // // GetAllRows() to confirm nothing exists and then call InsertRow() // Console.WriteLine("Calling GetAllRows() for partition {0}, expecting 0 rows...", i); IEnumerable <SampleRTableEntity> allRows = this.rtableWrapper.GetAllRows(partitionKey); Assert.AreEqual(0, allRows.Count(), "Partition {0} should have 0 rows in order for InsertRow() to work.", i); for (int j = 0; j < this.numberOfRowsPerPartition; j++) { SampleRTableEntity sampleRtableEntity = new SampleRTableEntity( this.GenerateJobType(jobType, i), this.GenerateJobId(jobId, i, j), this.GenerateMessage(entityMessage, i, j)); int attempts = 1; bool passed = true; while (attempts < 3) { try { Console.WriteLine("attempts={0}. partitionKey={1} rowKey={2}. Calling InsertRow API...", attempts, partitionKey, sampleRtableEntity.RowKey); this.rtableWrapper.InsertRow(sampleRtableEntity); passed = true; break; // get out of while(attempts) if no RTableConflictException } catch (RTableConflictException) { passed = false; Console.WriteLine("Got RTableConflictException. attempts={0}", attempts); attempts++; System.Threading.Thread.Sleep(1000); } } Assert.IsTrue(passed, "Keep getting RTableConflictException when calling InsertRow API"); } // // FindRow() // Console.WriteLine("Calling FindRow() for partitionKey={0}", partitionKey); for (int j = 0; j < rowKeys.Count; j++) { SampleRTableEntity retrievedEntity = this.rtableWrapper.FindRow(partitionKey, rowKeys[j]); this.ValidateRetrievedRTableEntity(retrievedEntity, jobType, jobId, entityMessage, i, j); } } Console.WriteLine("Passed Insert validation.\n"); }
/// <summary> /// Check whether this instance matches the specified object or not /// </summary> /// <param name="obj"></param> /// <returns></returns> public override bool Equals(object obj) { SampleRTableEntity dst = obj as SampleRTableEntity; if (dst == null) { return(false); } else { return((this.JobType == dst.JobType && this.JobId == dst.JobId && this.Message == dst.Message) && base.Equals(obj)); } }
private bool HeadIsLocked(SampleRTableEntity entry) { CloudTable table = this.cloudTableClients[0].GetTableReference(this.repTable.TableName); TableOperation retrieveOperation = TableOperation.Retrieve <SampleRTableEntity>(entry.PartitionKey, entry.RowKey); TableResult retrieveResult = table.Execute(retrieveOperation); if (retrieveResult == null || retrieveResult.HttpStatusCode != (int)HttpStatusCode.OK || retrieveResult.Result == null) { return(false); } SampleRTableEntity head = (SampleRTableEntity)retrieveResult.Result; return(head._rtable_RowLock == true); }
/// <summary> /// Helper function to insert the specified set of jobType and jobId. It is expected that InsertRow() will fail. /// </summary> /// <param name="jobType"></param> /// <param name="jobId"></param> /// <param name="entityMessage"></param> protected void PerformInsertOperationAndExpectToFail(string jobType, string jobId, string entityMessage) { Console.WriteLine("\nValidating Insert operation will fail. jobType={0} jobId={1}...", jobType, jobId); for (int i = 0; i < this.numberOfPartitions; i++) { List <string> rowKeys = new List <string>(); // list of rowKey for the given paritionKey string partitionKey; string rowKey; SampleRTableEntity.GenerateKeys(this.GenerateJobType(jobType, i), "don't care", out partitionKey, out rowKey); for (int j = 0; j < this.numberOfRowsPerPartition; j++) { SampleRTableEntity sampleRtableEntity = new SampleRTableEntity( this.GenerateJobType(jobType, i), this.GenerateJobId(jobId, i, j), this.GenerateMessage(entityMessage, i, j)); int attempts = 1; bool failed = false; while (attempts < 3) { try { Console.WriteLine("attempts={0}. partitionKey={1} rowKey={2}. Calling InsertRow API...", attempts, partitionKey, sampleRtableEntity.RowKey); this.rtableWrapper.InsertRow(sampleRtableEntity); failed = false; break; // get out of while(attempts) if no RTableConflictException } catch (RTableConflictException) { failed = true; Console.WriteLine("Got RTableConflictException. attempts={0}", attempts); attempts++; System.Threading.Thread.Sleep(1000); } } Assert.IsTrue(failed, "InsertRow() actually passed when we expect it to fail!"); } } Console.WriteLine("Passed 'Insert will fail' validation.\n"); }
/// <summary> /// Read and print the entity of the specified jobType and jobId from individual storage accounts for debugging. /// </summary> /// <param name="jobType"></param> /// <param name="jobId"></param> /// <param name="checkIndividualAccounts">True means will check the entity in each storage account for consistency</param> protected void ReadFromIndividualAccountsDirectly(string partitionKey, string rowKey, bool checkIndividualAccounts = false) { Console.WriteLine("\nBEGIN ReadFromIndividualAccountsDirectly()..."); partitionKey = partitionKey.ToLower().Replace(" ", ""); rowKey = rowKey.ToLower().Replace(" ", ""); string filter1 = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, partitionKey); string filter2 = TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.Equal, rowKey); TableQuery <SampleRTableEntity> query = new TableQuery <SampleRTableEntity>(). Where(TableQuery.CombineFilters(filter1, TableOperators.And, filter2)); SampleRTableEntity[] entities = new SampleRTableEntity[configurationWrapper.GetWriteView().Chain.Count]; for (int i = 0; i < configurationWrapper.GetWriteView().Chain.Count; i++) { Console.WriteLine("Executing query for CloudTable #{0}", i); foreach (var item in ((CloudTableClient)configurationWrapper.GetWriteView()[i]).GetTableReference(repTable.TableName).ExecuteQuery(query)) { Console.WriteLine("{0}", item.ToString()); entities[i] = item; } } if (checkIndividualAccounts) { Console.WriteLine("Checking for consistency..."); for (int i = 1; i < configurationWrapper.GetWriteView().Chain.Count; i++) { Assert.IsTrue((entities[0] == null && entities[i] == null) || (entities[0] != null && entities[0].Equals(entities[i])), "Entities in storage accounts: #0 and #{0} do NOT match", i); } Console.WriteLine("Entities in different accounts are consistent."); } Console.WriteLine("END ReadFromIndividualAccountsDirectly()\n"); }
public void BatchOperationUsingLargerViewId() { long currentViewId = 100; long futureViewId = currentViewId + 1; this.UpdateConfiguration(replicas, 0, false, currentViewId); string jobType = "jobType-BatchOperationUsingLargerViewId"; string jobId = "jobId-BatchOperationUsingLargerViewId"; int count = 2; // number of operations in the batch _rtable_Operation List <TableOperationType> opTypes = new List <TableOperationType>() { TableOperationType.Replace, TableOperationType.Delete, }; // // Insert // string jobIdTemplate = jobId + "-{0}"; string messageTemplate = "message-{0}"; string updatedMessageTemplate = "updated-" + messageTemplate; string partitionKey = string.Empty; // // Insert entities // for (int i = 0; i < count; i++) { SampleRTableEntity originalEntity = new SampleRTableEntity( jobType, string.Format(jobIdTemplate, i), string.Format(messageTemplate, i)); this.repTable.Execute(TableOperation.Insert(originalEntity)); partitionKey = originalEntity.PartitionKey; } // // Retrieve entities and use them to create batchOperation to Replace or Delete // IEnumerable <SampleRTableEntity> allEntities = this.rtableWrapper.GetAllRows(partitionKey); TableBatchOperation batchOperation = new TableBatchOperation(); int m = 0; foreach (SampleRTableEntity entity in allEntities) { Console.WriteLine("{0}", entity.ToString()); Console.WriteLine("---------------------------------------"); if (opTypes[m] == TableOperationType.Replace) { SampleRTableEntity replaceEntity = new SampleRTableEntity( entity.JobType, entity.JobId, string.Format(updatedMessageTemplate, m)) { ETag = entity._rtable_Version.ToString() }; batchOperation.Replace(replaceEntity); } else if (opTypes[m] == TableOperationType.InsertOrReplace) { SampleRTableEntity replaceEntity = new SampleRTableEntity( entity.JobType, entity.JobId, string.Format(updatedMessageTemplate, m)) { ETag = entity._rtable_Version.ToString() }; batchOperation.InsertOrReplace(replaceEntity); } else if (opTypes[m] == TableOperationType.Delete) { entity.ETag = entity._rtable_Version.ToString(); batchOperation.Delete(entity); } else { throw new ArgumentException( string.Format("opType={0} is NOT supported", opTypes[m]), "opType"); } m++; } // // Call ModifyConfigurationBlob to change the viewId of the wrapper to an older value // Console.WriteLine("Changing the viewId to larger viewId {0}", futureViewId); this.UpdateConfiguration(replicas, 0, false, futureViewId); Console.WriteLine("\nCalling BatchOperation with a larger viewId..."); this.repTable.ExecuteBatch(batchOperation); this.ExecuteBatchOperationAndValidate( count, partitionKey, jobType, jobId, opTypes); }
/// <summary> /// Helper function to set up an original entity and then run the specified HttpMangler Behavior. /// It will return the original entity created. /// Step (1): Make sure "entity1" with the specified partitionKey and rowKey exists. If not create it. Read "entity1". /// Step (2): Read "entity1" from individual storage tables directly for debugging. /// Step (3): Turn on HttpMangler and the specified behavior and call the specified RTable API to operate on "entity1". /// This function accepts a targetApi with this form: Func<string, string, SampleRTableEntity> targetApi /// </summary> /// <param name="entityPartitionKey"></param> /// <param name="entityRowKey"></param> /// <param name="behaviors"></param> /// <param name="targetApi"></param> /// <param name="targetApiExpectedToFail"></param> /// <returns></returns> private SampleRTableEntity SetupAndRunHttpManglerBehaviorHelper( string entityPartitionKey, string entityRowKey, ProxyBehavior[] behaviors, Func<string, string, SampleRTableEntity> targetApi, bool targetApiExpectedToFail) { DateTime httpManglerStartTime; SampleRTableEntity originalEntity = new SampleRTableEntity(); // "entity1": string jobType = entityPartitionKey; string jobId = entityRowKey; string referenceMessage = SampleRTableEntity.GenerateRandomMessage(); // // Arrange // // Make sure an entity exists with the specified partitionKey and rowKey Console.WriteLine("\nMaking sure entity1 exists..."); SampleRTableEntity entity1 = this.rtableWrapper.ReadEntity(jobType, jobId); if (entity1 == null) { SampleRTableEntity createEntity1 = new SampleRTableEntity(jobType, jobId, referenceMessage); this.rtableWrapper.InsertRow(createEntity1); entity1 = this.rtableWrapper.ReadEntity(jobType, jobId); } else { // entity1 already exists. Save the value of its Message for later use referenceMessage = entity1.Message; } Assert.IsNotNull(entity1, "entity1 is null UNEXPECTEDLY."); Console.WriteLine("entity1:\n{0}", entity1.ToString()); Assert.IsTrue(entity1.JobType == jobType, "entity.JobType is incorrect."); Assert.IsTrue(entity1.JobId == jobId, "entity.JobId is incorrect."); originalEntity = entity1.Clone(); // Read from Storage Accounts directly for debugging. Check for consistency of different accounts. this.ReadFromIndividualAccountsDirectly(jobType, jobId, true); this.RunHttpManglerBehaviorHelper( entity1, behaviors, targetApi, targetApiExpectedToFail, out httpManglerStartTime); return originalEntity; }
/// <summary> /// Helper function to perform Delete, InsertOrReplace, Merge, or Replace operation for the specified set of jobType and jobId. /// And validate the results. /// </summary> /// <param name="opType"></param> /// <param name="jobType"></param> /// <param name="jobId"></param> /// <param name="partition"></param> /// <param name="row"></param> /// <param name="originalMessage"></param> /// <param name="updatedMessage"></param> protected void PerformIndividualOperationAndValidate( TableOperationType opType, string jobType, string jobId, int partition, int row, string originalMessage, string updatedMessage = "") { Console.WriteLine("\nValidating {0} operation: updatedEntityMessage={1}...", opType, message); string partitionKey; string rowKey; SampleRTableEntity.GenerateKeys(this.GenerateJobType(jobType, partition), this.GenerateJobId(jobId, partition, row), out partitionKey, out rowKey); Console.WriteLine("PartitionKey = {0}, RowKey = {1}", partitionKey, rowKey); int attempts = 1; bool passed = true; bool resetRow = false; SampleRTableEntity retrievedEntity = null; while (attempts < 3) { try { if (opType != TableOperationType.Insert) { retrievedEntity = this.rtableWrapper.FindRow(partitionKey, rowKey); } Console.WriteLine("attempts={0}. partitionKey={1} rowKey={2}. Calling {3} API...", attempts, partitionKey, rowKey, opType); switch (opType) { case TableOperationType.Delete: { this.rtableWrapper.DeleteRow(retrievedEntity); //Validate try { retrievedEntity = this.rtableWrapper.FindRow(partitionKey, rowKey); Assert.Fail("After DeleteRow() was called, FindRow() did not throw RTableResourceNotFoundException"); } catch (RTableResourceNotFoundException) { } } break; case TableOperationType.Insert: { SampleRTableEntity sampleRtableEntity = new SampleRTableEntity( this.GenerateJobType(jobType, partition), this.GenerateJobId(jobId, partition, row), this.GenerateMessage(originalMessage, partition, row)); this.rtableWrapper.InsertRow(sampleRtableEntity); //Validate try { retrievedEntity = this.rtableWrapper.FindRow(partitionKey, rowKey); } catch (RTableResourceNotFoundException) { Assert.Fail("After InsertRow() was called, FindRow() threw RTableResourceNotFoundException"); } } break; case TableOperationType.InsertOrMerge: { retrievedEntity.Message = this.GenerateMessage(updatedMessage, partition, row); this.rtableWrapper.InsertOrMergeRow(retrievedEntity); retrievedEntity = this.rtableWrapper.FindRow(partitionKey, rowKey); this.ValidateRetrievedRTableEntity(retrievedEntity, jobType, jobId, updatedMessage, partition, row); } break; case TableOperationType.InsertOrReplace: { retrievedEntity.Message = this.GenerateMessage(updatedMessage, partition, row); this.rtableWrapper.InsertOrReplaceRow(retrievedEntity); retrievedEntity = this.rtableWrapper.FindRow(partitionKey, rowKey); this.ValidateRetrievedRTableEntity(retrievedEntity, jobType, jobId, updatedMessage, partition, row); resetRow = true; } break; case TableOperationType.Merge: { retrievedEntity.Message = this.GenerateMessage(updatedMessage, partition, row); this.rtableWrapper.MergeRow(retrievedEntity); retrievedEntity = this.rtableWrapper.FindRow(partitionKey, rowKey); this.ValidateRetrievedRTableEntity(retrievedEntity, jobType, jobId, updatedMessage, partition, row); } break; case TableOperationType.Replace: { retrievedEntity.Message = this.GenerateMessage(updatedMessage, partition, row); this.rtableWrapper.ReplaceRow(retrievedEntity); retrievedEntity = this.rtableWrapper.FindRow(partitionKey, rowKey); this.ValidateRetrievedRTableEntity(retrievedEntity, jobType, jobId, updatedMessage, partition, row); resetRow = true; } break; case TableOperationType.Retrieve: { this.rtableWrapper.FindRow(partitionKey, rowKey); } break; default: { throw new InvalidOperationException(string.Format("opType={0} is NOT supported", opType)); } } passed = true; break; // get out of while(attempts) if no RTableConflictException } catch (RTableConflictException) { passed = false; Console.WriteLine("Got RTableConflictException. attempts={0}", attempts); attempts++; System.Threading.Thread.Sleep(1000); } } Assert.IsTrue(passed, "Keep getting RTableConflictException when calling {0} API", opType); Console.WriteLine("Passed {0} validation.\n", opType); //Reset row to original values so subsequent updates on the same row get validated correctly if (resetRow == true) { Console.WriteLine("Resetting row to original values"); retrievedEntity.Message = this.GenerateMessage(originalMessage, partition, row); this.rtableWrapper.ReplaceRow(retrievedEntity); } }
public void A00DelayTwoConflictingInsertOrReplaceCalls() { this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable); string entityPartitionKey = "jobType-DelayInsertOrReplaceRowHeadTest"; string entityRowKey = "jobId-DelayInsertOrReplaceRowHeadTest"; this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey); string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0]; Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper); int delayInMs = 3000; int insertRequestCount = 0; int conflictResponseCount = 0; bool secondUpsertConflicted = false; int failedCallIndex = -1; // Delay behavior ProxyBehavior[] behaviors = new[] { // Delay Insert calls so they end up conflicting TamperBehaviors.TamperAllRequestsIf( (session => { Interlocked.Increment(ref insertRequestCount); while (insertRequestCount != 2) { Console.WriteLine("insertRequestCount={0}. Waiting on count to reach 2 ...", insertRequestCount); Thread.Sleep(delayInMs); } }), (session => { if (session.hostname.Contains(accountNameToTamper + ".") && session.HTTPMethodIs("POST") && session.GetRequestBodyAsString().Contains("\"_rtable_Operation\":\"Insert\"")) { return(true); } return(false); })), // Delay conflict response DelayBehaviors.DelayAllResponsesIf( delayInMs, (session => { if (session.hostname.Contains(accountNameToTamper + ".") && session.GetRequestBodyAsString().Contains("\"_rtable_Operation\":\"Insert\"")) { if (session.responseCode == (int)HttpStatusCode.Conflict) { Interlocked.Increment(ref conflictResponseCount); return(true); } } return(false); })), }; using (new HttpMangler(false, behaviors)) { Parallel.For(0, 2, (index) => { var entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, string.Format("upsert message {0}", index)); try { this.rtableWrapper.InsertOrReplaceRow(entry); } catch (RTableConflictException) { if (secondUpsertConflicted) { // should never reach here throw; } // That's possible, but that's the Replace step of upsert which conflicted with ongoing write // can't do anything, client should retry on conflict secondUpsertConflicted = true; } }); } // got 2 inserts? Assert.AreEqual(2, insertRequestCount, "Two insert calls expected!"); // got one conflict? Assert.AreEqual(1, conflictResponseCount, "One conflict response expected!"); // at least one upsert would have succeeded SampleRTableEntity upsertedEntity = this.rtableWrapper.ReadEntity(entityPartitionKey, entityRowKey); Assert.NotNull(upsertedEntity, "at least one upsert should have succeeded"); // second upsert failed? if (secondUpsertConflicted) { Assert.AreEqual(1, upsertedEntity._rtable_Version, "one upsert succeeded so version should be = 1"); Assert.AreEqual(upsertedEntity.Message, string.Format("upsert message {0}", (1 - failedCallIndex))); } else { Assert.AreEqual(2, upsertedEntity._rtable_Version, "both upserts succeeded so version should be = 2"); } // After recovery from delay, confirm that we can update the row. //this.ExecuteReplaceRowAndValidate(entityPartitionKey, entityRowKey); }
/// <summary> /// Copy the values of JobType, JobId, Message to the specified object /// </summary> /// <param name="dst"></param> public void CopyTo(SampleRTableEntity dst) { dst.JobType = this.JobType; dst.JobId = this.JobId; dst.Message = this.Message; }
public void A00InsertOrReplaceCallConflictingWithDeleteOnTheHead() { this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable); string entityPartitionKey = "jobType-DelayInsertOrReplaceWhileDelete"; string entityRowKey = "jobType-DelayInsertOrReplaceWhileDelete"; this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey); // Insert one entry Console.WriteLine("Inserting entry ..."); var entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, "insert message"); this.rtableWrapper.InsertRow(entry); // 1 - Launch an Upsert task in wait mode ... bool deleteLockedHead = false; TableResult upsertResult = null; var upsertTask = Task.Run(() => { while (!deleteLockedHead) { Thread.Sleep(5); } try { entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, "upsert message"); var table = new ReplicatedTable(this.repTable.TableName, this.configurationService); TableOperation upserOperation = TableOperation.InsertOrReplace(entry); Console.WriteLine("Upsert started ..."); upsertResult = table.Execute(upserOperation); Console.WriteLine("Upsert completed with HttpStatus={0}", upsertResult == null ? "NULL" : upsertResult.HttpStatusCode.ToString()); } catch (AggregateException ex) { Console.WriteLine(ex); } }); string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0]; Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper); int delayInMs = 5; // Delay behavior ProxyBehavior[] behaviors = new[] { DelayBehaviors.DelayAllResponsesIf( delayInMs, (session => { var body = session.GetRequestBodyAsString(); Console.WriteLine(body); // Delete locking the head response if (session.hostname.Contains(accountNameToTamper + ".") && session.HTTPMethodIs("PUT") && body.Contains("\"_rtable_Operation\":\"Replace\"") && body.Contains("\"_rtable_RowLock\":true") && body.Contains("\"_rtable_Tombstone\":true")) { // Signal upsert we locked the head, so it can continue ... deleteLockedHead = true; return(true); } return(false); })), }; // Launch a delete using (new HttpMangler(false, behaviors)) { try { var table = new ReplicatedTable(this.repTable.TableName, this.configurationService); // Retrieve entity TableOperation retrieveOperation = TableOperation.Retrieve <SampleRTableEntity>(entry.PartitionKey, entry.RowKey); TableResult retrieveResult = table.Execute(retrieveOperation); Assert.IsNotNull(retrieveResult, "retrieveResult = null"); Assert.AreEqual((int)HttpStatusCode.OK, retrieveResult.HttpStatusCode, "retrieveResult.HttpStatusCode mismatch"); Assert.IsNotNull((SampleRTableEntity)retrieveResult.Result, "Retrieve: customer = null"); // Delete entity TableOperation deleteOperation = TableOperation.Delete((SampleRTableEntity)retrieveResult.Result); TableResult deleteResult = table.Execute(deleteOperation); Assert.IsNotNull(deleteResult, "deleteResult = null"); Assert.AreEqual((int)HttpStatusCode.NoContent, deleteResult.HttpStatusCode, "deleteResult.HttpStatusCode mismatch"); } catch (AggregateException ex) { Console.WriteLine(ex); } } // wait on upsert to finish ... upsertTask.Wait(); // Upsert suceeded? Assert.AreEqual((int)HttpStatusCode.NoContent, upsertResult.HttpStatusCode, "Upsert expected to suceed!"); SampleRTableEntity upsertedEntity = this.rtableWrapper.ReadEntity(entityPartitionKey, entityRowKey); Assert.NotNull(upsertedEntity, "upsert should succeed"); Assert.AreEqual(upsertedEntity.Message, "upsert message", "upsert should succeeded"); }
/// <summary> /// Helper function to perform Delete, InsertOrReplace, Merge, or Replace operation for the specified set of jobType and jobId. /// And validate the results. /// </summary> /// <param name="opType"></param> /// <param name="jobType"></param> /// <param name="jobId"></param> /// <param name="updatedEntityMessage"></param> protected void PerformOperationAndValidate( TableOperationType opType, string jobType, string jobId, string updatedEntityMessage = "") { Console.WriteLine("\nValidating {0} operation: updatedEntityMessage={1}...", opType, updatedEntityMessage); for (int i = 0; i < this.numberOfPartitions; i++) { List <string> rowKeys = new List <string>(); // list of rowKey for the given paritionKey string partitionKey; string rowKey; SampleRTableEntity.GenerateKeys(this.GenerateJobType(jobType, i), "don't care", out partitionKey, out rowKey); // // GetAllRows() // Console.WriteLine("Calling GetAllRows() and then ReplaceRow() for partition {0}", i); IEnumerable <SampleRTableEntity> allRows = this.rtableWrapper.GetAllRows(partitionKey); int j = 0; // counting the number of rows per partition List <SampleRTableEntity> allRetrievedEntities = new List <SampleRTableEntity>(); foreach (var retrievedEntity in allRows) { allRetrievedEntities.Add(retrievedEntity); j++; } Assert.AreEqual(this.numberOfRowsPerPartition, j, "Partition {0} only has {1} rows. Expected {2} rows", i, j, this.numberOfRowsPerPartition); j = 0; foreach (var oneEntry in allRetrievedEntities) { int attempts = 1; bool passed = true; while (attempts < 3) { try { SampleRTableEntity retrievedEntity = oneEntry; Console.WriteLine("attempts={0}. partitionKey={1} rowKey={2}. Calling {3} API...", attempts, partitionKey, retrievedEntity.RowKey, opType); switch (opType) { case TableOperationType.Delete: { this.rtableWrapper.DeleteRow(retrievedEntity); } break; case TableOperationType.InsertOrReplace: { retrievedEntity.Message = this.GenerateMessage(updatedEntityMessage, i, j); this.rtableWrapper.InsertOrReplaceRow(retrievedEntity); rowKeys.Add(retrievedEntity.RowKey); } break; case TableOperationType.Merge: { retrievedEntity.Message = this.GenerateMessage(updatedEntityMessage, i, j); this.rtableWrapper.MergeRow(retrievedEntity); rowKeys.Add(retrievedEntity.RowKey); } break; case TableOperationType.Replace: { retrievedEntity.Message = this.GenerateMessage(updatedEntityMessage, i, j); this.rtableWrapper.ReplaceRow(retrievedEntity); rowKeys.Add(retrievedEntity.RowKey); } break; default: { throw new InvalidOperationException(string.Format("opType={0} is NOT supported", opType)); } } passed = true; break; // get out of while(attempts) if no RTableConflictException } catch (RTableConflictException) { passed = false; Console.WriteLine("Got RTableConflictException. attempts={0}", attempts); attempts++; System.Threading.Thread.Sleep(1000); } } Assert.IsTrue(passed, "Keep getting RTableConflictException when calling {0} API", opType); j++; } Assert.AreEqual(this.numberOfRowsPerPartition, j, "Partition {0} only has {1} rows. Expected {2} rows", i, j, this.numberOfRowsPerPartition); // // FindRow() // Console.WriteLine("Calling FindRow() for partitionKey={0}", partitionKey); for (j = 0; j < rowKeys.Count; j++) { if (opType == TableOperationType.Delete) { try { SampleRTableEntity retrievedEntity = this.rtableWrapper.FindRow(partitionKey, rowKeys[j]); Assert.Fail("After DeleteRow() was called, FindRow() did not throw RTableResourceNotFoundException"); } catch (RTableResourceNotFoundException) { } } else { SampleRTableEntity retrievedEntity = this.rtableWrapper.FindRow(partitionKey, rowKeys[j]); this.ValidateRetrievedRTableEntity(retrievedEntity, jobType, jobId, updatedEntityMessage, i, j); } } } Console.WriteLine("Passed {0} validation.\n", opType); }
public void RTableRepairRowDelete() { // Insert entity Assert.IsTrue(this.repTable.Exists(), "RTable does not exist"); View fullView = configurationWrapper.GetWriteView(); List<ReplicaInfo> fullViewReplicas = new List<ReplicaInfo>(); for (int i = 0; i <= fullView.TailIndex; i++) { fullViewReplicas.Add(fullView.GetReplicaInfo(i)); } List<ReplicaInfo> newReplicas = new List<ReplicaInfo>(); for (int i = 1; i <= fullView.TailIndex; i++) { newReplicas.Add(fullView.GetReplicaInfo(i)); } SampleRTableEntity newCustomer = new SampleRTableEntity("firstName1", "lastName1", "*****@*****.**"); TableOperation operation = TableOperation.Insert(newCustomer); TableResult result = repTable.Execute(operation); Assert.AreNotEqual(null, result, "result = null"); ReplicatedTableEntity row = (ReplicatedTableEntity)result.Result; Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch"); Assert.AreNotEqual(null, result.Result, "result.Result = null"); Assert.AreEqual("1", result.Etag, "result.Etag mismatch"); Assert.AreEqual(false, row._rtable_RowLock, "row._rtable_RowLock mismatch"); Assert.AreEqual(1, row._rtable_Version, "row._rtable_Version mismatch"); Assert.AreEqual(false, row._rtable_Tombstone, "row._rtable_Tombstone mismatch"); Assert.AreEqual(configurationWrapper.GetWriteView().ViewId, row._rtable_ViewId, "row._rtable_ViewId mismatch"); ReadFromIndividualAccountsDirectly(newCustomer.PartitionKey, newCustomer.RowKey, true); // remove replica from the head this.UpdateConfiguration(newReplicas, 0); Assert.IsTrue(configurationWrapper.IsViewStable()); // delete row TableOperation deleteOperation = TableOperation.Delete(newCustomer); result = repTable.Execute(deleteOperation); Assert.AreNotEqual(null, result, "result = null"); Assert.AreEqual((int) HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch"); //Add replica at head this.UpdateConfiguration(fullViewReplicas, 1); // repair row on the new head Console.WriteLine("Calling repair row"); result = repTable.RepairRow(row.PartitionKey, row.RowKey, null); Assert.AreNotEqual(null, result, "result = null"); // Retrieve Entity Console.WriteLine("Calling TableOperation.Retrieve<SampleRtableEntity>(firstName, lastName)..."); operation = TableOperation.Retrieve<SampleRTableEntity>("firstname1", "lastName1"); TableResult retrievedResult = repTable.Execute(operation); Assert.AreNotEqual(null, retrievedResult, "retrievedResult = null"); Assert.AreEqual((int)HttpStatusCode.NotFound, retrievedResult.HttpStatusCode, "retrievedResult.HttpStatusCode mismatch"); ReadFromIndividualAccountsDirectly(newCustomer.PartitionKey, newCustomer.RowKey, true); }
public void BatchOperationExceptionWhenUsingSmallerViewId() { long currentViewId = 100; long badViewId = currentViewId - 1; this.UpdateConfiguration(replicas, 0, false, currentViewId); string jobType = "jobType-BatchOperationExceptionWhenUsingSmallerViewId"; string jobId = "jobId-BatchOperationExceptionWhenUsingSmallerViewId"; int count = 3; // number of operations in the batch _rtable_Operation List <TableOperationType> opTypes = new List <TableOperationType>() { TableOperationType.Replace, TableOperationType.InsertOrReplace, TableOperationType.Delete, }; // // Insert // string jobIdTemplate = jobId + "-{0}"; string messageTemplate = "message-{0}"; string updatedMessageTemplate = "updated-" + messageTemplate; string partitionKey = string.Empty; // // Insert entities // for (int i = 0; i < count; i++) { SampleRTableEntity originalEntity = new SampleRTableEntity( jobType, string.Format(jobIdTemplate, i), string.Format(messageTemplate, i)); this.repTable.Execute(TableOperation.Insert(originalEntity)); partitionKey = originalEntity.PartitionKey; } // // Retrieve entities and use them to create batchOperation to Replace or Delete // IEnumerable <SampleRTableEntity> allEntities = this.rtableWrapper.GetAllRows(partitionKey); TableBatchOperation batchOperation = new TableBatchOperation(); int m = 0; foreach (SampleRTableEntity entity in allEntities) { Console.WriteLine("{0}", entity.ToString()); Console.WriteLine("---------------------------------------"); if (opTypes[m] == TableOperationType.Replace) { SampleRTableEntity replaceEntity = new SampleRTableEntity( entity.JobType, entity.JobId, string.Format(updatedMessageTemplate, m)) { ETag = entity._rtable_Version.ToString() }; batchOperation.Replace(replaceEntity); } else if (opTypes[m] == TableOperationType.InsertOrReplace) { SampleRTableEntity replaceEntity = new SampleRTableEntity( entity.JobType, entity.JobId, string.Format(updatedMessageTemplate, m)) { ETag = entity._rtable_Version.ToString() }; batchOperation.InsertOrReplace(replaceEntity); } else if (opTypes[m] == TableOperationType.Delete) { entity.ETag = entity._rtable_Version.ToString(); batchOperation.Delete(entity); } else { throw new ArgumentException( string.Format("opType={0} is NOT supported", opTypes[m]), "opType"); } m++; } // // Call ModifyConfigurationBlob to change the viewId of the wrapper to an older value // Console.WriteLine("Changing the viewId to badViewId {0}", badViewId); this.UpdateConfiguration(replicas, 0, false, badViewId); // // Execute Batch _rtable_Operation with bad viewId // Console.WriteLine("\nCalling BatchOperation with badViewId..."); try { this.repTable.ExecuteBatch(batchOperation); } catch (ReplicatedTableStaleViewException ex) { Console.WriteLine("Get this RTableStaleViewException: {0}", ex.Message); Assert.IsTrue(ex.ErrorCode == ReplicatedTableViewErrorCodes.ViewIdSmallerThanEntryViewId); Assert.IsTrue(ex.Message.Contains(string.Format("current _rtable_ViewId {0} is smaller than", badViewId)), "Got unexpected exception message"); } }
/// <summary> /// Helper function to create a new entity and validate the creation was successful. /// </summary> /// <param name="entityPartitionKey"></param> /// <param name="entityRowKey"></param> protected void ExecuteCreateRowAndValidate( string entityPartitionKey, string entityRowKey) { Console.WriteLine("\nExecuteCreateRowAndValidate(): Create entity and validate..."); string jobType = entityPartitionKey; string jobId = entityRowKey; string referenceMessage = SampleRTableEntity.GenerateRandomMessage(); SampleRTableEntity createEntity = new SampleRTableEntity(jobType, jobId, referenceMessage); bool gotExceptionInLastAttempt = true; int retries = 0; while (retries < MaxRetries) { try { Console.WriteLine("\nretries={0}. Calling InsertRow(). referenceMessage={1}", retries, referenceMessage); this.rtableWrapper.InsertRow(createEntity); gotExceptionInLastAttempt = false; break; } catch (RTableConflictException ex) { Console.WriteLine("retries={0}. InsertRow() got an RTableConflictException: {1}", retries, ex.ToString()); retries++; gotExceptionInLastAttempt = true; Thread.Sleep(this.configurationWrapper.GetLockTimeout()); // For debug purposes: read from the Head and Tail accounts: this.ReadFromIndividualAccountsDirectly(entityPartitionKey, entityRowKey); } catch (RTableRetriableException ex) { Console.WriteLine("retries={0}. InsertRow() got an RTableRetriableException: {1}", retries, ex.ToString()); retries++; gotExceptionInLastAttempt = true; Thread.Sleep(ConflictExceptionSleepTimeInMsec); // For debug purposes: read from the Head and Tail accounts: this.ReadFromIndividualAccountsDirectly(entityPartitionKey, entityRowKey); } } Console.WriteLine("gotExceptionInLastAttempt = {0}", gotExceptionInLastAttempt); Console.WriteLine("\nRead entity back and validate..."); SampleRTableEntity retrievedEntity = this.rtableWrapper.ReadEntity(jobType, jobId); Assert.IsNotNull(retrievedEntity, "retrievedEntity is null UNEXPECTEDLY."); Console.WriteLine("retrievedEntity:\n{0}", retrievedEntity.ToString()); Assert.IsTrue(retrievedEntity.JobType == jobType, "retrievedEntity.JobType is incorrect."); Assert.IsTrue(retrievedEntity.JobId == jobId, "retrievedEntity.JobId is incorrect."); Assert.IsTrue(retrievedEntity.Message == referenceMessage, "retrievedEntity.Message is incorrect."); Console.WriteLine("Passed validation"); }
/// <summary> /// This help function is for getting the reference behavior of the Storage DLL when HttpMangler is enabled. /// Edit HttpMangler.cs to use HandleFiddlerEvent_DEBUG(). /// </summary> /// <param name="entityPartitionKey"></param> /// <param name="entityRowKey"></param> /// <param name="targetStorageAccount"></param> /// <param name="targetApiExpectedToFail"></param> /// <param name="behaviors"></param> protected void SetupAndRunXStoreHttpManglerTest( string entityPartitionKey, string entityRowKey, int targetStorageAccount, bool targetApiExpectedToFail, ProxyBehavior[] behaviors) { Assert.IsTrue(0 <= targetStorageAccount && targetStorageAccount < this.actualStorageAccountsUsed.Count, "targetStorageAccount={0} is out-of-range", targetStorageAccount); int index = this.actualStorageAccountsUsed[targetStorageAccount]; string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[index]; Console.WriteLine("accountNameToTamper={0}", accountNameToTamper); CloudTableClient tableClient = this.cloudTableClients[targetStorageAccount]; CloudTable table = tableClient.GetTableReference(this.repTable.TableName); // // Insert // string jobType = entityPartitionKey; string jobId = entityRowKey; string referenceMessage = SampleRTableEntity.GenerateRandomMessage(); SampleRTableEntity originalEntity = new SampleRTableEntity(jobType, jobId, referenceMessage); Console.WriteLine("\nCalling XStore Insert..."); TableOperation insertOperation = TableOperation.Insert(originalEntity); TableResult insertResult = table.Execute(insertOperation); Assert.IsNotNull(insertResult, "insertResult = null"); Console.WriteLine("insertResult.HttpStatusCode = {0}", insertResult.HttpStatusCode); Console.WriteLine("insertResult.ETag = {0}", insertResult.Etag); Assert.AreEqual((int)HttpStatusCode.NoContent, insertResult.HttpStatusCode, "insertResult.HttpStatusCode mismatch"); Assert.IsFalse(string.IsNullOrEmpty(insertResult.Etag), "insertResult.ETag = null or empty"); ITableEntity row = (ITableEntity)insertResult.Result; // // Retrieve // Console.WriteLine("Calling XStore Retrieve..."); TableOperation retrieveOperation = TableOperation.Retrieve<SampleRTableEntity>(row.PartitionKey, row.RowKey); TableResult retrieveResult = table.Execute(retrieveOperation); Assert.IsNotNull(retrieveResult, "retrieveResult = null"); Console.WriteLine("retrieveResult.HttpStatusCode = {0}", retrieveResult.HttpStatusCode); Assert.AreEqual((int)HttpStatusCode.OK, retrieveResult.HttpStatusCode, "retrieveResult.HttpStatusCode mismatch"); SampleRTableEntity retrievedEntity = (SampleRTableEntity)retrieveResult.Result; Console.WriteLine("retrieveEntity:\n{0}", retrievedEntity); Assert.IsTrue(originalEntity.Equals(retrievedEntity), "originalEntity != retrievedEntity"); // // Replace with HttpMangler enabled // Console.WriteLine("Calling XStore TableOperation.Replace with HttpMangler enabled..."); referenceMessage = SampleRTableEntity.GenerateRandomMessage(); retrievedEntity.Message = referenceMessage; TableOperation updateOperation = TableOperation.Replace(retrievedEntity); bool abortTest = false; try { using (HttpMangler proxy = new HttpMangler(false, behaviors)) { Console.WriteLine("Calling table.Execute(updateOperation)"); TableResult updateResult = table.Execute(updateOperation); if (targetApiExpectedToFail) { // if targetApi is expected to fail, and we are here, that means something is wrong. abortTest = true; throw new Exception("SetupAndRunXStoreHttpManglerTest(): Should not reach here. HttpMangler allowed an targetApi() to go through UNEXPECTEDLY."); } } } catch (Exception ex) { if (abortTest) { throw; } else { Console.WriteLine("\nException is Expected. targetApi(entity1) threw an exception: {0}\n", ex.ToString()); } } // // Retrieve again // Console.WriteLine("After HttpMangler is disabled, calling XStore Retrieve again..."); retrieveOperation = TableOperation.Retrieve<SampleRTableEntity>(row.PartitionKey, row.RowKey); retrieveResult = table.Execute(retrieveOperation); Assert.IsNotNull(retrieveResult, "retrieveResult = null"); Console.WriteLine("retrieveResult.HttpStatusCode = {0}", retrieveResult.HttpStatusCode); Assert.AreEqual((int)HttpStatusCode.OK, retrieveResult.HttpStatusCode, "retrieveResult.HttpStatusCode mismatch"); SampleRTableEntity retrievedEntity2 = (SampleRTableEntity)retrieveResult.Result; Console.WriteLine("retrieveEntity2:\n{0}", retrievedEntity2); Assert.IsTrue(originalEntity.Equals(retrievedEntity2), "originalEntity != retrievedEntity2"); }
/// <summary> /// Helper function to make ReadEntity() API call and validate the retrieve contents match the specified originalEntity. /// Call this helper after you have finished tampered some operation on the same entity. /// </summary> /// <param name="entityPartitionKey"></param> /// <param name="entityRowKey"></param> /// <param name="originalEntity">Set it to null to by-pass validation against originalEntity.</param> /// <param name="checkIndividualAccounts">True means will check the entity in each storage account for consistency</param> protected void ExecuteReadRowAndValidate( string entityPartitionKey, string entityRowKey, SampleRTableEntity originalEntity, bool checkIndividualAccounts = false) { Console.WriteLine("\nExecuteReadRowAndValidate(): Read entity back and validate..."); string jobType = entityPartitionKey; string jobId = entityRowKey; SampleRTableEntity retrievedEntity = this.rtableWrapper.ReadEntity(jobType, jobId); if (retrievedEntity == null) { Console.WriteLine("ERROR: retrievedEntity = null"); } else { Console.WriteLine("retrievedEntity = \n{0}", retrievedEntity.ToString()); } if (originalEntity != null) { Console.WriteLine("originalEntity = \n{0}", originalEntity.ToString()); } // For debug purposes: read from the Head and Tail accounts: this.ReadFromIndividualAccountsDirectly(jobType, jobId, checkIndividualAccounts); Assert.IsNotNull(retrievedEntity, "retrievedEntity is null UNEXPECTEDLY"); if (originalEntity != null) { Assert.IsTrue(retrievedEntity.Equals(originalEntity), "retrievedEntity NOT equal to originalEntity"); Console.WriteLine("Passed validation"); } }
/// <summary> /// Helper function to create some initial entities and then call the specified batchOperation with HttpMangler enabled. /// </summary> /// <param name="count"></param> /// <param name="jobType"></param> /// <param name="jobId"></param> /// <param name="targetStorageAccount"></param> /// <param name="opTypes"></param> /// <param name="targetApiExpectedToFail"></param> /// <param name="checkOriginalEntityUnchanged"></param> /// <param name="checkStorageAccountsConsistent"></param> /// <param name="httpManglerStartTime"></param> /// <param name="skipInitialSessions"></param> /// <returns>ParitionKey of the initial entities created</returns> protected string SetupAndRunTemperBatchOperation( int count, string jobType, string jobId, int targetStorageAccount, List<TableOperationType> opTypes, bool targetApiExpectedToFail, bool checkOriginalEntityUnchanged, bool checkStorageAccountsConsistent, out DateTime httpManglerStartTime, int skipInitialSessions = 0) { Assert.IsTrue(0 <= targetStorageAccount && targetStorageAccount < this.actualStorageAccountsUsed.Count, "SetupAndRunTemperBatchOperation() is called with out-of-range targetStorageAccount={0}", targetStorageAccount); int index = this.actualStorageAccountsUsed[targetStorageAccount]; string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[index]; Console.WriteLine("SetupAndRunTemperBatchOperation(): accountNameToTamper={0} skipInitialSessions={1}", accountNameToTamper, skipInitialSessions); Assert.AreEqual(count, opTypes.Count, "count and opTypes.Count should be the same"); // // Tamper behavior // ProxyBehavior[] behaviors = new[] { TamperBehaviors.TamperAllRequestsIf( (session => { session.Abort();}), skipInitialSessions, AzureStorageSelectors.TableTraffic().IfHostNameContains(accountNameToTamper + ".")) }; string jobIdTemplate = jobId + "-{0}"; string messageTemplate = "message-{0}"; string updatedMessageTemplate = "updated-" + messageTemplate; string partitionKey = string.Empty; // // Insert entities // for (int i = 0; i < count; i++) { SampleRTableEntity originalEntity = new SampleRTableEntity( jobType, string.Format(jobIdTemplate, i), string.Format(messageTemplate, i)); this.repTable.Execute(TableOperation.Insert(originalEntity)); partitionKey = originalEntity.PartitionKey; } // // Retrieve entities and use them to create batchOperation to Replace or Delete // IEnumerable<SampleRTableEntity> allEntities = this.rtableWrapper.GetAllRows(partitionKey); TableBatchOperation batchOperation = new TableBatchOperation(); int m = 0; foreach (SampleRTableEntity entity in allEntities) { Console.WriteLine("{0}", entity.ToString()); Console.WriteLine("---------------------------------------"); if (opTypes[m] == TableOperationType.Replace) { SampleRTableEntity replaceEntity = new SampleRTableEntity( entity.JobType, entity.JobId, string.Format(updatedMessageTemplate, m)) { ETag = entity._rtable_Version.ToString() }; batchOperation.Replace(replaceEntity); } else if (opTypes[m] == TableOperationType.Delete) { entity.ETag = entity._rtable_Version.ToString(); batchOperation.Delete(entity); } else { throw new ArgumentException( string.Format("opType={0} is NOT supported", opTypes[m]), "opType"); } m++; } // // Enable HttpMangler // Call this.repTable.ExecuteBatch(batchOperation) // this.RunHttpManglerBehaviorHelper( batchOperation, behaviors, targetApiExpectedToFail, out httpManglerStartTime); if (checkOriginalEntityUnchanged) { Console.WriteLine("Validate originalEntity remain unchanged."); allEntities = this.rtableWrapper.GetAllRows(partitionKey); batchOperation = new TableBatchOperation(); m = 0; foreach (SampleRTableEntity entity in allEntities) { Console.WriteLine("{0}", entity.ToString()); Console.WriteLine("---------------------------------------"); Assert.AreEqual(string.Format(jobType, m), entity.JobType, "JobType does not match"); Assert.AreEqual(string.Format(jobIdTemplate, m), entity.JobId, "JobId does not match"); Assert.AreEqual(string.Format(messageTemplate, m), entity.Message, "Message does not match"); m++; } Console.WriteLine("Passed validation"); } // // After httpMangler is turned off, read from individual accounts... // Console.WriteLine("\nAfter httpMangler is turned off, read from individual accounts..."); for (int i = 0; i < count; i++) { this.ReadFromIndividualAccountsDirectly( jobType, string.Format(jobIdTemplate, i), checkStorageAccountsConsistent); } return partitionKey; }
/// <summary> /// Run the specified HttpMangler proxy behaviors for the specified Api, which takes this form: /// Func<string, string, SampleRTableEntity> targetApi /// </summary> /// <param name="originalEntity"></param> /// <param name="behaviors"></param> /// <param name="targetApi"></param> /// <param name="targetApiExpectedToFail"></param> /// <param name="httpManglerStartTime"></param> protected void RunHttpManglerBehaviorHelper( SampleRTableEntity originalEntity, ProxyBehavior[] behaviors, Func<string, string, SampleRTableEntity> targetApi, bool targetApiExpectedToFail, out DateTime httpManglerStartTime) { // // Act // // Call targetApi(entity1) and tamper the request httpManglerStartTime = DateTime.UtcNow; Console.WriteLine("\nRunHttpManglerBehaviorHelper(): Call targetApi(entity1) with HttpMangler enabled..."); originalEntity.Message = SampleRTableEntity.GenerateRandomMessage(); bool abortTest = false; try { using (HttpMangler proxy = new HttpMangler(false, behaviors)) { Console.WriteLine("Calling targetApi(entity1)"); targetApi(originalEntity.PartitionKey, originalEntity.RowKey); if (targetApiExpectedToFail) { // if targetApi is expected to fail, and we are here, that means something is wrong. abortTest = true; throw new Exception("RunHttpManglerBehaviorHelper(): Should not reach here. HttpMangler allowed an targetApi() to go through UNEXPECTEDLY."); } } } catch (Exception ex) { if (targetApiExpectedToFail == false) { // if targetApi is NOT expected to fail, and we are here, that means something is wrong. throw new Exception( string.Format("RunHttpManglerBehaviorHelper(): targetApi() is NOT unexpected to throw an exception. Got this exception: {0}", ex.ToString())); } if (abortTest) { throw; } else { Console.WriteLine("\nException is Expected. targetApi(entity1) threw an exception: {0}\n", ex.ToString()); } } }
public void RepairMidAnUpdate() { this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable); // Not-Stable chain // Reconfigure RTable so Head is WriteOnly. View view = this.configurationService.GetTableView(this.repTable.TableName); ReplicatedTableConfiguration config; ReplicatedTableQuorumReadResult readStatus = this.configurationService.RetrieveConfiguration(out config); Assert.IsTrue(readStatus.Code == ReplicatedTableQuorumReadCode.Success); // Set Head as WriteOnly mode ReplicatedTableConfigurationStore viewConfg = config.GetView(view.Name); viewConfg.ReplicaChain[0].Status = ReplicaStatus.WriteOnly; config.SetView(view.Name, viewConfg); // Upload RTable config back this.configurationService.UpdateConfiguration(config); // Sanity: Replicated mode and chain Not-Stable view = this.configurationService.GetTableView(this.repTable.TableName); Assert.IsTrue(view != null && view.Chain.Count > 1, "Two replicas should be used."); Assert.IsFalse(view.IsStable); // Insert one entry Console.WriteLine("Inserting entry ..."); string entityPartitionKey = "jobType-RepairMidAnUpdate-Replace"; string entityRowKey = "jobId-RepairMidAnUpdate-Replace"; var entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, "message"); this.rtableWrapper.InsertRow(entry); // 1 - Launch a RepairRow task in wait mode ... bool triggerRepair = false; bool repaireDone = false; bool headAfterRepairWasLocked = false; TableResult repairResult = null; Task.Run(() => { ReplicatedTable repairTable = new ReplicatedTable(this.repTable.TableName, this.configurationService); while (!triggerRepair) { Thread.Sleep(5); } Console.WriteLine("RepairRow started ..."); repairResult = repairTable.RepairRow(entry.PartitionKey, entry.RowKey, null); Console.WriteLine("RepairRow completed with HttpStatus={0}", repairResult == null ? "NULL" : repairResult.HttpStatusCode.ToString()); // Check the entry at the Head is still locked i.e. RepairRow was NOP headAfterRepairWasLocked = HeadIsLocked(entry); Console.WriteLine("Signal the commit to Head job"); repaireDone = true; }); // 2 - Configure Mangler ... string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0]; Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper); ProxyBehavior[] behaviors = new[] { TamperBehaviors.TamperAllRequestsIf( (session => { Console.WriteLine("Delaying commit to the Head ... => signal RepairRow job"); // Let RepairRow task go through triggerRepair = true; int iter = 0; while (!repaireDone) { // TODO: break the loop after couple of iteration ... Thread.Sleep(100); Console.WriteLine("Waiting on RepairRow to finish ({0}) ...", ++iter); } Console.WriteLine("Request a commit to the head"); }), (session => { // Commit on head i.e. a PUT with RowLock == false if (session.hostname.Contains(accountNameToTamper + ".") && session.HTTPMethodIs("PUT") && session.GetRequestBodyAsString().Contains("\"_rtable_RowLock\":false")) { return(true); } return(false); })) }; using (new HttpMangler(false, behaviors)) { Console.WriteLine("Updating entry ..."); entry = this.rtableWrapper.FindRow(entry.PartitionKey, entry.RowKey); entry.Message = "updated message"; this.rtableWrapper.ReplaceRow(entry); } Assert.IsTrue(triggerRepair); Assert.IsTrue(repairResult != null && repairResult.HttpStatusCode == (int)HttpStatusCode.OK, "Repair failed."); Assert.IsTrue(repaireDone); Assert.IsTrue(headAfterRepairWasLocked); Console.WriteLine("DONE. Test passed."); }
public void TxnViewExpiresAndViewIdHasChanged() { // Insert an entry string entityPartitionKey = "jobType-ReplaceWhenTxnViewExpires"; string entityRowKey = "jobType-ReplaceWhenTxnViewExpires"; this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey); Console.WriteLine("Inserting entry ..."); var entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, "insert message"); this.rtableWrapper.InsertRow(entry); // Retrieve the config ... ReplicatedTableConfiguration config; ReplicatedTableQuorumReadResult readStatus = this.configurationService.RetrieveConfiguration(out config); Assert.IsTrue(readStatus.Code == ReplicatedTableQuorumReadCode.Success); // Reconfigure RTable config with a short LeaseDuration int leaseDurationInSec = 3; config.LeaseDuration = leaseDurationInSec; this.configurationService.UpdateConfiguration(config); string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0]; Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper); // Delay behavior ProxyBehavior[] behaviors = new[] { DelayBehaviors.DelayAllResponsesIf( 1, (session => { var body = session.GetRequestBodyAsString(); // Delay lock of the Head response enough so the txView expires ... // And, upload a new config with a new viewId if (session.hostname.Contains(accountNameToTamper + ".") && session.HTTPMethodIs("PUT") && body.Contains("\"_rtable_Operation\":\"Replace\"") && body.Contains("\"_rtable_RowLock\":true")) { // Upload a new config with a new ViewId View view = this.configurationService.GetTableView(this.repTable.TableName); config.GetView(view.Name).ViewId++; this.configurationService.UpdateConfiguration(config, false); // ensure txView expires Thread.Sleep(2 * leaseDurationInSec * 1000); return(true); } return(false); })), }; // Launch an Update using (new HttpMangler(false, behaviors)) { try { entry = this.rtableWrapper.ReadEntity(entityPartitionKey, entityRowKey); entry.Message = "update message"; var table = new ReplicatedTable(this.repTable.TableName, this.configurationService); TableOperation replaceOperation = TableOperation.Replace(entry); // Get a snapshot of txView before transaction starts ... View viewBeforeTx = this.configurationService.GetTableView(this.repTable.TableName); // Transaction will get delayed 2*LeaseDuration sec. and Config/ViewId will be changed TableResult replaceResult = table.Execute(replaceOperation); Assert.IsTrue(viewBeforeTx.IsExpired(), "txView expected to expire!"); Assert.AreNotEqual(this.configurationService.GetTableView(this.repTable.TableName).ViewId, viewBeforeTx.ViewId, "ViewId should have changed!"); Assert.IsNotNull(replaceResult, "upsertResult = null"); Assert.AreNotEqual((int)HttpStatusCode.NoContent, replaceResult.HttpStatusCode, "upsertResult.HttpStatusCode mismatch"); } catch (ReplicatedTableStaleViewException ex) { // ValidateTxnView is called form everywhere ... we may endup here too switch (ex.ErrorCode) { case ReplicatedTableViewErrorCodes.ViewIdChanged: break; default: Console.WriteLine(ex); Assert.IsTrue(false); break; } } } }
public void RTableRepairTable() { // Insert entity Assert.IsTrue(this.repTable.Exists(), "RTable does not exist"); View fullView = configurationWrapper.GetWriteView(); List <ReplicaInfo> fullViewReplicas = new List <ReplicaInfo>(); for (int i = 0; i <= fullView.TailIndex; i++) { fullViewReplicas.Add(fullView.GetReplicaInfo(i)); } List <ReplicaInfo> newReplicas = new List <ReplicaInfo>(); for (int i = 1; i <= fullView.TailIndex; i++) { newReplicas.Add(fullView.GetReplicaInfo(i)); } SampleRTableEntity customer1 = new SampleRTableEntity("firstName1", "lastName1", "*****@*****.**"); TableOperation operation = TableOperation.Insert(customer1); TableResult result = repTable.Execute(operation); Assert.AreNotEqual(null, result, "result = null"); Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch"); SampleRTableEntity customer2 = new SampleRTableEntity("firstName2", "lastName2", "*****@*****.**"); operation = TableOperation.Insert(customer2); result = repTable.Execute(operation); Assert.AreNotEqual(null, result, "result = null"); Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch"); customer2 = (SampleRTableEntity)result.Result; SampleRTableEntity customer3 = new SampleRTableEntity("firstName3", "lastName3", "*****@*****.**"); operation = TableOperation.Insert(customer3); result = repTable.Execute(operation); Assert.AreNotEqual(null, result, "result = null"); Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch"); ReadFromIndividualAccountsDirectly(customer1.PartitionKey, customer1.RowKey, true); ReadFromIndividualAccountsDirectly(customer2.PartitionKey, customer2.RowKey, true); ReadFromIndividualAccountsDirectly(customer3.PartitionKey, customer3.RowKey, true); // remove replica from the head this.UpdateConfiguration(newReplicas, 0); Assert.IsTrue(configurationWrapper.IsViewStable()); // delete a row TableOperation deleteOperation = TableOperation.Delete(customer1); result = repTable.Execute(deleteOperation); Assert.AreNotEqual(null, result, "result = null"); Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch"); // add a row SampleRTableEntity customer4 = new SampleRTableEntity("firstName4", "lastName4", "*****@*****.**"); operation = TableOperation.Insert(customer4); result = repTable.Execute(operation); Assert.AreNotEqual(null, result, "result = null"); Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch"); // replace a row customer2.Message = "updated after view update"; operation = TableOperation.Replace(customer2); result = repTable.Execute(operation); Assert.AreNotEqual(null, result, "result = null"); Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch"); //Add replica at head this.UpdateConfiguration(fullViewReplicas, 1); // repair table on the new head Console.WriteLine("Calling repair table"); ReconfigurationStatus status = repTable.RepairTable(0, null); Assert.AreEqual(ReconfigurationStatus.SUCCESS, status, "RepairTable status is not success: {0}", status); ReadFromIndividualAccountsDirectly(customer1.PartitionKey, customer1.RowKey, true); ReadFromIndividualAccountsDirectly(customer2.PartitionKey, customer2.RowKey, true); ReadFromIndividualAccountsDirectly(customer3.PartitionKey, customer3.RowKey, true); ReadFromIndividualAccountsDirectly(customer4.PartitionKey, customer4.RowKey, true); }
public void BatchOperationExceptionWhenUsingSmallerViewId() { long currentViewId = 100; long badViewId = currentViewId - 1; this.UpdateConfiguration(replicas, 0, false, currentViewId); string jobType = "jobType-BatchOperationExceptionWhenUsingSmallerViewId"; string jobId = "jobId-BatchOperationExceptionWhenUsingSmallerViewId"; int count = 3; // number of operations in the batch _rtable_Operation List<TableOperationType> opTypes = new List<TableOperationType>() { TableOperationType.Replace, TableOperationType.InsertOrReplace, TableOperationType.Delete, }; // // Insert // string jobIdTemplate = jobId + "-{0}"; string messageTemplate = "message-{0}"; string updatedMessageTemplate = "updated-" + messageTemplate; string partitionKey = string.Empty; // // Insert entities // for (int i = 0; i < count; i++) { SampleRTableEntity originalEntity = new SampleRTableEntity( jobType, string.Format(jobIdTemplate, i), string.Format(messageTemplate, i)); this.repTable.Execute(TableOperation.Insert(originalEntity)); partitionKey = originalEntity.PartitionKey; } // // Retrieve entities and use them to create batchOperation to Replace or Delete // IEnumerable<SampleRTableEntity> allEntities = this.rtableWrapper.GetAllRows(partitionKey); TableBatchOperation batchOperation = new TableBatchOperation(); int m = 0; foreach (SampleRTableEntity entity in allEntities) { Console.WriteLine("{0}", entity.ToString()); Console.WriteLine("---------------------------------------"); if (opTypes[m] == TableOperationType.Replace) { SampleRTableEntity replaceEntity = new SampleRTableEntity( entity.JobType, entity.JobId, string.Format(updatedMessageTemplate, m)) { ETag = entity._rtable_Version.ToString() }; batchOperation.Replace(replaceEntity); } else if (opTypes[m] == TableOperationType.InsertOrReplace) { SampleRTableEntity replaceEntity = new SampleRTableEntity( entity.JobType, entity.JobId, string.Format(updatedMessageTemplate, m)) { ETag = entity._rtable_Version.ToString() }; batchOperation.InsertOrReplace(replaceEntity); } else if (opTypes[m] == TableOperationType.Delete) { entity.ETag = entity._rtable_Version.ToString(); batchOperation.Delete(entity); } else { throw new ArgumentException( string.Format("opType={0} is NOT supported", opTypes[m]), "opType"); } m++; } // // Call ModifyConfigurationBlob to change the viewId of the wrapper to an older value // Console.WriteLine("Changing the viewId to badViewId {0}", badViewId); this.UpdateConfiguration(replicas, 0, false, badViewId); // // Execute Batch _rtable_Operation with bad viewId // Console.WriteLine("\nCalling BatchOperation with badViewId..."); try { this.repTable.ExecuteBatch(batchOperation); } catch (ReplicatedTableStaleViewException ex) { Console.WriteLine("Get this RTableStaleViewException: {0}", ex.Message); Assert.IsTrue(ex.Message.Contains(string.Format("current _rtable_ViewId {0} is smaller than", badViewId)), "Got unexpected exception message"); } }
/// <summary> /// Helper function to insert the specified set of jobType and jobId. It is expected that InsertRow() will fail. /// </summary> /// <param name="jobType"></param> /// <param name="jobId"></param> /// <param name="entityMessage"></param> protected void PerformInsertOperationAndExpectToFail(string jobType, string jobId, string entityMessage) { Console.WriteLine("\nValidating Insert operation will fail. jobType={0} jobId={1}...", jobType, jobId); for (int i = 0; i < this.numberOfPartitions; i++) { List<string> rowKeys = new List<string>(); // list of rowKey for the given paritionKey string partitionKey; string rowKey; SampleRTableEntity.GenerateKeys(this.GenerateJobType(jobType, i), "don't care", out partitionKey, out rowKey); for (int j = 0; j < this.numberOfRowsPerPartition; j++) { SampleRTableEntity sampleRtableEntity = new SampleRTableEntity( this.GenerateJobType(jobType, i), this.GenerateJobId(jobId, i, j), this.GenerateMessage(entityMessage, i, j)); int attempts = 1; bool failed = false; while (attempts < 3) { try { Console.WriteLine("attempts={0}. partitionKey={1} rowKey={2}. Calling InsertRow API...", attempts, partitionKey, sampleRtableEntity.RowKey); this.rtableWrapper.InsertRow(sampleRtableEntity); failed = false; break; // get out of while(attempts) if no RTableConflictException } catch (RTableConflictException) { failed = true; Console.WriteLine("Got RTableConflictException. attempts={0}", attempts); attempts++; System.Threading.Thread.Sleep(1000); } } Assert.IsTrue(failed, "InsertRow() actually passed when we expect it to fail!"); } } Console.WriteLine("Passed 'Insert will fail' validation.\n"); }
public void A00TwoInsertOrReplaceCallsConflictingOnTheHead() { this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable); string entityPartitionKey = "jobType-DelayInsertOrReplaceRowHeadTest"; string entityRowKey = "jobId-DelayInsertOrReplaceRowHeadTest"; this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey); // Insert one entry Console.WriteLine("Inserting entry ..."); var entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, "insert message"); this.rtableWrapper.InsertRow(entry); string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0]; Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper); int delayInMs = 3000; bool firstWritterInitiatingCommit = false; bool secondWritterTriedLockingHead = false; // Delay behavior ProxyBehavior[] behaviors = new[] { // Writter-1 tampering TamperBehaviors.TamperAllRequestsIf( (session => { int iter = 0; // Signal Writter-2 firstWritterInitiatingCommit = true; // Blobk commit to head ... until Writter-2 try to lock the head while (!secondWritterTriedLockingHead) { Console.WriteLine("Writter-1 waiting on Writter-2 to try to lock the Head (#{0})", iter); Thread.Sleep(delayInMs); if (++iter > 10) { break; } } }), (session => { var body = session.GetRequestBodyAsString(); // Writter-1 committing to head if (session.hostname.Contains(accountNameToTamper + ".") && session.HTTPMethodIs("PUT") && body.Contains("\"_rtable_Operation\":\"Replace\"") && body.Contains("\"_rtable_RowLock\":false") && body.Contains("\"Message\":\"upsert message 0\"") ) { return(true); } return(false); })), // Writter-2 tampering TamperBehaviors.TamperAllRequestsIf( (session => { // Block till Writter-1 issues a commit to head while (!firstWritterInitiatingCommit) { Console.WriteLine("Writter-2 waiting on Writter-1 to issue a commit to head"); Thread.Sleep(delayInMs); } }), (session => { var body = session.GetRequestBodyAsString(); // Writter-2 locking the head if (session.hostname.Contains(accountNameToTamper + ".") && session.HTTPMethodIs("PUT") && body.Contains("\"_rtable_Operation\":\"Replace\"") && body.Contains("\"_rtable_RowLock\":true") && body.Contains("\"Message\":\"upsert message 1\"") ) { return(true); } return(false); })), // Delay Writter-2 lock-to-the-head's response, so Writter-1 can continue with its commit. DelayBehaviors.DelayAllResponsesIf( delayInMs, (session => { var body = session.GetRequestBodyAsString(); // Writter-2 locking the head response if (session.hostname.Contains(accountNameToTamper + ".") && session.HTTPMethodIs("PUT") && body.Contains("\"_rtable_Operation\":\"Replace\"") && body.Contains("\"_rtable_RowLock\":true") && body.Contains("\"Message\":\"upsert message 1\"") ) { // Signal Writter-1 so it can continue with commit to head secondWritterTriedLockingHead = true; return(true); } return(false); })), }; // Launch 2 concurrent Upserts var results = new TableResult[2]; using (new HttpMangler(false, behaviors)) { Parallel.For(0, 2, (index) => { entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, string.Format("upsert message {0}", index)); try { var table = new ReplicatedTable(this.repTable.TableName, this.configurationService); TableOperation operation = TableOperation.InsertOrReplace(entry); results[index] = table.Execute(operation); } catch (AggregateException ex) { Console.WriteLine(ex); } }); } // Writter-1 suceed? Assert.AreEqual((int)HttpStatusCode.NoContent, results[0].HttpStatusCode, "Writter-1 expected to suceed!"); // Writter-2 suceeded? Assert.AreEqual((int)HttpStatusCode.NoContent, results[1].HttpStatusCode, "Writter-2 expected to suceed!"); // Writter-2 upsert succeeded SampleRTableEntity upsertedEntity = this.rtableWrapper.ReadEntity(entityPartitionKey, entityRowKey); Assert.NotNull(upsertedEntity, "Writter-2 upsert succeeded"); Assert.AreEqual(upsertedEntity.Message, string.Format("upsert message {0}", 1), "Writter-2 upsert succeeded"); }
/// <summary> /// Helper function to insert the specified set of jobType and jobId and validate the results. /// Make sure there are no existing entries for the jobType and jobId before calling this function. /// </summary> /// <param name="jobType"></param> /// <param name="jobId"></param> /// <param name="entityMessage"></param> protected void PerformInsertOperationAndValidate(string jobType, string jobId, string entityMessage) { Console.WriteLine("\nValidating Insert operation..."); for (int i = 0; i < this.numberOfPartitions; i++) { List<string> rowKeys = new List<string>(); // list of rowKey for the given paritionKey string partitionKey; string rowKey; SampleRTableEntity.GenerateKeys(this.GenerateJobType(jobType, i), "don't care", out partitionKey, out rowKey); // // GetAllRows() to confirm nothing exists and then call InsertRow() // Console.WriteLine("Calling GetAllRows() for partition {0}, expecting 0 rows...", i); IEnumerable<SampleRTableEntity> allRows = this.rtableWrapper.GetAllRows(partitionKey); Assert.AreEqual(0, allRows.Count(), "Partition {0} should have 0 rows in order for InsertRow() to work.", i); for (int j = 0; j < this.numberOfRowsPerPartition; j++) { SampleRTableEntity sampleRtableEntity = new SampleRTableEntity( this.GenerateJobType(jobType, i), this.GenerateJobId(jobId, i, j), this.GenerateMessage(entityMessage, i, j)); int attempts = 1; bool passed = true; while (attempts < 3) { try { Console.WriteLine("attempts={0}. partitionKey={1} rowKey={2}. Calling InsertRow API...", attempts, partitionKey, sampleRtableEntity.RowKey); this.rtableWrapper.InsertRow(sampleRtableEntity); passed = true; break; // get out of while(attempts) if no RTableConflictException } catch (RTableConflictException) { passed = false; Console.WriteLine("Got RTableConflictException. attempts={0}", attempts); attempts++; System.Threading.Thread.Sleep(1000); } } Assert.IsTrue(passed, "Keep getting RTableConflictException when calling InsertRow API"); } // // FindRow() // Console.WriteLine("Calling FindRow() for partitionKey={0}", partitionKey); for (int j = 0; j < rowKeys.Count; j++) { SampleRTableEntity retrievedEntity = this.rtableWrapper.FindRow(partitionKey, rowKeys[j]); this.ValidateRetrievedRTableEntity(retrievedEntity, jobType, jobId, entityMessage, i, j); } } Console.WriteLine("Passed Insert validation.\n"); }
public void RTableRepairTable() { // Insert entity Assert.IsTrue(this.repTable.Exists(), "RTable does not exist"); View fullView = configurationWrapper.GetWriteView(); List<ReplicaInfo> fullViewReplicas = new List<ReplicaInfo>(); for (int i = 0; i <= fullView.TailIndex; i++) { fullViewReplicas.Add(fullView.GetReplicaInfo(i)); } List<ReplicaInfo> newReplicas = new List<ReplicaInfo>(); for (int i = 1; i <= fullView.TailIndex; i++) { newReplicas.Add(fullView.GetReplicaInfo(i)); } SampleRTableEntity customer1 = new SampleRTableEntity("firstName1", "lastName1", "*****@*****.**"); TableOperation operation = TableOperation.Insert(customer1); TableResult result = repTable.Execute(operation); Assert.AreNotEqual(null, result, "result = null"); Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch"); SampleRTableEntity customer2 = new SampleRTableEntity("firstName2", "lastName2", "*****@*****.**"); operation = TableOperation.Insert(customer2); result = repTable.Execute(operation); Assert.AreNotEqual(null, result, "result = null"); Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch"); customer2 = (SampleRTableEntity) result.Result; SampleRTableEntity customer3 = new SampleRTableEntity("firstName3", "lastName3", "*****@*****.**"); operation = TableOperation.Insert(customer3); result = repTable.Execute(operation); Assert.AreNotEqual(null, result, "result = null"); Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch"); ReadFromIndividualAccountsDirectly(customer1.PartitionKey, customer1.RowKey, true); ReadFromIndividualAccountsDirectly(customer2.PartitionKey, customer2.RowKey, true); ReadFromIndividualAccountsDirectly(customer3.PartitionKey, customer3.RowKey, true); // remove replica from the head this.UpdateConfiguration(newReplicas, 0); Assert.IsTrue(configurationWrapper.IsViewStable()); // delete a row TableOperation deleteOperation = TableOperation.Delete(customer1); result = repTable.Execute(deleteOperation); Assert.AreNotEqual(null, result, "result = null"); Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch"); // add a row SampleRTableEntity customer4 = new SampleRTableEntity("firstName4", "lastName4", "*****@*****.**"); operation = TableOperation.Insert(customer4); result = repTable.Execute(operation); Assert.AreNotEqual(null, result, "result = null"); Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch"); // replace a row customer2.Message = "updated after view update"; operation = TableOperation.Replace(customer2); result = repTable.Execute(operation); Assert.AreNotEqual(null, result, "result = null"); Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch"); //Add replica at head this.UpdateConfiguration(fullViewReplicas, 1); // repair table on the new head Console.WriteLine("Calling repair table"); ReconfigurationStatus status = repTable.RepairTable(0, null); Assert.AreEqual(ReconfigurationStatus.SUCCESS, status, "RepairTable status is not success: {0}", status); ReadFromIndividualAccountsDirectly(customer1.PartitionKey, customer1.RowKey, true); ReadFromIndividualAccountsDirectly(customer2.PartitionKey, customer2.RowKey, true); ReadFromIndividualAccountsDirectly(customer3.PartitionKey, customer3.RowKey, true); ReadFromIndividualAccountsDirectly(customer4.PartitionKey, customer4.RowKey, true); }
/// <summary> /// Helper function to execute the specified BatchOperation after HttpMangler is turned off and validate correctness. /// </summary> /// <param name="count">Number of operations in the batch</param> /// <param name="partitionKey">partitionKey to operate on</param> /// <param name="jobType">partitionKey is generated from jobType</param> /// <param name="jobId">RowKey is generated from jobId. JobIdTemplate = jobId-{0}</param> /// <param name="opTypes">Specifies the batch operation to be performed</param> protected void ExecuteBatchOperationAndValidate( int count, string partitionKey, string jobType, string jobId, List<TableOperationType> opTypes) { Console.WriteLine("\nExecuteBatchOperationAndValidate(): Trying to batch update {0} entities...", count); Assert.IsNotNull(opTypes, "opTypes = null"); Assert.AreEqual(count, opTypes.Count, "count and opTypes.Count should be the same"); string jobIdTemplate = jobId + "-{0}"; string replaceMessageTemplate = "updated-after-httpMangler-{0}"; IEnumerable<SampleRTableEntity> allEntities = null; TableBatchOperation batchOperation = null; int m = 0; bool gotExceptionInLastAttempt = true; int retries = 0; while (retries < MaxRetries) { try { // // GetAllRows() // allEntities = this.rtableWrapper.GetAllRows(partitionKey); // // Create a batchOperation to perform the specified opTypes // batchOperation = new TableBatchOperation(); m = 0; foreach (SampleRTableEntity entity in allEntities) { if (opTypes[m] == TableOperationType.Replace) { // set up the new entity to be used in the batch operation SampleRTableEntity replaceEntity = new SampleRTableEntity( entity.JobType, entity.JobId, string.Format(replaceMessageTemplate, m)) { ETag = entity._rtable_Version.ToString() }; // add the operation to the batch operation batchOperation.Replace(replaceEntity); } else if (opTypes[m] == TableOperationType.Delete) { batchOperation.Delete(entity); } else { throw new ArgumentException( string.Format("opType={0} is NOT supported", opTypes[m]), "opType"); } m++; } // // Call this.repTable.ExecuteBatch(batchOperation); // if (batchOperation.Count == 0) { Console.WriteLine("retries={0}. Done. batchOperation.Count == 0", retries); } else { this.repTable.ExecuteBatch(batchOperation); Console.WriteLine("retries={0}. Done ExecuteBatch()", retries); } gotExceptionInLastAttempt = false; break; } catch (RTableConflictException ex) { Console.WriteLine("retries={0}. ExecuteBatch() got an RTableConflictException: {1}", retries, ex.ToString()); retries++; gotExceptionInLastAttempt = true; Thread.Sleep(ConflictExceptionSleepTimeInMsec); } } Console.WriteLine("gotExceptionInLastAttempt={0} retries={1} MaxRetries={2}", gotExceptionInLastAttempt, retries, MaxRetries); Assert.IsFalse(gotExceptionInLastAttempt, "The last API call should not throw an exception."); // // Final validation // Console.WriteLine("Final validation..."); allEntities = this.rtableWrapper.GetAllRows(partitionKey); m = 0; int opTypesCounter = 0; foreach (SampleRTableEntity entity in allEntities) { Console.WriteLine("{0}", entity.ToString()); Console.WriteLine("---------------------------------------"); // If the operation is Delete, then skip it. No need to validate. while (opTypesCounter < count && opTypes[m] == TableOperationType.Delete) { m++; } Assert.IsTrue(m < count, "m={0} count={1}: m shoud be < count, but it is not.", m, count); if (opTypes[m] == TableOperationType.Replace) { Assert.AreEqual(string.Format(jobType, m), entity.JobType, "JobType does not match"); Assert.AreEqual(string.Format(jobIdTemplate, m), entity.JobId, "JobId does not match"); Assert.AreEqual(string.Format(replaceMessageTemplate, m), entity.Message, "Message does not match"); m++; } else { throw new ArgumentException( string.Format("opType={0} is NOT supported", opTypes[opTypesCounter]), "opType"); } } for (int i = 0; i < count; i++) { this.ReadFromIndividualAccountsDirectly( jobType, string.Format(jobIdTemplate, i), true); } Console.WriteLine("Passed final validation."); }