public void A00XStoreThrottleTest() { string entityPartitionKey = "jobType-XStoreThrottleTest"; string entityRowKey = "jobId-XStoreThrottleTest"; this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey); int targetStorageAccount = 0; bool targetApiExpectedToFail = true; Assert.IsTrue(0 <= targetStorageAccount && targetStorageAccount < this.actualStorageAccountsUsed.Count, "targetStorageAccount={0} is out-of-range", targetStorageAccount); int index = this.actualStorageAccountsUsed[targetStorageAccount]; string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[index]; Console.WriteLine("accountNameToTamper={0}", accountNameToTamper); // Throttle behavior ProxyBehavior[] behaviors = new[] { TamperBehaviors.TamperAllRequestsIf( Actions.ThrottleTableRequest, AzureStorageSelectors.TableTraffic().IfHostNameContains(accountNameToTamper)) }; this.SetupAndRunXStoreHttpManglerTest( entityPartitionKey, entityRowKey, targetStorageAccount, targetApiExpectedToFail, behaviors); }
public void RetrieveThrowsBadRequest() { string jobType = "jobType//RTableWrapperCRUDTest"; string jobId = "jobId//RTableWrapperCRUDTest"; int getCallCounts = 0; var manglingBehaviors = new[] { TamperBehaviors.TamperAllRequestsIf( (session) => { getCallCounts++; }, (session) => { if (session.HTTPMethodIs("GET")) { return(true); } return(false); }) }; using (new HttpMangler(false, manglingBehaviors)) { Assert.Throws <StorageException>(() => { try { this.rtableWrapper.ReadEntity(jobType, jobId); } catch (StorageException se) { Assert.IsNotNull(se.InnerException); var webException = se.InnerException as WebException; Assert.IsNotNull(webException); var response = (HttpWebResponse)webException.Response; Assert.IsNotNull(response); Assert.AreEqual(HttpStatusCode.BadRequest, response.StatusCode); throw; } }); } Assert.AreEqual(1, getCallCounts); }
/// <summary> /// GetTableNotFound returns a behavior saying the request failed with 404 (not found) /// </summary> /// <returns>A new behavior.</returns> public static ProxyBehavior GetTableNotFound() { return(TamperBehaviors.TamperAllRequestsIf( session => CreateTableError(session, 404, "ResourceNotFound", "The specified resource does not exist."), Selectors.IfGet().ForTableTraffic())); }
/// <summary> /// EchoTableEntry echos back the contents that were sent to the server. /// </summary> /// <returns>A new behavior.</returns> public static ProxyBehavior EchoTableEntry() { return(TamperBehaviors.TamperAllRequestsIf(EchoEntry, Selectors.IfPost().ForTableTraffic())); }
/// <summary> /// CreateTableErrorTableAlreadyExists returns a behavior saying the request failed with 409 (conflict) and substatus of "already exists" /// </summary> /// <returns>A new behavior.</returns> public static ProxyBehavior CreateTableErrorTableAlreadyExists() { return(TamperBehaviors.TamperAllRequestsIf( session => CreateTableError(session, 409, "TableAlreadyExists", "The table specified already exists."), Selectors.IfPost().ForTableTraffic())); }
/// <summary> /// CreateTableErrorTableBeingDeleted returns a behavior saying the request failed with 409 (conflict) and substatus of "being deleted" /// </summary> /// <returns>A new behavior.</returns> public static ProxyBehavior CreateTableErrorTableBeingDeleted() { return(TamperBehaviors.TamperAllRequestsIf( session => CreateTableError(session, 409, "TableBeingDeleted", "The table specified is in the process of being deleted."), Selectors.IfPost().ForTableTraffic())); }
public void CloudBlockBlobDownloadRangeToStreamAPMRetry() { byte[] buffer = GetRandomBuffer(1 * 1024 * 1024); int offset = 1024; CloudBlobContainer container = GetRandomContainerReference(); try { container.Create(); CloudBlockBlob blob = container.GetBlockBlobReference("blob1"); using (MemoryStream originalBlob = new MemoryStream(buffer)) { using (AutoResetEvent waitHandle = new AutoResetEvent(false)) { ICancellableAsyncResult result = blob.BeginUploadFromStream(originalBlob, ar => waitHandle.Set(), null); waitHandle.WaitOne(); blob.EndUploadFromStream(result); } } using (MemoryStream originalBlob = new MemoryStream()) { originalBlob.Write(buffer, offset, buffer.Length - offset); using (AutoResetEvent waitHandle = new AutoResetEvent(false)) { Exception manglerEx = null; using (HttpMangler proxy = new HttpMangler(false, new[] { TamperBehaviors.TamperNRequestsIf( session => ThreadPool.QueueUserWorkItem(state => { Thread.Sleep(1000); try { session.Abort(); } catch (Exception e) { manglerEx = e; } }), 2, AzureStorageSelectors.BlobTraffic().IfHostNameContains(container.ServiceClient.Credentials.AccountName)) })) { foreach (var options in new[] { new BlobRequestOptions() { ChecksumOptions = new ChecksumOptions { UseTransactionalMD5 = true, UseTransactionalCRC64 = false } }, new BlobRequestOptions() { ChecksumOptions = new ChecksumOptions { UseTransactionalMD5 = false, UseTransactionalCRC64 = true } } }) { using (MemoryStream downloadedBlob = new MemoryStream()) { OperationContext operationContext = new OperationContext(); ICancellableAsyncResult result = blob.BeginDownloadRangeToStream(downloadedBlob, offset, buffer.Length - offset, null, options, operationContext, ar => waitHandle.Set(), null); waitHandle.WaitOne(); blob.EndDownloadToStream(result); TestHelper.AssertStreamsAreEqual(originalBlob, downloadedBlob); if (manglerEx != null) { throw manglerEx; } } } } } } } finally { container.DeleteIfExists(); } }
/// <summary> /// CreateTableOk returns a behavior which causes all GET requests for table creation traffic to return successfully. /// </summary> /// <returns>A new behavior.</returns> public static ProxyBehavior GetTableOk() { return(TamperBehaviors.TamperAllRequestsIf(session => GetTableWithCode(session, 200), Selectors.IfGet().ForTableTraffic())); }
public void WhenReadFromTailFailsWithServiceUnavailableWeReadFromHeadAndSucceed() { View view = this.configurationWrapper.GetWriteView(); Assert.IsTrue(view.Chain.Count > 1, "expects at least 2 replicas!"); // Insert one entry var rtable = new ReplicatedTable(this.repTable.TableName, this.configurationService); string firstName = "FirstName01"; string lastName = "LastName01"; var customer = new CustomerEntity(firstName, lastName); TableOperation operation = TableOperation.Insert(customer); rtable.Execute(operation); // Using xstore modify the row in each replica individually ... so we know, later, which replica RTable will retrieve from for (int replicaIndex = 0; replicaIndex < this.cloudTableClients.Count; replicaIndex++) { CloudTableClient tableClient = this.cloudTableClients[replicaIndex]; CloudTable table = tableClient.GetTableReference(this.repTable.TableName); TableOperation retrieveOperation = TableOperation.Retrieve <CustomerEntity>(firstName, lastName); TableResult retrieveResult = table.Execute(retrieveOperation); customer = (CustomerEntity)retrieveResult.Result; customer.Email = replicaIndex.ToString(); // Head = 0 // ... = 1 // Tail = 2 TableOperation updateOperation = TableOperation.Replace(customer); TableResult updateResult = table.Execute(updateOperation); Assert.IsNotNull(updateResult, "updateResult = null"); Console.WriteLine("updateResult.HttpStatusCode = {0}", updateResult.HttpStatusCode); Assert.AreEqual((int)HttpStatusCode.NoContent, updateResult.HttpStatusCode, "updateResult.HttpStatusCode mismatch"); } string accountNameToNotTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0]; Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToNotTamper={0}", accountNameToNotTamper); // Delay behavior ProxyBehavior[] behaviors = { TamperBehaviors.TamperAllRequestsIf( (session => { session.oRequest.FailSession((int)HttpStatusCode.ServiceUnavailable, "ServerBusy", ""); }), (session => { if (session.HTTPMethodIs("Get")) { // Fail Get from all replicas, except Head if (!session.hostname.Contains(accountNameToNotTamper + ".")) { return(true); } } return(false); })), }; using (new HttpMangler(false, behaviors)) { operation = TableOperation.Retrieve <CustomerEntity>(firstName, lastName); TableResult retrievedResult = repTable.Execute(operation); Assert.AreNotEqual(null, retrievedResult, "retrievedResult = null"); Assert.AreEqual((int)HttpStatusCode.OK, retrievedResult.HttpStatusCode, "retrievedResult.HttpStatusCode mismatch"); Assert.AreNotEqual(null, retrievedResult.Result, "retrievedResult.Result = null"); customer = (CustomerEntity)retrievedResult.Result; Assert.AreEqual(customer.Email, (0).ToString(), "we should have read the row from Head"); } }
public void ReadingUncommittedDataFromReplicaIdentifiedByReadTailIndexWontThrow() { ReplicatedTableConfigurationServiceV2 configServiceOne, configServiceTwo; // setup: Acc0 Acc1 // stale view = [H] -> [T] // new view = [H] SetupStaleViewAndNewView(out configServiceOne, out configServiceTwo); long staleViewId = configServiceOne.GetTableView(this.repTable.TableName).ViewId; long latestViewId = configServiceTwo.GetTableView(this.repTable.TableName).ViewId; string firstName = "FirstName01"; string lastName = "LastName01"; /* * 1 - WorkerOne => inserts an entry in stale View */ var workerOne = new ReplicatedTable(this.repTable.TableName, configServiceOne); var workerTwo = new ReplicatedTable(this.repTable.TableName, configServiceTwo); var customer = new CustomerEntity(firstName, lastName); customer.Email = "***"; InsertCustormer(customer, workerOne); string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[1]; Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper); TableResult oldUpdateResult = null; CustomerEntity customerSeenByNewView = null; // Delay behavior ProxyBehavior[] behaviors = { TamperBehaviors.TamperAllRequestsIf( (session => { session.oRequest.FailSession((int)HttpStatusCode.ServiceUnavailable, "ServerBusy", ""); }), (session => { var body = session.GetRequestBodyAsString(); // Fail Lock to Tail by stale view if (session.hostname.Contains(accountNameToTamper + ".") && session.HTTPMethodIs("PUT") && body.Contains("\"Email\":\"workerOne\"") && body.Contains(string.Format("\"_rtable_ViewId\":\"{0}\"", staleViewId)) && body.Contains("\"_rtable_RowLock\":true")) { return(true); } return(false); })), }; /* * 2 - WorkerOne => update an entry using the stale View */ using (new HttpMangler(false, behaviors)) { customer = RetrieveCustomer(firstName, lastName, workerOne); customer.Email = "workerOne"; TableOperation operation = TableOperation.Replace(customer); oldUpdateResult = workerOne.Execute(operation); } // Expected behavior: // Thread_1 (stale ViewId) fails to commit to Tail Assert.IsNotNull(oldUpdateResult, "oldUpdateResult = null"); Assert.AreEqual((int)HttpStatusCode.ServiceUnavailable, oldUpdateResult.HttpStatusCode, "oldUpdateResult.HttpStatusCode mismatch"); Console.WriteLine("Update in stale View Succeeded with HttpStatus={0}", oldUpdateResult.HttpStatusCode); // Thread_1 (stale ViewId): Reads the entry => Succeeds eventhough the row is "uncommitted". // "ReadTailIndex = [Head]" means [Head] is treated as [Tail] for Reads. // => row is assumed "committed" customer = RetrieveCustomer(firstName, lastName, workerOne); Assert.IsNotNull(customer, "customer = null"); Assert.AreEqual("workerOne", customer.Email, "customer.Email mismatch"); // Thread_2 (new ViewId): Reads the entry => Succeed since reading from Head customerSeenByNewView = RetrieveCustomer(firstName, lastName, workerTwo); Assert.IsNotNull(customerSeenByNewView, "customerSeenByNewView = null"); Assert.AreEqual("workerOne", customer.Email, "customer.Email mismatch"); }
public void LinqQueriesAreServedFromHeadWhenReadViewTailIndexIsSet() { var rtable = new ReplicatedTable(this.repTable.TableName, this.configurationService); string firstName = "FirstName"; string lastName = "LastName"; int dataSize = 5; /* * 1 - insert entries */ for (int i = 0; i < dataSize; i++) { var customer = new CustomerEntity(firstName + i, lastName + i); customer.Email = "***"; TableOperation operation = TableOperation.Insert(customer); rtable.Execute(operation); } // Identify the Tail account View view = this.configurationWrapper.GetWriteView(); Assert.IsTrue(view.Chain.Count > 1, "expects at least one replica!"); string accountNameToTamper = view.Chain.Last().Item1.StorageAccountName; Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper); // We will fail requests to update a row in Tail ProxyBehavior[] behaviors = { TamperBehaviors.TamperAllRequestsIf( (session => { session.oRequest.FailSession((int)HttpStatusCode.ServiceUnavailable, "ServerBusy", ""); }), (session => { var body = session.GetRequestBodyAsString(); // Fail Lock to Tail by stale view if (session.hostname.Contains(accountNameToTamper + ".") && session.HTTPMethodIs("PUT") && body.Contains("\"_rtable_RowLock\":true")) { return(true); } return(false); })), }; using (new HttpMangler(false, behaviors)) { /* * 2 - update entries */ for (int i = 0; i < dataSize; i++) { TableOperation operation = TableOperation.Retrieve <CustomerEntity>(firstName + i, lastName + i); TableResult retrievedResult = rtable.Execute(operation); var customer = (CustomerEntity)retrievedResult.Result; customer.Email = "updated"; TableOperation replaceOperation = TableOperation.Replace(customer); TableResult replaceResult = rtable.Execute(replaceOperation); Assert.IsNotNull(replaceResult, "replaceResult = null"); Assert.AreEqual((int)HttpStatusCode.ServiceUnavailable, replaceResult.HttpStatusCode, "replaceResult.HttpStatusCode mismatch"); } } SetReadViewTailIndex(0); Assert.AreEqual(0, this.configurationService.GetTableView(this.repTable.TableName).ReadTailIndex, "ReadTailIndex should be 0!!!"); /* * 3 - CreateQuery is served from [Head], data should be new */ foreach (var customer in rtable.CreateReplicatedQuery <CustomerEntity>().AsEnumerable()) { Assert.AreEqual(customer.Email, "updated", "expected new data"); } /* * 4 - ExecuteQuery is served from [Head], data should be new */ foreach (var customer in rtable.ExecuteQuery <CustomerEntity>(new TableQuery <CustomerEntity>())) { Assert.AreEqual(customer.Email, "updated", "expected new data"); } }
public void InsertInStaleViewConflictingWithInsertInNewView() { ReplicatedTableConfigurationServiceV2 configServiceOne, configServiceTwo; // setup: Acc0 Acc1 // stale view = [None] -> [RW] // new view = [WO] -> [RW] SetupStaleViewAndNewView(out configServiceOne, out configServiceTwo); long staleViewId = configServiceOne.GetTableView(this.repTable.TableName).ViewId; long latestViewId = configServiceTwo.GetTableView(this.repTable.TableName).ViewId; string firstName = "FirstName01"; string lastName = "LastName01"; var workerOne = new ReplicatedTable(this.repTable.TableName, configServiceOne); var workerTwo = new ReplicatedTable(this.repTable.TableName, configServiceTwo); string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[1]; Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper); TableResult oldInsertResult = null; TableResult newInsertResult = null; bool triggerInsertWithNewView = false; bool oldInsertResume = false; // Start new newInsertTask in wait var newInsertTask = Task.Run(() => { while (!triggerInsertWithNewView) { Thread.Sleep(100); } /* * 2 - Executes after step 1 below: * WorkerTwo => Insert a new row using new view */ var customer = new CustomerEntity(firstName, lastName); customer.Email = "workerTwo"; TableOperation operation = TableOperation.Insert(customer); newInsertResult = workerTwo.Execute(operation); // Signal old Insert to resume oldInsertResume = true; }); // Delay behavior ProxyBehavior[] behaviors = { TamperBehaviors.TamperAllRequestsIf( (session => { // => trigger new insert to start triggerInsertWithNewView = true; // Delaying commit to the Tail by stale view Update while (!oldInsertResume) { Thread.Sleep(100); } }), (session => { var body = session.GetRequestBodyAsString(); // Commit to Tail by stale view if (session.hostname.Contains(accountNameToTamper + ".") && session.HTTPMethodIs("PUT") && body.Contains("\"Email\":\"workerOne\"") && body.Contains(string.Format("\"_rtable_ViewId\":\"{0}\"", staleViewId)) && body.Contains("\"_rtable_RowLock\":false")) { return(true); } return(false); })), }; /* * 1 - WorkerOne => update an entry using the stale View */ using (new HttpMangler(false, behaviors)) { var customer = new CustomerEntity(firstName, lastName); customer.Email = "workerOne"; TableOperation operation = TableOperation.Insert(customer); oldInsertResult = workerOne.Execute(operation); } // Wait on new Insert to finish newInsertTask.Wait(); // Expected behavior: // Thread_1 (stale ViewId) Assert.IsNotNull(oldInsertResult, "oldInsertResult = null"); Assert.AreEqual((int)HttpStatusCode.NoContent, oldInsertResult.HttpStatusCode, "oldInsertResult.HttpStatusCode mismatch"); Console.WriteLine("Insert in stale View Succeeded with HttpStatus={0}", oldInsertResult.HttpStatusCode); // Thread_2 (new ViewId) Assert.IsNotNull(newInsertResult, "newInsertResult = null"); Assert.AreEqual((int)HttpStatusCode.Conflict, newInsertResult.HttpStatusCode, "newUpdateResult.HttpStatusCode mismatch"); Console.WriteLine("Insert in new View failed with HttpStatus={0}", newInsertResult.HttpStatusCode); var currCustomer = RetrieveCustomer(firstName, lastName, workerTwo); Assert.AreEqual(staleViewId, currCustomer._rtable_ViewId, "customer._rtable_ViewId mismatch"); Assert.AreEqual("workerOne", currCustomer.Email, "customer.Email mismatch"); Console.WriteLine("workerOne write Succeeded"); Console.WriteLine("workerTwo got Conflict because row was lock."); }
public void LinqQueriesDontReturnEntriesWithTombstone() { var rtable = new ReplicatedTable(this.repTable.TableName, this.configurationService); string firstName = "FirstName"; string lastName = "LastName"; /* * 1 - insert entries */ for (int i = 0; i < 10; i++) { var customer = new CustomerEntity(firstName + i, lastName + i); TableOperation operation = TableOperation.Insert(customer); rtable.Execute(operation); } // Identify the Tail account View view = this.configurationWrapper.GetWriteView(); Assert.IsTrue(view.Chain.Count > 1, "expects at least one replica!"); string accountNameToTamper = view.Chain.Last().Item1.StorageAccountName; Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper); // We will fail requests to delete a row in Tail ProxyBehavior[] behaviors = { TamperBehaviors.TamperAllRequestsIf( (session => { session.oRequest.FailSession((int)HttpStatusCode.ServiceUnavailable, "ServerBusy", ""); }), (session => { // Commit to Tail i.e. physically delete the row if (session.hostname.Contains(accountNameToTamper + ".") && session.HTTPMethodIs("DELETE")) { return(true); } return(false); })), }; using (new HttpMangler(false, behaviors)) { /* * 2 - delete entries #2 and #4 */ foreach (var i in new int[] { 2, 4 }) { TableOperation operation = TableOperation.Retrieve <CustomerEntity>(firstName + i, lastName + i); TableResult retrievedResult = rtable.Execute(operation); var customer = (CustomerEntity)retrievedResult.Result; TableOperation deleteOperation = TableOperation.Delete(customer); TableResult deleteResult = rtable.Execute(deleteOperation); Assert.IsNotNull(deleteResult, "deleteResult = null"); Assert.AreEqual((int)HttpStatusCode.ServiceUnavailable, deleteResult.HttpStatusCode, "deleteResult.HttpStatusCode mismatch"); } } // Verify, Retrieve doesn't return entries #2 and #4 int deleteId = 2; var result = rtable.Execute(TableOperation.Retrieve <CustomerEntity>(firstName + deleteId, lastName + deleteId)); Assert.IsNotNull(result, "result = null"); Assert.AreEqual((int)HttpStatusCode.NotFound, result.HttpStatusCode, "result.HttpStatusCode mismatch"); deleteId = 4; result = rtable.Execute(TableOperation.Retrieve <CustomerEntity>(firstName + deleteId, lastName + deleteId)); Assert.IsNotNull(result, "result = null"); Assert.AreEqual((int)HttpStatusCode.NotFound, result.HttpStatusCode, "result.HttpStatusCode mismatch"); /* * 3 - CreateQuery doesn't return entries #2 and #4 */ foreach (var customer in rtable.CreateReplicatedQuery <CustomerEntity>().AsEnumerable()) { int id = int.Parse(customer.PartitionKey.Replace(firstName, "")); Assert.AreNotEqual(id, 2, "entry #2 should have been deleted"); Assert.AreNotEqual(id, 4, "entry #4 should have been deleted"); } /* * 4 - ExecuteQuery doesn't return entries #2 and #4 */ foreach (var customer in rtable.ExecuteQuery <CustomerEntity>(new TableQuery <CustomerEntity>())) { int id = int.Parse(customer.PartitionKey.Replace(firstName, "")); Assert.AreNotEqual(id, 2, "entry #2 should have been deleted"); Assert.AreNotEqual(id, 4, "entry #4 should have been deleted"); } }
public void RepairMidAnUpdate() { this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable); // Not-Stable chain // Reconfigure RTable so Head is WriteOnly. View view = this.configurationService.GetTableView(this.repTable.TableName); ReplicatedTableConfiguration config; ReplicatedTableQuorumReadResult readStatus = this.configurationService.RetrieveConfiguration(out config); Assert.IsTrue(readStatus.Code == ReplicatedTableQuorumReadCode.Success); // Set Head as WriteOnly mode ReplicatedTableConfigurationStore viewConfg = config.GetView(view.Name); viewConfg.ReplicaChain[0].Status = ReplicaStatus.WriteOnly; config.SetView(view.Name, viewConfg); // Upload RTable config back this.configurationService.UpdateConfiguration(config); // Sanity: Replicated mode and chain Not-Stable view = this.configurationService.GetTableView(this.repTable.TableName); Assert.IsTrue(view != null && view.Chain.Count > 1, "Two replicas should be used."); Assert.IsFalse(view.IsStable); // Insert one entry Console.WriteLine("Inserting entry ..."); string entityPartitionKey = "jobType-RepairMidAnUpdate-Replace"; string entityRowKey = "jobId-RepairMidAnUpdate-Replace"; var entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, "message"); this.rtableWrapper.InsertRow(entry); // 1 - Launch a RepairRow task in wait mode ... bool triggerRepair = false; bool repaireDone = false; bool headAfterRepairWasLocked = false; TableResult repairResult = null; Task.Run(() => { ReplicatedTable repairTable = new ReplicatedTable(this.repTable.TableName, this.configurationService); while (!triggerRepair) { Thread.Sleep(5); } Console.WriteLine("RepairRow started ..."); repairResult = repairTable.RepairRow(entry.PartitionKey, entry.RowKey, null); Console.WriteLine("RepairRow completed with HttpStatus={0}", repairResult == null ? "NULL" : repairResult.HttpStatusCode.ToString()); // Check the entry at the Head is still locked i.e. RepairRow was NOP headAfterRepairWasLocked = HeadIsLocked(entry); Console.WriteLine("Signal the commit to Head job"); repaireDone = true; }); // 2 - Configure Mangler ... string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0]; Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper); ProxyBehavior[] behaviors = new[] { TamperBehaviors.TamperAllRequestsIf( (session => { Console.WriteLine("Delaying commit to the Head ... => signal RepairRow job"); // Let RepairRow task go through triggerRepair = true; int iter = 0; while (!repaireDone) { // TODO: break the loop after couple of iteration ... Thread.Sleep(100); Console.WriteLine("Waiting on RepairRow to finish ({0}) ...", ++iter); } Console.WriteLine("Request a commit to the head"); }), (session => { // Commit on head i.e. a PUT with RowLock == false if (session.hostname.Contains(accountNameToTamper + ".") && session.HTTPMethodIs("PUT") && session.GetRequestBodyAsString().Contains("\"_rtable_RowLock\":false")) { return(true); } return(false); })) }; using (new HttpMangler(false, behaviors)) { Console.WriteLine("Updating entry ..."); entry = this.rtableWrapper.FindRow(entry.PartitionKey, entry.RowKey); entry.Message = "updated message"; this.rtableWrapper.ReplaceRow(entry); } Assert.IsTrue(triggerRepair); Assert.IsTrue(repairResult != null && repairResult.HttpStatusCode == (int)HttpStatusCode.OK, "Repair failed."); Assert.IsTrue(repaireDone); Assert.IsTrue(headAfterRepairWasLocked); Console.WriteLine("DONE. Test passed."); }
public void A00TwoInsertOrReplaceCallsConflictingOnTheHead() { this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable); string entityPartitionKey = "jobType-DelayInsertOrReplaceRowHeadTest"; string entityRowKey = "jobId-DelayInsertOrReplaceRowHeadTest"; this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey); // Insert one entry Console.WriteLine("Inserting entry ..."); var entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, "insert message"); this.rtableWrapper.InsertRow(entry); string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0]; Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper); int delayInMs = 3000; bool firstWritterInitiatingCommit = false; bool secondWritterTriedLockingHead = false; // Delay behavior ProxyBehavior[] behaviors = new[] { // Writter-1 tampering TamperBehaviors.TamperAllRequestsIf( (session => { int iter = 0; // Signal Writter-2 firstWritterInitiatingCommit = true; // Blobk commit to head ... until Writter-2 try to lock the head while (!secondWritterTriedLockingHead) { Console.WriteLine("Writter-1 waiting on Writter-2 to try to lock the Head (#{0})", iter); Thread.Sleep(delayInMs); if (++iter > 10) { break; } } }), (session => { var body = session.GetRequestBodyAsString(); // Writter-1 committing to head if (session.hostname.Contains(accountNameToTamper + ".") && session.HTTPMethodIs("PUT") && body.Contains("\"_rtable_Operation\":\"Replace\"") && body.Contains("\"_rtable_RowLock\":false") && body.Contains("\"Message\":\"upsert message 0\"") ) { return(true); } return(false); })), // Writter-2 tampering TamperBehaviors.TamperAllRequestsIf( (session => { // Block till Writter-1 issues a commit to head while (!firstWritterInitiatingCommit) { Console.WriteLine("Writter-2 waiting on Writter-1 to issue a commit to head"); Thread.Sleep(delayInMs); } }), (session => { var body = session.GetRequestBodyAsString(); // Writter-2 locking the head if (session.hostname.Contains(accountNameToTamper + ".") && session.HTTPMethodIs("PUT") && body.Contains("\"_rtable_Operation\":\"Replace\"") && body.Contains("\"_rtable_RowLock\":true") && body.Contains("\"Message\":\"upsert message 1\"") ) { return(true); } return(false); })), // Delay Writter-2 lock-to-the-head's response, so Writter-1 can continue with its commit. DelayBehaviors.DelayAllResponsesIf( delayInMs, (session => { var body = session.GetRequestBodyAsString(); // Writter-2 locking the head response if (session.hostname.Contains(accountNameToTamper + ".") && session.HTTPMethodIs("PUT") && body.Contains("\"_rtable_Operation\":\"Replace\"") && body.Contains("\"_rtable_RowLock\":true") && body.Contains("\"Message\":\"upsert message 1\"") ) { // Signal Writter-1 so it can continue with commit to head secondWritterTriedLockingHead = true; return(true); } return(false); })), }; // Launch 2 concurrent Upserts var results = new TableResult[2]; using (new HttpMangler(false, behaviors)) { Parallel.For(0, 2, (index) => { entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, string.Format("upsert message {0}", index)); try { var table = new ReplicatedTable(this.repTable.TableName, this.configurationService); TableOperation operation = TableOperation.InsertOrReplace(entry); results[index] = table.Execute(operation); } catch (AggregateException ex) { Console.WriteLine(ex); } }); } // Writter-1 suceed? Assert.AreEqual((int)HttpStatusCode.NoContent, results[0].HttpStatusCode, "Writter-1 expected to suceed!"); // Writter-2 suceeded? Assert.AreEqual((int)HttpStatusCode.NoContent, results[1].HttpStatusCode, "Writter-2 expected to suceed!"); // Writter-2 upsert succeeded SampleRTableEntity upsertedEntity = this.rtableWrapper.ReadEntity(entityPartitionKey, entityRowKey); Assert.NotNull(upsertedEntity, "Writter-2 upsert succeeded"); Assert.AreEqual(upsertedEntity.Message, string.Format("upsert message {0}", 1), "Writter-2 upsert succeeded"); }
public void A00DelayTwoConflictingInsertOrReplaceCalls() { this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable); string entityPartitionKey = "jobType-DelayInsertOrReplaceRowHeadTest"; string entityRowKey = "jobId-DelayInsertOrReplaceRowHeadTest"; this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey); string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0]; Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper); int delayInMs = 3000; int insertRequestCount = 0; int conflictResponseCount = 0; bool secondUpsertConflicted = false; int failedCallIndex = -1; // Delay behavior ProxyBehavior[] behaviors = new[] { // Delay Insert calls so they end up conflicting TamperBehaviors.TamperAllRequestsIf( (session => { Interlocked.Increment(ref insertRequestCount); while (insertRequestCount != 2) { Console.WriteLine("insertRequestCount={0}. Waiting on count to reach 2 ...", insertRequestCount); Thread.Sleep(delayInMs); } }), (session => { if (session.hostname.Contains(accountNameToTamper + ".") && session.HTTPMethodIs("POST") && session.GetRequestBodyAsString().Contains("\"_rtable_Operation\":\"Insert\"")) { return(true); } return(false); })), // Delay conflict response DelayBehaviors.DelayAllResponsesIf( delayInMs, (session => { if (session.hostname.Contains(accountNameToTamper + ".") && session.GetRequestBodyAsString().Contains("\"_rtable_Operation\":\"Insert\"")) { if (session.responseCode == (int)HttpStatusCode.Conflict) { Interlocked.Increment(ref conflictResponseCount); return(true); } } return(false); })), }; using (new HttpMangler(false, behaviors)) { Parallel.For(0, 2, (index) => { var entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, string.Format("upsert message {0}", index)); try { this.rtableWrapper.InsertOrReplaceRow(entry); } catch (RTableConflictException) { if (secondUpsertConflicted) { // should never reach here throw; } // That's possible, but that's the Replace step of upsert which conflicted with ongoing write // can't do anything, client should retry on conflict secondUpsertConflicted = true; } }); } // got 2 inserts? Assert.AreEqual(2, insertRequestCount, "Two insert calls expected!"); // got one conflict? Assert.AreEqual(1, conflictResponseCount, "One conflict response expected!"); // at least one upsert would have succeeded SampleRTableEntity upsertedEntity = this.rtableWrapper.ReadEntity(entityPartitionKey, entityRowKey); Assert.NotNull(upsertedEntity, "at least one upsert should have succeeded"); // second upsert failed? if (secondUpsertConflicted) { Assert.AreEqual(1, upsertedEntity._rtable_Version, "one upsert succeeded so version should be = 1"); Assert.AreEqual(upsertedEntity.Message, string.Format("upsert message {0}", (1 - failedCallIndex))); } else { Assert.AreEqual(2, upsertedEntity._rtable_Version, "both upserts succeeded so version should be = 2"); } // After recovery from delay, confirm that we can update the row. //this.ExecuteReplaceRowAndValidate(entityPartitionKey, entityRowKey); }
public void CloudFileDownloadToStreamAPMRetry() { byte[] buffer = GetRandomBuffer(1 * 1024 * 1024); CloudFileShare share = GetRandomShareReference(); try { share.Create(); CloudFile file = share.GetRootDirectoryReference().GetFileReference("file1"); using (MemoryStream originalFile = new MemoryStream(buffer)) { using (AutoResetEvent waitHandle = new AutoResetEvent(false)) { ICancellableAsyncResult result = file.BeginUploadFromStream(originalFile, ar => waitHandle.Set(), null); waitHandle.WaitOne(); file.EndUploadFromStream(result); using (MemoryStream downloadedFile = new MemoryStream()) { Exception manglerEx = null; using (HttpMangler proxy = new HttpMangler(false, new[] { TamperBehaviors.TamperNRequestsIf( session => ThreadPool.QueueUserWorkItem(state => { Thread.Sleep(1000); try { session.Abort(); } catch (Exception e) { manglerEx = e; } }), 2, AzureStorageSelectors.FileTraffic().IfHostNameContains(share.ServiceClient.Credentials.AccountName)) })) { OperationContext operationContext = new OperationContext(); result = file.BeginDownloadToStream(downloadedFile, null, null, operationContext, ar => waitHandle.Set(), null); waitHandle.WaitOne(); file.EndDownloadToStream(result); TestHelper.AssertStreamsAreEqual(originalFile, downloadedFile); } if (manglerEx != null) { throw manglerEx; } } } } } finally { share.DeleteIfExists(); } }