Ejemplo n.º 1
0
        public void A00XStoreThrottleTest()
        {
            string entityPartitionKey = "jobType-XStoreThrottleTest";
            string entityRowKey       = "jobId-XStoreThrottleTest";

            this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);

            int  targetStorageAccount    = 0;
            bool targetApiExpectedToFail = true;

            Assert.IsTrue(0 <= targetStorageAccount && targetStorageAccount < this.actualStorageAccountsUsed.Count,
                          "targetStorageAccount={0} is out-of-range", targetStorageAccount);
            int    index = this.actualStorageAccountsUsed[targetStorageAccount];
            string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[index];

            Console.WriteLine("accountNameToTamper={0}", accountNameToTamper);

            // Throttle behavior
            ProxyBehavior[] behaviors = new[]
            {
                TamperBehaviors.TamperAllRequestsIf(
                    Actions.ThrottleTableRequest,
                    AzureStorageSelectors.TableTraffic().IfHostNameContains(accountNameToTamper))
            };

            this.SetupAndRunXStoreHttpManglerTest(
                entityPartitionKey,
                entityRowKey,
                targetStorageAccount,
                targetApiExpectedToFail,
                behaviors);
        }
Ejemplo n.º 2
0
        public void RetrieveThrowsBadRequest()
        {
            string jobType       = "jobType//RTableWrapperCRUDTest";
            string jobId         = "jobId//RTableWrapperCRUDTest";
            int    getCallCounts = 0;

            var manglingBehaviors = new[]
            {
                TamperBehaviors.TamperAllRequestsIf(
                    (session) =>
                {
                    getCallCounts++;
                },
                    (session) =>
                {
                    if (session.HTTPMethodIs("GET"))
                    {
                        return(true);
                    }
                    return(false);
                })
            };

            using (new HttpMangler(false, manglingBehaviors))
            {
                Assert.Throws <StorageException>(() =>
                {
                    try
                    {
                        this.rtableWrapper.ReadEntity(jobType, jobId);
                    }
                    catch (StorageException se)
                    {
                        Assert.IsNotNull(se.InnerException);
                        var webException = se.InnerException as WebException;
                        Assert.IsNotNull(webException);
                        var response = (HttpWebResponse)webException.Response;
                        Assert.IsNotNull(response);
                        Assert.AreEqual(HttpStatusCode.BadRequest, response.StatusCode);
                        throw;
                    }
                });
            }

            Assert.AreEqual(1, getCallCounts);
        }
Ejemplo n.º 3
0
 /// <summary>
 /// EchoTableEntry echos back the contents that were sent to the server.
 /// </summary>
 /// <returns>A new behavior.</returns>
 public static ProxyBehavior EchoTableEntry()
 {
     return(TamperBehaviors.TamperAllRequestsIf(EchoEntry, Selectors.IfPost().ForTableTraffic()));
 }
Ejemplo n.º 4
0
 /// <summary>
 /// GetTableNotFound returns a behavior saying the request failed with 404 (not found)
 /// </summary>
 /// <returns>A new behavior.</returns>
 public static ProxyBehavior GetTableNotFound()
 {
     return(TamperBehaviors.TamperAllRequestsIf(
                session => CreateTableError(session, 404, "ResourceNotFound", "The specified resource does not exist."),
                Selectors.IfGet().ForTableTraffic()));
 }
Ejemplo n.º 5
0
 /// <summary>
 /// CreateTableErrorTableBeingDeleted returns a behavior saying the request failed with 409 (conflict) and substatus of "being deleted"
 /// </summary>
 /// <returns>A new behavior.</returns>
 public static ProxyBehavior CreateTableErrorTableBeingDeleted()
 {
     return(TamperBehaviors.TamperAllRequestsIf(
                session => CreateTableError(session, 409, "TableBeingDeleted", "The table specified is in the process of being deleted."),
                Selectors.IfPost().ForTableTraffic()));
 }
Ejemplo n.º 6
0
 /// <summary>
 /// CreateTableErrorTableAlreadyExists returns a behavior saying the request failed with 409 (conflict) and substatus of "already exists"
 /// </summary>
 /// <returns>A new behavior.</returns>
 public static ProxyBehavior CreateTableErrorTableAlreadyExists()
 {
     return(TamperBehaviors.TamperAllRequestsIf(
                session => CreateTableError(session, 409, "TableAlreadyExists", "The table specified already exists."),
                Selectors.IfPost().ForTableTraffic()));
 }
Ejemplo n.º 7
0
 /// <summary>
 /// CreateTableOk returns a behavior which causes all GET requests for table creation traffic to return successfully.
 /// </summary>
 /// <returns>A new behavior.</returns>
 public static ProxyBehavior GetTableOk()
 {
     return(TamperBehaviors.TamperAllRequestsIf(session => GetTableWithCode(session, 200), Selectors.IfGet().ForTableTraffic()));
 }
Ejemplo n.º 8
0
        public void ReadingUncommittedDataFromReplicaIdentifiedByReadTailIndexWontThrow()
        {
            ReplicatedTableConfigurationServiceV2 configServiceOne, configServiceTwo;

            // setup:        Acc0    Acc1
            //  stale view = [H] ->  [T]
            //  new view   = [H]
            SetupStaleViewAndNewView(out configServiceOne, out configServiceTwo);

            long staleViewId  = configServiceOne.GetTableView(this.repTable.TableName).ViewId;
            long latestViewId = configServiceTwo.GetTableView(this.repTable.TableName).ViewId;

            string firstName = "FirstName01";
            string lastName  = "LastName01";

            /*
             * 1 - WorkerOne => inserts an entry in stale View
             */
            var workerOne = new ReplicatedTable(this.repTable.TableName, configServiceOne);
            var workerTwo = new ReplicatedTable(this.repTable.TableName, configServiceTwo);

            var customer = new CustomerEntity(firstName, lastName);

            customer.Email = "***";
            InsertCustormer(customer, workerOne);

            string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[1];

            Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper);

            TableResult    oldUpdateResult       = null;
            CustomerEntity customerSeenByNewView = null;

            // Delay behavior
            ProxyBehavior[] behaviors =
            {
                TamperBehaviors.TamperAllRequestsIf(
                    (session =>
                {
                    session.oRequest.FailSession((int)HttpStatusCode.ServiceUnavailable, "ServerBusy", "");
                }),
                    (session =>
                {
                    var body = session.GetRequestBodyAsString();

                    // Fail Lock to Tail by stale view
                    if (session.hostname.Contains(accountNameToTamper + ".") &&
                        session.HTTPMethodIs("PUT") &&
                        body.Contains("\"Email\":\"workerOne\"") &&
                        body.Contains(string.Format("\"_rtable_ViewId\":\"{0}\"", staleViewId)) &&
                        body.Contains("\"_rtable_RowLock\":true"))
                    {
                        return(true);
                    }

                    return(false);
                })),
            };

            /*
             * 2 - WorkerOne => update an entry using the stale View
             */
            using (new HttpMangler(false, behaviors))
            {
                customer       = RetrieveCustomer(firstName, lastName, workerOne);
                customer.Email = "workerOne";

                TableOperation operation = TableOperation.Replace(customer);
                oldUpdateResult = workerOne.Execute(operation);
            }


            // Expected behavior:

            // Thread_1 (stale ViewId) fails to commit to Tail
            Assert.IsNotNull(oldUpdateResult, "oldUpdateResult = null");
            Assert.AreEqual((int)HttpStatusCode.ServiceUnavailable, oldUpdateResult.HttpStatusCode, "oldUpdateResult.HttpStatusCode mismatch");
            Console.WriteLine("Update in stale View Succeeded with HttpStatus={0}", oldUpdateResult.HttpStatusCode);

            // Thread_1 (stale ViewId): Reads the entry => Succeeds eventhough the row is "uncommitted".
            //                          "ReadTailIndex = [Head]" means [Head] is treated as [Tail] for Reads.
            //                           => row is assumed "committed"
            customer = RetrieveCustomer(firstName, lastName, workerOne);
            Assert.IsNotNull(customer, "customer = null");
            Assert.AreEqual("workerOne", customer.Email, "customer.Email mismatch");

            // Thread_2 (new ViewId): Reads the entry => Succeed since reading from Head
            customerSeenByNewView = RetrieveCustomer(firstName, lastName, workerTwo);
            Assert.IsNotNull(customerSeenByNewView, "customerSeenByNewView = null");
            Assert.AreEqual("workerOne", customer.Email, "customer.Email mismatch");
        }
Ejemplo n.º 9
0
        public void WhenReadFromTailFailsWithServiceUnavailableWeReadFromHeadAndSucceed()
        {
            View view = this.configurationWrapper.GetWriteView();

            Assert.IsTrue(view.Chain.Count > 1, "expects at least 2 replicas!");

            // Insert one entry
            var rtable = new ReplicatedTable(this.repTable.TableName, this.configurationService);

            string firstName = "FirstName01";
            string lastName  = "LastName01";

            var            customer  = new CustomerEntity(firstName, lastName);
            TableOperation operation = TableOperation.Insert(customer);

            rtable.Execute(operation);


            // Using xstore modify the row in each replica individually ... so we know, later, which replica RTable will retrieve from
            for (int replicaIndex = 0; replicaIndex < this.cloudTableClients.Count; replicaIndex++)
            {
                CloudTableClient tableClient = this.cloudTableClients[replicaIndex];
                CloudTable       table       = tableClient.GetTableReference(this.repTable.TableName);

                TableOperation retrieveOperation = TableOperation.Retrieve <CustomerEntity>(firstName, lastName);
                TableResult    retrieveResult    = table.Execute(retrieveOperation);

                customer       = (CustomerEntity)retrieveResult.Result;
                customer.Email = replicaIndex.ToString(); // Head = 0
                // ...  = 1
                // Tail = 2

                TableOperation updateOperation = TableOperation.Replace(customer);
                TableResult    updateResult    = table.Execute(updateOperation);

                Assert.IsNotNull(updateResult, "updateResult = null");
                Console.WriteLine("updateResult.HttpStatusCode = {0}", updateResult.HttpStatusCode);
                Assert.AreEqual((int)HttpStatusCode.NoContent, updateResult.HttpStatusCode, "updateResult.HttpStatusCode mismatch");
            }

            string accountNameToNotTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0];

            Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToNotTamper={0}", accountNameToNotTamper);

            // Delay behavior
            ProxyBehavior[] behaviors =
            {
                TamperBehaviors.TamperAllRequestsIf(
                    (session =>
                {
                    session.oRequest.FailSession((int)HttpStatusCode.ServiceUnavailable, "ServerBusy", "");
                }),
                    (session =>
                {
                    if (session.HTTPMethodIs("Get"))
                    {
                        // Fail Get from all replicas, except Head
                        if (!session.hostname.Contains(accountNameToNotTamper + "."))
                        {
                            return(true);
                        }
                    }

                    return(false);
                })),
            };

            using (new HttpMangler(false, behaviors))
            {
                operation = TableOperation.Retrieve <CustomerEntity>(firstName, lastName);
                TableResult retrievedResult = repTable.Execute(operation);

                Assert.AreNotEqual(null, retrievedResult, "retrievedResult = null");
                Assert.AreEqual((int)HttpStatusCode.OK, retrievedResult.HttpStatusCode, "retrievedResult.HttpStatusCode mismatch");
                Assert.AreNotEqual(null, retrievedResult.Result, "retrievedResult.Result = null");

                customer = (CustomerEntity)retrievedResult.Result;
                Assert.AreEqual(customer.Email, (0).ToString(), "we should have read the row from Head");
            }
        }
        public void LinqQueriesAreServedFromHeadWhenReadViewTailIndexIsSet()
        {
            var rtable = new ReplicatedTable(this.repTable.TableName, this.configurationService);

            string firstName = "FirstName";
            string lastName  = "LastName";

            int dataSize = 5;

            /*
             * 1 - insert entries
             */
            for (int i = 0; i < dataSize; i++)
            {
                var customer = new CustomerEntity(firstName + i, lastName + i);
                customer.Email = "***";

                TableOperation operation = TableOperation.Insert(customer);
                rtable.Execute(operation);
            }


            // Identify the Tail account
            View view = this.configurationWrapper.GetWriteView();

            Assert.IsTrue(view.Chain.Count > 1, "expects at least one replica!");

            string accountNameToTamper = view.Chain.Last().Item1.StorageAccountName;

            Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper);

            // We will fail requests to update a row in Tail
            ProxyBehavior[] behaviors =
            {
                TamperBehaviors.TamperAllRequestsIf(
                    (session =>
                {
                    session.oRequest.FailSession((int)HttpStatusCode.ServiceUnavailable, "ServerBusy", "");
                }),
                    (session =>
                {
                    var body = session.GetRequestBodyAsString();

                    // Fail Lock to Tail by stale view
                    if (session.hostname.Contains(accountNameToTamper + ".") &&
                        session.HTTPMethodIs("PUT") &&
                        body.Contains("\"_rtable_RowLock\":true"))
                    {
                        return(true);
                    }

                    return(false);
                })),
            };

            using (new HttpMangler(false, behaviors))
            {
                /*
                 * 2 - update entries
                 */
                for (int i = 0; i < dataSize; i++)
                {
                    TableOperation operation       = TableOperation.Retrieve <CustomerEntity>(firstName + i, lastName + i);
                    TableResult    retrievedResult = rtable.Execute(operation);

                    var customer = (CustomerEntity)retrievedResult.Result;
                    customer.Email = "updated";

                    TableOperation replaceOperation = TableOperation.Replace(customer);
                    TableResult    replaceResult    = rtable.Execute(replaceOperation);

                    Assert.IsNotNull(replaceResult, "replaceResult = null");
                    Assert.AreEqual((int)HttpStatusCode.ServiceUnavailable, replaceResult.HttpStatusCode, "replaceResult.HttpStatusCode mismatch");
                }
            }


            SetReadViewTailIndex(0);
            Assert.AreEqual(0, this.configurationService.GetTableView(this.repTable.TableName).ReadTailIndex, "ReadTailIndex should be 0!!!");


            /*
             * 3 - CreateQuery is served from [Head], data should be new
             */
            foreach (var customer in rtable.CreateReplicatedQuery <CustomerEntity>().AsEnumerable())
            {
                Assert.AreEqual(customer.Email, "updated", "expected new data");
            }


            /*
             * 4 - ExecuteQuery is served from [Head], data should be new
             */
            foreach (var customer in rtable.ExecuteQuery <CustomerEntity>(new TableQuery <CustomerEntity>()))
            {
                Assert.AreEqual(customer.Email, "updated", "expected new data");
            }
        }
        public void LinqQueriesDontReturnEntriesWithTombstone()
        {
            var rtable = new ReplicatedTable(this.repTable.TableName, this.configurationService);

            string firstName = "FirstName";
            string lastName  = "LastName";

            /*
             * 1 - insert entries
             */
            for (int i = 0; i < 10; i++)
            {
                var customer = new CustomerEntity(firstName + i, lastName + i);

                TableOperation operation = TableOperation.Insert(customer);
                rtable.Execute(operation);
            }


            // Identify the Tail account
            View view = this.configurationWrapper.GetWriteView();

            Assert.IsTrue(view.Chain.Count > 1, "expects at least one replica!");

            string accountNameToTamper = view.Chain.Last().Item1.StorageAccountName;

            Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper);

            // We will fail requests to delete a row in Tail
            ProxyBehavior[] behaviors =
            {
                TamperBehaviors.TamperAllRequestsIf(
                    (session =>
                {
                    session.oRequest.FailSession((int)HttpStatusCode.ServiceUnavailable, "ServerBusy", "");
                }),
                    (session =>
                {
                    // Commit to Tail i.e. physically delete the row
                    if (session.hostname.Contains(accountNameToTamper + ".") &&
                        session.HTTPMethodIs("DELETE"))
                    {
                        return(true);
                    }

                    return(false);
                })),
            };

            using (new HttpMangler(false, behaviors))
            {
                /*
                 * 2 - delete entries #2 and #4
                 */
                foreach (var i in new int[] { 2, 4 })
                {
                    TableOperation operation       = TableOperation.Retrieve <CustomerEntity>(firstName + i, lastName + i);
                    TableResult    retrievedResult = rtable.Execute(operation);

                    var customer = (CustomerEntity)retrievedResult.Result;

                    TableOperation deleteOperation = TableOperation.Delete(customer);
                    TableResult    deleteResult    = rtable.Execute(deleteOperation);

                    Assert.IsNotNull(deleteResult, "deleteResult = null");
                    Assert.AreEqual((int)HttpStatusCode.ServiceUnavailable, deleteResult.HttpStatusCode, "deleteResult.HttpStatusCode mismatch");
                }
            }


            // Verify, Retrieve doesn't return entries #2 and #4
            int deleteId = 2;
            var result   = rtable.Execute(TableOperation.Retrieve <CustomerEntity>(firstName + deleteId, lastName + deleteId));

            Assert.IsNotNull(result, "result = null");
            Assert.AreEqual((int)HttpStatusCode.NotFound, result.HttpStatusCode, "result.HttpStatusCode mismatch");

            deleteId = 4;
            result   = rtable.Execute(TableOperation.Retrieve <CustomerEntity>(firstName + deleteId, lastName + deleteId));
            Assert.IsNotNull(result, "result = null");
            Assert.AreEqual((int)HttpStatusCode.NotFound, result.HttpStatusCode, "result.HttpStatusCode mismatch");


            /*
             * 3 - CreateQuery doesn't return entries #2 and #4
             */
            foreach (var customer in rtable.CreateReplicatedQuery <CustomerEntity>().AsEnumerable())
            {
                int id = int.Parse(customer.PartitionKey.Replace(firstName, ""));
                Assert.AreNotEqual(id, 2, "entry #2 should have been deleted");
                Assert.AreNotEqual(id, 4, "entry #4 should have been deleted");
            }


            /*
             * 4 - ExecuteQuery doesn't return entries #2 and #4
             */
            foreach (var customer in rtable.ExecuteQuery <CustomerEntity>(new TableQuery <CustomerEntity>()))
            {
                int id = int.Parse(customer.PartitionKey.Replace(firstName, ""));
                Assert.AreNotEqual(id, 2, "entry #2 should have been deleted");
                Assert.AreNotEqual(id, 4, "entry #4 should have been deleted");
            }
        }
Ejemplo n.º 12
0
        public void RepairMidAnUpdate()
        {
            this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);

            // Not-Stable chain
            // Reconfigure RTable so Head is WriteOnly.
            View view = this.configurationService.GetTableView(this.repTable.TableName);

            ReplicatedTableConfiguration    config;
            ReplicatedTableQuorumReadResult readStatus = this.configurationService.RetrieveConfiguration(out config);

            Assert.IsTrue(readStatus.Code == ReplicatedTableQuorumReadCode.Success);

            // Set Head as WriteOnly mode
            ReplicatedTableConfigurationStore viewConfg = config.GetView(view.Name);

            viewConfg.ReplicaChain[0].Status = ReplicaStatus.WriteOnly;
            config.SetView(view.Name, viewConfg);

            // Upload RTable config back
            this.configurationService.UpdateConfiguration(config);

            // Sanity: Replicated mode and chain Not-Stable
            view = this.configurationService.GetTableView(this.repTable.TableName);
            Assert.IsTrue(view != null && view.Chain.Count > 1, "Two replicas should be used.");
            Assert.IsFalse(view.IsStable);


            // Insert one entry
            Console.WriteLine("Inserting entry ...");
            string entityPartitionKey = "jobType-RepairMidAnUpdate-Replace";
            string entityRowKey       = "jobId-RepairMidAnUpdate-Replace";
            var    entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, "message");

            this.rtableWrapper.InsertRow(entry);


            // 1 - Launch a RepairRow task in wait mode ...
            bool triggerRepair            = false;
            bool repaireDone              = false;
            bool headAfterRepairWasLocked = false;

            TableResult repairResult = null;

            Task.Run(() =>
            {
                ReplicatedTable repairTable = new ReplicatedTable(this.repTable.TableName, this.configurationService);

                while (!triggerRepair)
                {
                    Thread.Sleep(5);
                }

                Console.WriteLine("RepairRow started ...");
                repairResult = repairTable.RepairRow(entry.PartitionKey, entry.RowKey, null);
                Console.WriteLine("RepairRow completed with HttpStatus={0}", repairResult == null ? "NULL" : repairResult.HttpStatusCode.ToString());

                // Check the entry at the Head is still locked i.e. RepairRow was NOP
                headAfterRepairWasLocked = HeadIsLocked(entry);

                Console.WriteLine("Signal the commit to Head job");
                repaireDone = true;
            });


            // 2 - Configure Mangler ...
            string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0];

            Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper);

            ProxyBehavior[] behaviors = new[]
            {
                TamperBehaviors.TamperAllRequestsIf(
                    (session =>
                {
                    Console.WriteLine("Delaying commit to the Head ... => signal RepairRow job");

                    // Let RepairRow task go through
                    triggerRepair = true;

                    int iter = 0;
                    while (!repaireDone)
                    {
                        // TODO: break the loop after couple of iteration ...

                        Thread.Sleep(100);
                        Console.WriteLine("Waiting on RepairRow to finish ({0}) ...", ++iter);
                    }

                    Console.WriteLine("Request a commit to the head");
                }),
                    (session =>
                {
                    // Commit on head i.e. a PUT with RowLock == false
                    if (session.hostname.Contains(accountNameToTamper + ".") &&
                        session.HTTPMethodIs("PUT") &&
                        session.GetRequestBodyAsString().Contains("\"_rtable_RowLock\":false"))
                    {
                        return(true);
                    }

                    return(false);
                }))
            };

            using (new HttpMangler(false, behaviors))
            {
                Console.WriteLine("Updating entry ...");
                entry         = this.rtableWrapper.FindRow(entry.PartitionKey, entry.RowKey);
                entry.Message = "updated message";

                this.rtableWrapper.ReplaceRow(entry);
            }

            Assert.IsTrue(triggerRepair);
            Assert.IsTrue(repairResult != null && repairResult.HttpStatusCode == (int)HttpStatusCode.OK, "Repair failed.");
            Assert.IsTrue(repaireDone);
            Assert.IsTrue(headAfterRepairWasLocked);

            Console.WriteLine("DONE. Test passed.");
        }
Ejemplo n.º 13
0
        public void A00TwoInsertOrReplaceCallsConflictingOnTheHead()
        {
            this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);

            string entityPartitionKey = "jobType-DelayInsertOrReplaceRowHeadTest";
            string entityRowKey       = "jobId-DelayInsertOrReplaceRowHeadTest";

            this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);

            // Insert one entry
            Console.WriteLine("Inserting entry ...");
            var entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, "insert message");

            this.rtableWrapper.InsertRow(entry);

            string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0];

            Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper);

            int  delayInMs = 3000;
            bool firstWritterInitiatingCommit  = false;
            bool secondWritterTriedLockingHead = false;

            // Delay behavior
            ProxyBehavior[] behaviors = new[]
            {
                // Writter-1 tampering
                TamperBehaviors.TamperAllRequestsIf(
                    (session =>
                {
                    int iter = 0;

                    // Signal Writter-2
                    firstWritterInitiatingCommit = true;

                    // Blobk commit to head ... until Writter-2 try to lock the head
                    while (!secondWritterTriedLockingHead)
                    {
                        Console.WriteLine("Writter-1 waiting on Writter-2 to try to lock the Head (#{0})", iter);
                        Thread.Sleep(delayInMs);

                        if (++iter > 10)
                        {
                            break;
                        }
                    }
                }),
                    (session =>
                {
                    var body = session.GetRequestBodyAsString();

                    // Writter-1 committing to head
                    if (session.hostname.Contains(accountNameToTamper + ".") &&
                        session.HTTPMethodIs("PUT") &&
                        body.Contains("\"_rtable_Operation\":\"Replace\"") &&
                        body.Contains("\"_rtable_RowLock\":false") &&
                        body.Contains("\"Message\":\"upsert message 0\"")
                        )
                    {
                        return(true);
                    }

                    return(false);
                })),

                // Writter-2 tampering
                TamperBehaviors.TamperAllRequestsIf(
                    (session =>
                {
                    // Block till Writter-1 issues a commit to head
                    while (!firstWritterInitiatingCommit)
                    {
                        Console.WriteLine("Writter-2 waiting on Writter-1 to issue a commit to head");
                        Thread.Sleep(delayInMs);
                    }
                }),
                    (session =>
                {
                    var body = session.GetRequestBodyAsString();

                    // Writter-2 locking the head
                    if (session.hostname.Contains(accountNameToTamper + ".") &&
                        session.HTTPMethodIs("PUT") &&
                        body.Contains("\"_rtable_Operation\":\"Replace\"") &&
                        body.Contains("\"_rtable_RowLock\":true") &&
                        body.Contains("\"Message\":\"upsert message 1\"")
                        )
                    {
                        return(true);
                    }

                    return(false);
                })),

                // Delay Writter-2 lock-to-the-head's response, so Writter-1 can continue with its commit.
                DelayBehaviors.DelayAllResponsesIf(
                    delayInMs,
                    (session =>
                {
                    var body = session.GetRequestBodyAsString();

                    // Writter-2 locking the head response
                    if (session.hostname.Contains(accountNameToTamper + ".") &&
                        session.HTTPMethodIs("PUT") &&
                        body.Contains("\"_rtable_Operation\":\"Replace\"") &&
                        body.Contains("\"_rtable_RowLock\":true") &&
                        body.Contains("\"Message\":\"upsert message 1\"")
                        )
                    {
                        // Signal Writter-1 so it can continue with commit to head
                        secondWritterTriedLockingHead = true;
                        return(true);
                    }

                    return(false);
                })),
            };

            // Launch 2 concurrent Upserts
            var results = new TableResult[2];

            using (new HttpMangler(false, behaviors))
            {
                Parallel.For(0, 2, (index) =>
                {
                    entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, string.Format("upsert message {0}", index));

                    try
                    {
                        var table = new ReplicatedTable(this.repTable.TableName, this.configurationService);

                        TableOperation operation = TableOperation.InsertOrReplace(entry);
                        results[index]           = table.Execute(operation);
                    }
                    catch (AggregateException ex)
                    {
                        Console.WriteLine(ex);
                    }
                });
            }

            // Writter-1 suceed?
            Assert.AreEqual((int)HttpStatusCode.NoContent, results[0].HttpStatusCode, "Writter-1 expected to suceed!");

            // Writter-2 suceeded?
            Assert.AreEqual((int)HttpStatusCode.NoContent, results[1].HttpStatusCode, "Writter-2 expected to suceed!");

            // Writter-2 upsert succeeded
            SampleRTableEntity upsertedEntity = this.rtableWrapper.ReadEntity(entityPartitionKey, entityRowKey);

            Assert.NotNull(upsertedEntity, "Writter-2 upsert succeeded");
            Assert.AreEqual(upsertedEntity.Message, string.Format("upsert message {0}", 1), "Writter-2 upsert succeeded");
        }
Ejemplo n.º 14
0
        public void A00DelayTwoConflictingInsertOrReplaceCalls()
        {
            this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);

            string entityPartitionKey = "jobType-DelayInsertOrReplaceRowHeadTest";
            string entityRowKey       = "jobId-DelayInsertOrReplaceRowHeadTest";

            this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);

            string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0];

            Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper);

            int  delayInMs              = 3000;
            int  insertRequestCount     = 0;
            int  conflictResponseCount  = 0;
            bool secondUpsertConflicted = false;
            int  failedCallIndex        = -1;

            // Delay behavior
            ProxyBehavior[] behaviors = new[]
            {
                // Delay Insert calls so they end up conflicting
                TamperBehaviors.TamperAllRequestsIf(
                    (session =>
                {
                    Interlocked.Increment(ref insertRequestCount);

                    while (insertRequestCount != 2)
                    {
                        Console.WriteLine("insertRequestCount={0}. Waiting on count to reach 2 ...", insertRequestCount);
                        Thread.Sleep(delayInMs);
                    }
                }),
                    (session =>
                {
                    if (session.hostname.Contains(accountNameToTamper + ".") &&
                        session.HTTPMethodIs("POST") &&
                        session.GetRequestBodyAsString().Contains("\"_rtable_Operation\":\"Insert\""))
                    {
                        return(true);
                    }

                    return(false);
                })),

                // Delay conflict response
                DelayBehaviors.DelayAllResponsesIf(
                    delayInMs,
                    (session =>
                {
                    if (session.hostname.Contains(accountNameToTamper + ".") &&
                        session.GetRequestBodyAsString().Contains("\"_rtable_Operation\":\"Insert\""))
                    {
                        if (session.responseCode == (int)HttpStatusCode.Conflict)
                        {
                            Interlocked.Increment(ref conflictResponseCount);

                            return(true);
                        }
                    }

                    return(false);
                })),
            };

            using (new HttpMangler(false, behaviors))
            {
                Parallel.For(0, 2, (index) =>
                {
                    var entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, string.Format("upsert message {0}", index));
                    try
                    {
                        this.rtableWrapper.InsertOrReplaceRow(entry);
                    }
                    catch (RTableConflictException)
                    {
                        if (secondUpsertConflicted)
                        {
                            // should never reach here
                            throw;
                        }

                        // That's possible, but that's the Replace step of upsert which conflicted with ongoing write
                        // can't do anything, client should retry on conflict
                        secondUpsertConflicted = true;
                    }
                });
            }

            // got 2 inserts?
            Assert.AreEqual(2, insertRequestCount, "Two insert calls expected!");

            // got one conflict?
            Assert.AreEqual(1, conflictResponseCount, "One conflict response expected!");

            // at least one upsert would have succeeded
            SampleRTableEntity upsertedEntity = this.rtableWrapper.ReadEntity(entityPartitionKey, entityRowKey);

            Assert.NotNull(upsertedEntity, "at least one upsert should have succeeded");

            // second upsert failed?
            if (secondUpsertConflicted)
            {
                Assert.AreEqual(1, upsertedEntity._rtable_Version, "one upsert succeeded so version should be = 1");
                Assert.AreEqual(upsertedEntity.Message, string.Format("upsert message {0}", (1 - failedCallIndex)));
            }
            else
            {
                Assert.AreEqual(2, upsertedEntity._rtable_Version, "both upserts succeeded so version should be = 2");
            }


            // After recovery from delay, confirm that we can update the row.
            //this.ExecuteReplaceRowAndValidate(entityPartitionKey, entityRowKey);
        }
Ejemplo n.º 15
0
        public void InsertInStaleViewConflictingWithInsertInNewView()
        {
            ReplicatedTableConfigurationServiceV2 configServiceOne, configServiceTwo;

            // setup:        Acc0       Acc1
            //  stale view = [None] ->  [RW]
            //  new view   = [WO]   ->  [RW]
            SetupStaleViewAndNewView(out configServiceOne, out configServiceTwo);

            long staleViewId  = configServiceOne.GetTableView(this.repTable.TableName).ViewId;
            long latestViewId = configServiceTwo.GetTableView(this.repTable.TableName).ViewId;

            string firstName = "FirstName01";
            string lastName  = "LastName01";

            var workerOne = new ReplicatedTable(this.repTable.TableName, configServiceOne);
            var workerTwo = new ReplicatedTable(this.repTable.TableName, configServiceTwo);

            string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[1];

            Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper);

            TableResult oldInsertResult          = null;
            TableResult newInsertResult          = null;
            bool        triggerInsertWithNewView = false;
            bool        oldInsertResume          = false;

            // Start new newInsertTask in wait
            var newInsertTask = Task.Run(() =>
            {
                while (!triggerInsertWithNewView)
                {
                    Thread.Sleep(100);
                }

                /*
                 * 2 - Executes after step 1 below:
                 *     WorkerTwo => Insert a new row using new view
                 */
                var customer   = new CustomerEntity(firstName, lastName);
                customer.Email = "workerTwo";

                TableOperation operation = TableOperation.Insert(customer);
                newInsertResult          = workerTwo.Execute(operation);

                // Signal old Insert to resume
                oldInsertResume = true;
            });


            // Delay behavior
            ProxyBehavior[] behaviors =
            {
                TamperBehaviors.TamperAllRequestsIf(
                    (session =>
                {
                    // => trigger new insert to start
                    triggerInsertWithNewView = true;

                    // Delaying commit to the Tail by stale view Update
                    while (!oldInsertResume)
                    {
                        Thread.Sleep(100);
                    }
                }),
                    (session =>
                {
                    var body = session.GetRequestBodyAsString();

                    // Commit to Tail by stale view
                    if (session.hostname.Contains(accountNameToTamper + ".") &&
                        session.HTTPMethodIs("PUT") &&
                        body.Contains("\"Email\":\"workerOne\"") &&
                        body.Contains(string.Format("\"_rtable_ViewId\":\"{0}\"", staleViewId)) &&
                        body.Contains("\"_rtable_RowLock\":false"))
                    {
                        return(true);
                    }

                    return(false);
                })),
            };

            /*
             * 1 - WorkerOne => update an entry using the stale View
             */
            using (new HttpMangler(false, behaviors))
            {
                var customer = new CustomerEntity(firstName, lastName);
                customer.Email = "workerOne";

                TableOperation operation = TableOperation.Insert(customer);
                oldInsertResult = workerOne.Execute(operation);
            }

            // Wait on new Insert to finish
            newInsertTask.Wait();


            // Expected behavior:

            // Thread_1 (stale ViewId)
            Assert.IsNotNull(oldInsertResult, "oldInsertResult = null");
            Assert.AreEqual((int)HttpStatusCode.NoContent, oldInsertResult.HttpStatusCode, "oldInsertResult.HttpStatusCode mismatch");
            Console.WriteLine("Insert in stale View Succeeded with HttpStatus={0}", oldInsertResult.HttpStatusCode);

            // Thread_2 (new ViewId)
            Assert.IsNotNull(newInsertResult, "newInsertResult = null");
            Assert.AreEqual((int)HttpStatusCode.Conflict, newInsertResult.HttpStatusCode, "newUpdateResult.HttpStatusCode mismatch");
            Console.WriteLine("Insert in new View failed with HttpStatus={0}", newInsertResult.HttpStatusCode);

            var currCustomer = RetrieveCustomer(firstName, lastName, workerTwo);

            Assert.AreEqual(staleViewId, currCustomer._rtable_ViewId, "customer._rtable_ViewId mismatch");
            Assert.AreEqual("workerOne", currCustomer.Email, "customer.Email mismatch");

            Console.WriteLine("workerOne write Succeeded");
            Console.WriteLine("workerTwo got Conflict because row was lock.");
        }