Ejemplo n.º 1
0
        public void ReplaceRowTest()
        {
            this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);

            string entityPartitionKey = "jobType-TamperReplaceRowHeadTest";
            string entityRowKey       = "jobId-TamperReplaceRowHeadTest";

            this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);

            int targetStorageAccount = 1;
            TargetRTableWrapperApi <SampleRTableEntity> targetApi = this.rtableWrapper.ReplaceRow;
            bool targetApiExpectedToFail        = false;
            bool checkOriginalEntityUnchanged   = false;
            bool checkStorageAccountsConsistent = false;

            DateTime httpManglerStartTime;
            Task     task = Task.Run(() =>
                                     this.SetupAndRunDelayTableBehavior(
                                         entityPartitionKey,
                                         entityRowKey,
                                         targetStorageAccount,
                                         targetApi,
                                         targetApiExpectedToFail,
                                         checkOriginalEntityUnchanged,
                                         checkStorageAccountsConsistent,
                                         out httpManglerStartTime));

            task.Wait();
        }
Ejemplo n.º 2
0
        public void B00TamperReadRowInnerTest()
        {
            this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);

            string entityPartitionKey = "jobType-TamperReadRowInnerTest";
            string entityRowKey       = "jobId-TamperReadRowInnerTest";

            this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);

            int  targetStorageAccount           = 1;
            bool targetApiExpectedToFail        = false;
            bool checkOriginalEntityUnchanged   = true;
            bool checkStorageAccountsConsistent = true;

            // Try this many times. It should not fail.
            for (int i = 0; i < 3 * this.actualStorageAccountsUsed.Count; i++)
            {
                this.SetupAndRunTamperTableBehavior(
                    entityPartitionKey,
                    entityRowKey,
                    targetStorageAccount,
                    this.rtableWrapper.ReadEntity,
                    targetApiExpectedToFail,
                    checkOriginalEntityUnchanged,
                    checkStorageAccountsConsistent);
                Console.WriteLine("\n ----------------------------------- Finished iteration {0}", i);
            }
        }
Ejemplo n.º 3
0
        public void A00TamperReplaceRowHeadTest()
        {
            this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);

            string entityPartitionKey = "jobType-TamperReplaceRowHeadTest";
            string entityRowKey       = "jobId-TamperReplaceRowHeadTest";

            this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);

            int targetStorageAccount = 0;
            TargetRTableWrapperApi <SampleRTableEntity> targetApi = this.rtableWrapper.ReplaceRow;
            bool targetApiExpectedToFail        = true;
            bool checkOriginalEntityUnchanged   = true;
            bool checkStorageAccountsConsistent = true;

            DateTime httpManglerStartTime;

            this.SetupAndRunTamperTableBehavior(
                entityPartitionKey,
                entityRowKey,
                targetStorageAccount,
                targetApi,
                targetApiExpectedToFail,
                checkOriginalEntityUnchanged,
                checkStorageAccountsConsistent,
                out httpManglerStartTime);

            // After recovery from short outage, confirm that we can update the row.
            this.ExecuteReplaceRowAndValidate(entityPartitionKey, entityRowKey);
        }
Ejemplo n.º 4
0
        public void A00DelayReplaceRowHeadTest()
        {
            this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);

            string entityPartitionKey = "jobType-DelayReplaceRowHeadTest";
            string entityRowKey       = "jobId-DelayReplaceRowHeadTest";

            this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);

            int targetStorageAccount = 0;
            TargetRTableWrapperApi <SampleRTableEntity> targetApi = this.rtableWrapper.ReplaceRow;
            bool targetApiExpectedToFail = false;
            int  delayInMs = 3000;

            this.SetupAndRunDelayTableBehavior(
                entityPartitionKey,
                entityRowKey,
                targetStorageAccount,
                delayInMs,
                targetApi,
                targetApiExpectedToFail);

            // After recovery from delay, confirm that we can update the row.
            this.ExecuteReplaceRowAndValidate(entityPartitionKey, entityRowKey);
        }
Ejemplo n.º 5
0
        public void C00TamperReplaceRowTailTest()
        {
            this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);

            string entityPartitionKey = "jobType-TamperReplaceRowTailTest";
            string entityRowKey       = "jobId-TamperReplaceRowTailTest";

            this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);

            int targetStorageAccount = this.actualStorageAccountsUsed.Count - 1;
            TargetRTableWrapperApi <SampleRTableEntity> targetApi = this.rtableWrapper.ReplaceRow;
            bool targetApiExpectedToFail        = true;
            bool checkOriginalEntityUnchanged   = true;
            bool checkStorageAccountsConsistent = false;

            DateTime httpManglerStartTime;

            this.SetupAndRunTamperTableBehavior(
                entityPartitionKey,
                entityRowKey,
                targetStorageAccount,
                targetApi,
                targetApiExpectedToFail,
                checkOriginalEntityUnchanged,
                checkStorageAccountsConsistent,
                out httpManglerStartTime);

            // After recovery from short outage, sleep some time to wait for entity to be unlocked, then confirm that we can update the row.
            this.SleepUntilRowLockHasExpired(httpManglerStartTime);

            this.ExecuteReplaceRowAndValidate(entityPartitionKey, entityRowKey);
        }
Ejemplo n.º 6
0
        public void B00DelayDeleteRowInnerTest()
        {
            this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);

            string entityPartitionKey = "jobType-DelayDeleteRowInnerTest";
            string entityRowKey       = "jobId-DelayDeleteRowInnerTest";

            this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);

            int targetStorageAccount = 1;
            TargetRTableWrapperApi <SampleRTableEntity> targetApi = this.rtableWrapper.DeleteRow;
            bool targetApiExpectedToFail = false;
            int  delayInMs = 3000;

            this.SetupAndRunDelayTableBehavior(
                entityPartitionKey,
                entityRowKey,
                targetStorageAccount,
                delayInMs,
                targetApi,
                targetApiExpectedToFail);

            if (targetApiExpectedToFail)
            {
                // After recovery from delay, confirm that we can delete the row.
                this.ExecuteDeleteRowAndValidate(entityPartitionKey, entityRowKey);
            }
            else
            {
                // After recovery from delay, confirm the entity is gone.
                this.ExecuteReadRowAndValidateNotExist(entityPartitionKey, entityRowKey);
            }
            // Confirm we can create entity using the same partition and row keys again.
            this.ExecuteCreateRowAndValidate(entityPartitionKey, entityRowKey);
        }
Ejemplo n.º 7
0
        /// <summary>
        /// Call this function to modify the contents of the RTable configuration blob to use the specified viewId
        /// and convertXStoreTableMode and update the appropriate wrapper.
        /// </summary>
        /// <param name="updatedViewId"></param>
        protected void RefreshRTableEnvJsonConfigBlob(long updatedViewId, bool convertXStoreTableMode = false,
                                                      int readViewHeadIndex = 0, List <int> blobIndexes = null, bool waitForConfig = true)
        {
            if (blobIndexes == null)
            {
                blobIndexes = new List <int> {
                    0
                };
            }

            Console.WriteLine(
                "Calling RefreshRTableEnvJsonConfigBlob() with updatedViewId={0} convertXStoreTableMode={1} readViewHeadIndex={2}",
                updatedViewId, convertXStoreTableMode, readViewHeadIndex);
            this.ModifyConfigurationBlob(updatedViewId, convertXStoreTableMode, readViewHeadIndex, blobIndexes);
            this.repTable      = new ReplicatedTable(this.repTable.TableName, this.configurationService);
            this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);

            if (!waitForConfig)
            {
                return;
            }

            while (this.configurationWrapper.GetWriteView().ViewId != updatedViewId)
            {
                Console.WriteLine("Sleeping {0} seconds for new viewId to take effect...",
                                  Constants.LeaseDurationInSec / 2);
                Thread.Sleep((Constants.LeaseDurationInSec / 2) * 1000);
            }
        }
Ejemplo n.º 8
0
        /// <summary>
        /// Setup the RTable Env in order to run unit tests.
        /// </summary>
        /// <param name="tableName">Name of the RTable. If string.Empty or null, then use this.rtableTestConfiguration.RTableInformation.RTableName</param>
        /// <param name="useHttps">When using HttpMangler, set this to false</param>
        /// <param name="viewIdString">_rtable_ViewId used in the RTable config blob. Set to string.Empty or null to use the _rtable_ViewId value in the xml config</param>
        /// <param name="actualStorageAccountsToBeUsed">Specify how/which Storage accounts in the config.xml will be used to construct the RTable. Set to null to use all accounts in xml. E.g., {0,1}</param>
        protected void SetupRTableEnv(
            string tableName    = "",
            bool useHttps       = true,
            string viewIdString = "",
            List <int> actualStorageAccountsToBeUsed = null,
            bool convertXStoreTableMode = false,
            int numberOfBlobs           = 0)
        {
            int numberOfStorageAccounts = this.rtableTestConfiguration.StorageInformation.AccountNames.Count();

            // If user does not specify which storage accounts to construct the RTable, then use all the storage accounts in the xml.
            this.actualStorageAccountsUsed = actualStorageAccountsToBeUsed;
            if (this.actualStorageAccountsUsed == null || this.actualStorageAccountsUsed.Count == 0)
            {
                this.actualStorageAccountsUsed = new List <int>();
                for (int i = 0; i < numberOfStorageAccounts; i++)
                {
                    this.actualStorageAccountsUsed.Add(i);
                }
            }

            this.UpdateConfigurationConstants();

            // Upload RTable configuration to blob
            int viewId = this.rtableTestConfiguration.RTableInformation.ViewId;

            if (!string.IsNullOrEmpty(viewIdString))
            {
                viewId = int.Parse(viewIdString);
            }

            numberOfBlobs = numberOfBlobs == 0
                ? this.rtableTestConfiguration.StorageInformation.AccountKeys.Count()
                : numberOfBlobs;

            if (string.IsNullOrEmpty(tableName))
            {
                tableName = this.rtableTestConfiguration.RTableInformation.RTableName;
            }

            this.InitConnectionStringMap(useHttps);

            InitialSetup(tableName, useHttps, viewId, convertXStoreTableMode, numberOfBlobs);

            this.repTable = new ReplicatedTable(tableName, this.configurationService);
            this.repTable.CreateIfNotExists();
            this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);

            this.cloudTableClients = new List <CloudTableClient>();
            this.cloudTables       = new List <CloudTable>();
            for (int i = 0; i < this.actualStorageAccountsUsed.Count; i++)
            {
                CloudTableClient cloudBloblClient = repTable.GetReplicaTableClient(i);
                this.cloudTableClients.Add(cloudBloblClient);
                this.cloudTables.Add(cloudBloblClient.GetTableReference(repTable.TableName));
            }
        }
Ejemplo n.º 9
0
        public void B00TamperSkipDeleteRowInnerTest()
        {
            this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);

            string entityPartitionKey = "jobType-TamperSkipDeleteRowInnerTest";
            string entityRowKey       = "jobId-TamperSkipDeleteRowInnerTest";

            this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);

            int targetStorageAccount = 1;
            TargetRTableWrapperApi <SampleRTableEntity> targetApi = this.rtableWrapper.DeleteRow;
            bool targetApiExpectedToFail        = true;
            int  skipInitialiSessions           = 2;
            bool checkOriginalEntityUnchanged   = false;
            bool checkStorageAccountsConsistent = false;

            DateTime httpManglerStartTime;

            this.SetupAndRunTamperTableBehavior(
                entityPartitionKey,
                entityRowKey,
                targetStorageAccount,
                targetApi,
                targetApiExpectedToFail,
                checkOriginalEntityUnchanged,
                checkStorageAccountsConsistent,
                out httpManglerStartTime,
                skipInitialiSessions);

            if (targetApiExpectedToFail)
            {
                // After recovery from outage, sleep some time to wait for entity to be unlocked, then confirm that we can delete the row.
                this.SleepUntilRowLockHasExpired(httpManglerStartTime);

                this.ExecuteDeleteRowAndValidate(entityPartitionKey, entityRowKey);
            }
            else
            {
                // After recovery from outage, confirm the entity is gone.
                this.ExecuteReadRowAndValidateNotExist(entityPartitionKey, entityRowKey);
            }
            // Confirm we can create entity using the same partition and row keys again.
            this.ExecuteCreateRowAndValidate(entityPartitionKey, entityRowKey);
        }
Ejemplo n.º 10
0
        public void A00TamperDeleteRowHeadTest()
        {
            this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);

            string entityPartitionKey = "jobType-TamperDeleteRowHeadTest";
            string entityRowKey       = "jobId-TamperDeleteRowHeadTest";

            this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);

            int targetStorageAccount = 0;
            TargetRTableWrapperApi <SampleRTableEntity> targetApi = this.rtableWrapper.DeleteRow;
            bool targetApiExpectedToFail        = true;
            bool checkOriginalEntityUnchanged   = true;
            bool checkStorageAccountsConsistent = true;

            this.SetupAndRunTamperTableBehavior(
                entityPartitionKey,
                entityRowKey,
                targetStorageAccount,
                targetApi,
                targetApiExpectedToFail,
                checkOriginalEntityUnchanged,
                checkStorageAccountsConsistent);

            if (targetApiExpectedToFail)
            {
                // After recovery from outage, confirm that we can delete the row.
                this.ExecuteDeleteRowAndValidate(entityPartitionKey, entityRowKey);
            }
            else
            {
                // After recovery from outage, confirm the entity is gone.
                this.ExecuteReadRowAndValidateNotExist(entityPartitionKey, entityRowKey);
            }
            // Confirm we can create entity using the same partition and row keys again.
            this.ExecuteCreateRowAndValidate(entityPartitionKey, entityRowKey);
        }
Ejemplo n.º 11
0
        public void RepairMidAnUpdate()
        {
            this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);

            // Not-Stable chain
            // Reconfigure RTable so Head is WriteOnly.
            View view = this.configurationService.GetTableView(this.repTable.TableName);

            ReplicatedTableConfiguration    config;
            ReplicatedTableQuorumReadResult readStatus = this.configurationService.RetrieveConfiguration(out config);

            Assert.IsTrue(readStatus.Code == ReplicatedTableQuorumReadCode.Success);

            // Set Head as WriteOnly mode
            ReplicatedTableConfigurationStore viewConfg = config.GetView(view.Name);

            viewConfg.ReplicaChain[0].Status = ReplicaStatus.WriteOnly;
            config.SetView(view.Name, viewConfg);

            // Upload RTable config back
            this.configurationService.UpdateConfiguration(config);

            // Sanity: Replicated mode and chain Not-Stable
            view = this.configurationService.GetTableView(this.repTable.TableName);
            Assert.IsTrue(view != null && view.Chain.Count > 1, "Two replicas should be used.");
            Assert.IsFalse(view.IsStable);


            // Insert one entry
            Console.WriteLine("Inserting entry ...");
            string entityPartitionKey = "jobType-RepairMidAnUpdate-Replace";
            string entityRowKey       = "jobId-RepairMidAnUpdate-Replace";
            var    entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, "message");

            this.rtableWrapper.InsertRow(entry);


            // 1 - Launch a RepairRow task in wait mode ...
            bool triggerRepair            = false;
            bool repaireDone              = false;
            bool headAfterRepairWasLocked = false;

            TableResult repairResult = null;

            Task.Run(() =>
            {
                ReplicatedTable repairTable = new ReplicatedTable(this.repTable.TableName, this.configurationService);

                while (!triggerRepair)
                {
                    Thread.Sleep(5);
                }

                Console.WriteLine("RepairRow started ...");
                repairResult = repairTable.RepairRow(entry.PartitionKey, entry.RowKey, null);
                Console.WriteLine("RepairRow completed with HttpStatus={0}", repairResult == null ? "NULL" : repairResult.HttpStatusCode.ToString());

                // Check the entry at the Head is still locked i.e. RepairRow was NOP
                headAfterRepairWasLocked = HeadIsLocked(entry);

                Console.WriteLine("Signal the commit to Head job");
                repaireDone = true;
            });


            // 2 - Configure Mangler ...
            string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0];

            Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper);

            ProxyBehavior[] behaviors = new[]
            {
                TamperBehaviors.TamperAllRequestsIf(
                    (session =>
                {
                    Console.WriteLine("Delaying commit to the Head ... => signal RepairRow job");

                    // Let RepairRow task go through
                    triggerRepair = true;

                    int iter = 0;
                    while (!repaireDone)
                    {
                        // TODO: break the loop after couple of iteration ...

                        Thread.Sleep(100);
                        Console.WriteLine("Waiting on RepairRow to finish ({0}) ...", ++iter);
                    }

                    Console.WriteLine("Request a commit to the head");
                }),
                    (session =>
                {
                    // Commit on head i.e. a PUT with RowLock == false
                    if (session.hostname.Contains(accountNameToTamper + ".") &&
                        session.HTTPMethodIs("PUT") &&
                        session.GetRequestBodyAsString().Contains("\"_rtable_RowLock\":false"))
                    {
                        return(true);
                    }

                    return(false);
                }))
            };

            using (new HttpMangler(false, behaviors))
            {
                Console.WriteLine("Updating entry ...");
                entry         = this.rtableWrapper.FindRow(entry.PartitionKey, entry.RowKey);
                entry.Message = "updated message";

                this.rtableWrapper.ReplaceRow(entry);
            }

            Assert.IsTrue(triggerRepair);
            Assert.IsTrue(repairResult != null && repairResult.HttpStatusCode == (int)HttpStatusCode.OK, "Repair failed.");
            Assert.IsTrue(repaireDone);
            Assert.IsTrue(headAfterRepairWasLocked);

            Console.WriteLine("DONE. Test passed.");
        }
Ejemplo n.º 12
0
        public void A00InsertOrReplaceCallConflictingWithDeleteOnTheHead()
        {
            this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);

            string entityPartitionKey = "jobType-DelayInsertOrReplaceWhileDelete";
            string entityRowKey       = "jobType-DelayInsertOrReplaceWhileDelete";

            this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);

            // Insert one entry
            Console.WriteLine("Inserting entry ...");
            var entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, "insert message");

            this.rtableWrapper.InsertRow(entry);

            // 1 - Launch an Upsert task in wait mode ...
            bool deleteLockedHead = false;

            TableResult upsertResult = null;
            var         upsertTask   = Task.Run(() =>
            {
                while (!deleteLockedHead)
                {
                    Thread.Sleep(5);
                }

                try
                {
                    entry     = new SampleRTableEntity(entityPartitionKey, entityRowKey, "upsert message");
                    var table = new ReplicatedTable(this.repTable.TableName, this.configurationService);

                    TableOperation upserOperation = TableOperation.InsertOrReplace(entry);

                    Console.WriteLine("Upsert started ...");
                    upsertResult = table.Execute(upserOperation);
                    Console.WriteLine("Upsert completed with HttpStatus={0}", upsertResult == null ? "NULL" : upsertResult.HttpStatusCode.ToString());
                }
                catch (AggregateException ex)
                {
                    Console.WriteLine(ex);
                }
            });


            string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0];

            Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper);

            int delayInMs = 5;

            // Delay behavior
            ProxyBehavior[] behaviors = new[]
            {
                DelayBehaviors.DelayAllResponsesIf(
                    delayInMs,
                    (session =>
                {
                    var body = session.GetRequestBodyAsString();

                    Console.WriteLine(body);

                    // Delete locking the head response
                    if (session.hostname.Contains(accountNameToTamper + ".") &&
                        session.HTTPMethodIs("PUT") &&
                        body.Contains("\"_rtable_Operation\":\"Replace\"") &&
                        body.Contains("\"_rtable_RowLock\":true") &&
                        body.Contains("\"_rtable_Tombstone\":true"))
                    {
                        // Signal upsert we locked the head, so it can continue ...
                        deleteLockedHead = true;
                        return(true);
                    }

                    return(false);
                })),
            };

            // Launch a delete
            using (new HttpMangler(false, behaviors))
            {
                try
                {
                    var table = new ReplicatedTable(this.repTable.TableName, this.configurationService);

                    // Retrieve entity
                    TableOperation retrieveOperation = TableOperation.Retrieve <SampleRTableEntity>(entry.PartitionKey, entry.RowKey);
                    TableResult    retrieveResult    = table.Execute(retrieveOperation);

                    Assert.IsNotNull(retrieveResult, "retrieveResult = null");
                    Assert.AreEqual((int)HttpStatusCode.OK, retrieveResult.HttpStatusCode, "retrieveResult.HttpStatusCode mismatch");
                    Assert.IsNotNull((SampleRTableEntity)retrieveResult.Result, "Retrieve: customer = null");


                    // Delete entity
                    TableOperation deleteOperation = TableOperation.Delete((SampleRTableEntity)retrieveResult.Result);
                    TableResult    deleteResult    = table.Execute(deleteOperation);

                    Assert.IsNotNull(deleteResult, "deleteResult = null");
                    Assert.AreEqual((int)HttpStatusCode.NoContent, deleteResult.HttpStatusCode, "deleteResult.HttpStatusCode mismatch");
                }
                catch (AggregateException ex)
                {
                    Console.WriteLine(ex);
                }
            }

            // wait on upsert to finish ...
            upsertTask.Wait();

            // Upsert suceeded?
            Assert.AreEqual((int)HttpStatusCode.NoContent, upsertResult.HttpStatusCode, "Upsert expected to suceed!");
            SampleRTableEntity upsertedEntity = this.rtableWrapper.ReadEntity(entityPartitionKey, entityRowKey);

            Assert.NotNull(upsertedEntity, "upsert should succeed");
            Assert.AreEqual(upsertedEntity.Message, "upsert message", "upsert should succeeded");
        }
Ejemplo n.º 13
0
        public void A00TwoInsertOrReplaceCallsConflictingOnTheHead()
        {
            this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);

            string entityPartitionKey = "jobType-DelayInsertOrReplaceRowHeadTest";
            string entityRowKey       = "jobId-DelayInsertOrReplaceRowHeadTest";

            this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);

            // Insert one entry
            Console.WriteLine("Inserting entry ...");
            var entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, "insert message");

            this.rtableWrapper.InsertRow(entry);

            string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0];

            Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper);

            int  delayInMs = 3000;
            bool firstWritterInitiatingCommit  = false;
            bool secondWritterTriedLockingHead = false;

            // Delay behavior
            ProxyBehavior[] behaviors = new[]
            {
                // Writter-1 tampering
                TamperBehaviors.TamperAllRequestsIf(
                    (session =>
                {
                    int iter = 0;

                    // Signal Writter-2
                    firstWritterInitiatingCommit = true;

                    // Blobk commit to head ... until Writter-2 try to lock the head
                    while (!secondWritterTriedLockingHead)
                    {
                        Console.WriteLine("Writter-1 waiting on Writter-2 to try to lock the Head (#{0})", iter);
                        Thread.Sleep(delayInMs);

                        if (++iter > 10)
                        {
                            break;
                        }
                    }
                }),
                    (session =>
                {
                    var body = session.GetRequestBodyAsString();

                    // Writter-1 committing to head
                    if (session.hostname.Contains(accountNameToTamper + ".") &&
                        session.HTTPMethodIs("PUT") &&
                        body.Contains("\"_rtable_Operation\":\"Replace\"") &&
                        body.Contains("\"_rtable_RowLock\":false") &&
                        body.Contains("\"Message\":\"upsert message 0\"")
                        )
                    {
                        return(true);
                    }

                    return(false);
                })),

                // Writter-2 tampering
                TamperBehaviors.TamperAllRequestsIf(
                    (session =>
                {
                    // Block till Writter-1 issues a commit to head
                    while (!firstWritterInitiatingCommit)
                    {
                        Console.WriteLine("Writter-2 waiting on Writter-1 to issue a commit to head");
                        Thread.Sleep(delayInMs);
                    }
                }),
                    (session =>
                {
                    var body = session.GetRequestBodyAsString();

                    // Writter-2 locking the head
                    if (session.hostname.Contains(accountNameToTamper + ".") &&
                        session.HTTPMethodIs("PUT") &&
                        body.Contains("\"_rtable_Operation\":\"Replace\"") &&
                        body.Contains("\"_rtable_RowLock\":true") &&
                        body.Contains("\"Message\":\"upsert message 1\"")
                        )
                    {
                        return(true);
                    }

                    return(false);
                })),

                // Delay Writter-2 lock-to-the-head's response, so Writter-1 can continue with its commit.
                DelayBehaviors.DelayAllResponsesIf(
                    delayInMs,
                    (session =>
                {
                    var body = session.GetRequestBodyAsString();

                    // Writter-2 locking the head response
                    if (session.hostname.Contains(accountNameToTamper + ".") &&
                        session.HTTPMethodIs("PUT") &&
                        body.Contains("\"_rtable_Operation\":\"Replace\"") &&
                        body.Contains("\"_rtable_RowLock\":true") &&
                        body.Contains("\"Message\":\"upsert message 1\"")
                        )
                    {
                        // Signal Writter-1 so it can continue with commit to head
                        secondWritterTriedLockingHead = true;
                        return(true);
                    }

                    return(false);
                })),
            };

            // Launch 2 concurrent Upserts
            var results = new TableResult[2];

            using (new HttpMangler(false, behaviors))
            {
                Parallel.For(0, 2, (index) =>
                {
                    entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, string.Format("upsert message {0}", index));

                    try
                    {
                        var table = new ReplicatedTable(this.repTable.TableName, this.configurationService);

                        TableOperation operation = TableOperation.InsertOrReplace(entry);
                        results[index]           = table.Execute(operation);
                    }
                    catch (AggregateException ex)
                    {
                        Console.WriteLine(ex);
                    }
                });
            }

            // Writter-1 suceed?
            Assert.AreEqual((int)HttpStatusCode.NoContent, results[0].HttpStatusCode, "Writter-1 expected to suceed!");

            // Writter-2 suceeded?
            Assert.AreEqual((int)HttpStatusCode.NoContent, results[1].HttpStatusCode, "Writter-2 expected to suceed!");

            // Writter-2 upsert succeeded
            SampleRTableEntity upsertedEntity = this.rtableWrapper.ReadEntity(entityPartitionKey, entityRowKey);

            Assert.NotNull(upsertedEntity, "Writter-2 upsert succeeded");
            Assert.AreEqual(upsertedEntity.Message, string.Format("upsert message {0}", 1), "Writter-2 upsert succeeded");
        }
Ejemplo n.º 14
0
        public void A00DelayTwoConflictingInsertOrReplaceCalls()
        {
            this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);

            string entityPartitionKey = "jobType-DelayInsertOrReplaceRowHeadTest";
            string entityRowKey       = "jobId-DelayInsertOrReplaceRowHeadTest";

            this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);

            string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0];

            Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper);

            int  delayInMs              = 3000;
            int  insertRequestCount     = 0;
            int  conflictResponseCount  = 0;
            bool secondUpsertConflicted = false;
            int  failedCallIndex        = -1;

            // Delay behavior
            ProxyBehavior[] behaviors = new[]
            {
                // Delay Insert calls so they end up conflicting
                TamperBehaviors.TamperAllRequestsIf(
                    (session =>
                {
                    Interlocked.Increment(ref insertRequestCount);

                    while (insertRequestCount != 2)
                    {
                        Console.WriteLine("insertRequestCount={0}. Waiting on count to reach 2 ...", insertRequestCount);
                        Thread.Sleep(delayInMs);
                    }
                }),
                    (session =>
                {
                    if (session.hostname.Contains(accountNameToTamper + ".") &&
                        session.HTTPMethodIs("POST") &&
                        session.GetRequestBodyAsString().Contains("\"_rtable_Operation\":\"Insert\""))
                    {
                        return(true);
                    }

                    return(false);
                })),

                // Delay conflict response
                DelayBehaviors.DelayAllResponsesIf(
                    delayInMs,
                    (session =>
                {
                    if (session.hostname.Contains(accountNameToTamper + ".") &&
                        session.GetRequestBodyAsString().Contains("\"_rtable_Operation\":\"Insert\""))
                    {
                        if (session.responseCode == (int)HttpStatusCode.Conflict)
                        {
                            Interlocked.Increment(ref conflictResponseCount);

                            return(true);
                        }
                    }

                    return(false);
                })),
            };

            using (new HttpMangler(false, behaviors))
            {
                Parallel.For(0, 2, (index) =>
                {
                    var entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, string.Format("upsert message {0}", index));
                    try
                    {
                        this.rtableWrapper.InsertOrReplaceRow(entry);
                    }
                    catch (RTableConflictException)
                    {
                        if (secondUpsertConflicted)
                        {
                            // should never reach here
                            throw;
                        }

                        // That's possible, but that's the Replace step of upsert which conflicted with ongoing write
                        // can't do anything, client should retry on conflict
                        secondUpsertConflicted = true;
                    }
                });
            }

            // got 2 inserts?
            Assert.AreEqual(2, insertRequestCount, "Two insert calls expected!");

            // got one conflict?
            Assert.AreEqual(1, conflictResponseCount, "One conflict response expected!");

            // at least one upsert would have succeeded
            SampleRTableEntity upsertedEntity = this.rtableWrapper.ReadEntity(entityPartitionKey, entityRowKey);

            Assert.NotNull(upsertedEntity, "at least one upsert should have succeeded");

            // second upsert failed?
            if (secondUpsertConflicted)
            {
                Assert.AreEqual(1, upsertedEntity._rtable_Version, "one upsert succeeded so version should be = 1");
                Assert.AreEqual(upsertedEntity.Message, string.Format("upsert message {0}", (1 - failedCallIndex)));
            }
            else
            {
                Assert.AreEqual(2, upsertedEntity._rtable_Version, "both upserts succeeded so version should be = 2");
            }


            // After recovery from delay, confirm that we can update the row.
            //this.ExecuteReplaceRowAndValidate(entityPartitionKey, entityRowKey);
        }
Ejemplo n.º 15
0
        public void TableQueryableCreateQueryNoPartitionKey()
        {
            Thread.Sleep(10000);

            string          tableName   = this.GenerateRandomTableName();
            ReplicatedTable localRTable = new ReplicatedTable(tableName, this.configurationService);

            localRTable.CreateIfNotExists();
            RTableWrapperForSampleRTableEntity localRTableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(localRTable);

            CloudTableClient tableClient = localRTable.GetTailTableClient();
            CloudTable       table       = tableClient.GetTableReference(localRTable.TableName);

            string pk = "0";

            try
            {
                try
                {
                    TableBatchOperation batch = new TableBatchOperation();

                    for (int j = 0; j < 10; j++)
                    {
                        BaseEntity ent = GenerateRandomEntity(pk);
                        ent.RowKey = string.Format("{0:0000}", j);
                        batch.Insert(ent);
                    }

                    localRTable.ExecuteBatch(batch);
                }
                catch (Exception ex)
                {
                    Console.WriteLine("Exception during test case init {0}", ex.ToString());
                    throw;
                }

                try
                {
                    pk = "1";
                    TableBatchOperation batch = new TableBatchOperation();

                    for (int j = 0; j < 10; j++)
                    {
                        BaseEntity ent = GenerateRandomEntity(pk);
                        ent.RowKey = string.Format("{0:0000}", j);
                        batch.Insert(ent);
                    }

                    localRTable.ExecuteBatch(batch);
                }
                catch (Exception ex)
                {
                    Console.WriteLine("Exception during test case init {0}", ex.ToString());
                    throw;
                }

                IQueryable <BaseEntity> tableQuery  = table.CreateQuery <BaseEntity>();
                IQueryable <BaseEntity> rtableQuery = localRTable.CreateQuery <BaseEntity>();

                var list = tableQuery.AsEnumerable();

                int tableCount  = 0;
                int rtableCount = 0;
                foreach (BaseEntity ent in list)
                {
                    tableCount++;
                }
                foreach (BaseEntity ent in rtableQuery.ToList())
                {
                    rtableCount++;

                    Assert.IsTrue(ent.ETag != ent._rtable_Version.ToString(), "ETag is not virtualized when using CreateQuery()");
                }

                Assert.IsTrue(tableCount == rtableCount, "Query counts are different");
                Assert.IsTrue(tableCount == 20, "Query counts are different");

                // But, with "CreateReplicatedQuery" ETag is virtualized
                IQueryable <BaseEntity> virtualizedRtableQuery = localRTable.CreateReplicatedQuery <BaseEntity>();

                foreach (BaseEntity ent in virtualizedRtableQuery.ToList())
                {
                    Assert.IsTrue(ent._rtable_Version == 0);
                    Assert.IsTrue(ent.ETag == ent._rtable_Version.ToString(), "ETag is virtualized when using CreateReplicatedQuery()");

                    ent.A += "`";

                    // Update should go fine since ETag is virtualized
                    TableOperation operation = TableOperation.Replace(ent);
                    TableResult    result    = localRTable.Execute(operation);
                    Assert.IsTrue(result != null && result.HttpStatusCode == (int)HttpStatusCode.NoContent);
                }

                virtualizedRtableQuery = localRTable.CreateReplicatedQuery <BaseEntity>();

                foreach (BaseEntity ent in virtualizedRtableQuery.ToList())
                {
                    Assert.IsTrue(ent._rtable_Version == 1);
                    Assert.IsTrue(ent.ETag == ent._rtable_Version.ToString(), "ETag is virtualized when using CreateReplicatedQuery()");
                }
            }
            catch (Exception e)
            {
                Console.WriteLine("Error during query processing: {0}", e.ToString());
            }
            finally
            {
                localRTable.DeleteIfExists();
            }
        }
Ejemplo n.º 16
0
        public void TableQueryableCreateQueryNoPartitionKey()
        {
            Thread.Sleep(10000);

            string          tableName   = this.GenerateRandomTableName();
            ReplicatedTable localRTable = new ReplicatedTable(tableName, this.configurationService);

            localRTable.CreateIfNotExists();
            RTableWrapperForSampleRTableEntity localRTableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(localRTable);

            CloudTableClient tableClient = localRTable.GetTailTableClient();
            CloudTable       table       = tableClient.GetTableReference(localRTable.TableName);

            string pk = "0";

            try
            {
                try
                {
                    TableBatchOperation batch = new TableBatchOperation();

                    for (int j = 0; j < 10; j++)
                    {
                        BaseEntity ent = GenerateRandomEntity(pk);
                        ent.RowKey = string.Format("{0:0000}", j);
                        batch.Insert(ent);
                    }

                    localRTable.ExecuteBatch(batch);
                }
                catch (Exception ex)
                {
                    Console.WriteLine("Exception during test case init {0}", ex.ToString());
                    throw;
                }

                try
                {
                    pk = "1";
                    TableBatchOperation batch = new TableBatchOperation();

                    for (int j = 0; j < 10; j++)
                    {
                        BaseEntity ent = GenerateRandomEntity(pk);
                        ent.RowKey = string.Format("{0:0000}", j);
                        batch.Insert(ent);
                    }

                    localRTable.ExecuteBatch(batch);
                }
                catch (Exception ex)
                {
                    Console.WriteLine("Exception during test case init {0}", ex.ToString());
                    throw;
                }

                IQueryable <BaseEntity> tableQuery  = table.CreateQuery <BaseEntity>();
                IQueryable <BaseEntity> rtableQuery = localRTable.CreateQuery <BaseEntity>();

                var list = tableQuery.AsEnumerable();

                int tableCount  = 0;
                int rtableCount = 0;
                foreach (BaseEntity ent in list)
                {
                    tableCount++;
                }
                foreach (BaseEntity ent in rtableQuery.ToList())
                {
                    rtableCount++;
                }

                Assert.IsTrue(tableCount == rtableCount, "Query counts are different");
                Assert.IsTrue(tableCount == 20, "Query counts are different");
            }
            catch (Exception e)
            {
                Console.WriteLine("Error during query processing: {0}", e.ToString());
            }
            finally
            {
                localRTable.DeleteIfExists();
            }
        }