public void ReplaceRowTest()
{
this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);
string entityPartitionKey = "jobType-TamperReplaceRowHeadTest";
string entityRowKey = "jobId-TamperReplaceRowHeadTest";
this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);
int targetStorageAccount = 1;
TargetRTableWrapperApi <SampleRTableEntity> targetApi = this.rtableWrapper.ReplaceRow;
bool targetApiExpectedToFail = false;
bool checkOriginalEntityUnchanged = false;
bool checkStorageAccountsConsistent = false;
DateTime httpManglerStartTime;
Task task = Task.Run(() =>
this.SetupAndRunDelayTableBehavior(
entityPartitionKey,
entityRowKey,
targetStorageAccount,
targetApi,
targetApiExpectedToFail,
checkOriginalEntityUnchanged,
checkStorageAccountsConsistent,
out httpManglerStartTime));
task.Wait();
}
public void B00TamperReadRowInnerTest()
{
this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);
string entityPartitionKey = "jobType-TamperReadRowInnerTest";
string entityRowKey = "jobId-TamperReadRowInnerTest";
this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);
int targetStorageAccount = 1;
bool targetApiExpectedToFail = false;
bool checkOriginalEntityUnchanged = true;
bool checkStorageAccountsConsistent = true;
// Try this many times. It should not fail.
for (int i = 0; i < 3 * this.actualStorageAccountsUsed.Count; i++)
{
this.SetupAndRunTamperTableBehavior(
entityPartitionKey,
entityRowKey,
targetStorageAccount,
this.rtableWrapper.ReadEntity,
targetApiExpectedToFail,
checkOriginalEntityUnchanged,
checkStorageAccountsConsistent);
Console.WriteLine("\n ----------------------------------- Finished iteration {0}", i);
}
}
public void A00TamperReplaceRowHeadTest()
{
this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);
string entityPartitionKey = "jobType-TamperReplaceRowHeadTest";
string entityRowKey = "jobId-TamperReplaceRowHeadTest";
this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);
int targetStorageAccount = 0;
TargetRTableWrapperApi <SampleRTableEntity> targetApi = this.rtableWrapper.ReplaceRow;
bool targetApiExpectedToFail = true;
bool checkOriginalEntityUnchanged = true;
bool checkStorageAccountsConsistent = true;
DateTime httpManglerStartTime;
this.SetupAndRunTamperTableBehavior(
entityPartitionKey,
entityRowKey,
targetStorageAccount,
targetApi,
targetApiExpectedToFail,
checkOriginalEntityUnchanged,
checkStorageAccountsConsistent,
out httpManglerStartTime);
// After recovery from short outage, confirm that we can update the row.
this.ExecuteReplaceRowAndValidate(entityPartitionKey, entityRowKey);
}
public void A00DelayReplaceRowHeadTest()
{
this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);
string entityPartitionKey = "jobType-DelayReplaceRowHeadTest";
string entityRowKey = "jobId-DelayReplaceRowHeadTest";
this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);
int targetStorageAccount = 0;
TargetRTableWrapperApi <SampleRTableEntity> targetApi = this.rtableWrapper.ReplaceRow;
bool targetApiExpectedToFail = false;
int delayInMs = 3000;
this.SetupAndRunDelayTableBehavior(
entityPartitionKey,
entityRowKey,
targetStorageAccount,
delayInMs,
targetApi,
targetApiExpectedToFail);
// After recovery from delay, confirm that we can update the row.
this.ExecuteReplaceRowAndValidate(entityPartitionKey, entityRowKey);
}
public void C00TamperReplaceRowTailTest()
{
this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);
string entityPartitionKey = "jobType-TamperReplaceRowTailTest";
string entityRowKey = "jobId-TamperReplaceRowTailTest";
this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);
int targetStorageAccount = this.actualStorageAccountsUsed.Count - 1;
TargetRTableWrapperApi <SampleRTableEntity> targetApi = this.rtableWrapper.ReplaceRow;
bool targetApiExpectedToFail = true;
bool checkOriginalEntityUnchanged = true;
bool checkStorageAccountsConsistent = false;
DateTime httpManglerStartTime;
this.SetupAndRunTamperTableBehavior(
entityPartitionKey,
entityRowKey,
targetStorageAccount,
targetApi,
targetApiExpectedToFail,
checkOriginalEntityUnchanged,
checkStorageAccountsConsistent,
out httpManglerStartTime);
// After recovery from short outage, sleep some time to wait for entity to be unlocked, then confirm that we can update the row.
this.SleepUntilRowLockHasExpired(httpManglerStartTime);
this.ExecuteReplaceRowAndValidate(entityPartitionKey, entityRowKey);
}
public void B00DelayDeleteRowInnerTest()
{
this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);
string entityPartitionKey = "jobType-DelayDeleteRowInnerTest";
string entityRowKey = "jobId-DelayDeleteRowInnerTest";
this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);
int targetStorageAccount = 1;
TargetRTableWrapperApi <SampleRTableEntity> targetApi = this.rtableWrapper.DeleteRow;
bool targetApiExpectedToFail = false;
int delayInMs = 3000;
this.SetupAndRunDelayTableBehavior(
entityPartitionKey,
entityRowKey,
targetStorageAccount,
delayInMs,
targetApi,
targetApiExpectedToFail);
if (targetApiExpectedToFail)
{
// After recovery from delay, confirm that we can delete the row.
this.ExecuteDeleteRowAndValidate(entityPartitionKey, entityRowKey);
}
else
{
// After recovery from delay, confirm the entity is gone.
this.ExecuteReadRowAndValidateNotExist(entityPartitionKey, entityRowKey);
}
// Confirm we can create entity using the same partition and row keys again.
this.ExecuteCreateRowAndValidate(entityPartitionKey, entityRowKey);
}
/// <summary>
/// Call this function to modify the contents of the RTable configuration blob to use the specified viewId
/// and convertXStoreTableMode and update the appropriate wrapper.
/// </summary>
/// <param name="updatedViewId"></param>
protected void RefreshRTableEnvJsonConfigBlob(long updatedViewId, bool convertXStoreTableMode = false,
int readViewHeadIndex = 0, List <int> blobIndexes = null, bool waitForConfig = true)
{
if (blobIndexes == null)
{
blobIndexes = new List <int> {
0
};
}
Console.WriteLine(
"Calling RefreshRTableEnvJsonConfigBlob() with updatedViewId={0} convertXStoreTableMode={1} readViewHeadIndex={2}",
updatedViewId, convertXStoreTableMode, readViewHeadIndex);
this.ModifyConfigurationBlob(updatedViewId, convertXStoreTableMode, readViewHeadIndex, blobIndexes);
this.repTable = new ReplicatedTable(this.repTable.TableName, this.configurationService);
this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);
if (!waitForConfig)
{
return;
}
while (this.configurationWrapper.GetWriteView().ViewId != updatedViewId)
{
Console.WriteLine("Sleeping {0} seconds for new viewId to take effect...",
Constants.LeaseDurationInSec / 2);
Thread.Sleep((Constants.LeaseDurationInSec / 2) * 1000);
}
}
/// <summary>
/// Setup the RTable Env in order to run unit tests.
/// </summary>
/// <param name="tableName">Name of the RTable. If string.Empty or null, then use this.rtableTestConfiguration.RTableInformation.RTableName</param>
/// <param name="useHttps">When using HttpMangler, set this to false</param>
/// <param name="viewIdString">_rtable_ViewId used in the RTable config blob. Set to string.Empty or null to use the _rtable_ViewId value in the xml config</param>
/// <param name="actualStorageAccountsToBeUsed">Specify how/which Storage accounts in the config.xml will be used to construct the RTable. Set to null to use all accounts in xml. E.g., {0,1}</param>
protected void SetupRTableEnv(
string tableName = "",
bool useHttps = true,
string viewIdString = "",
List <int> actualStorageAccountsToBeUsed = null,
bool convertXStoreTableMode = false,
int numberOfBlobs = 0)
{
int numberOfStorageAccounts = this.rtableTestConfiguration.StorageInformation.AccountNames.Count();
// If user does not specify which storage accounts to construct the RTable, then use all the storage accounts in the xml.
this.actualStorageAccountsUsed = actualStorageAccountsToBeUsed;
if (this.actualStorageAccountsUsed == null || this.actualStorageAccountsUsed.Count == 0)
{
this.actualStorageAccountsUsed = new List <int>();
for (int i = 0; i < numberOfStorageAccounts; i++)
{
this.actualStorageAccountsUsed.Add(i);
}
}
this.UpdateConfigurationConstants();
// Upload RTable configuration to blob
int viewId = this.rtableTestConfiguration.RTableInformation.ViewId;
if (!string.IsNullOrEmpty(viewIdString))
{
viewId = int.Parse(viewIdString);
}
numberOfBlobs = numberOfBlobs == 0
? this.rtableTestConfiguration.StorageInformation.AccountKeys.Count()
: numberOfBlobs;
if (string.IsNullOrEmpty(tableName))
{
tableName = this.rtableTestConfiguration.RTableInformation.RTableName;
}
this.InitConnectionStringMap(useHttps);
InitialSetup(tableName, useHttps, viewId, convertXStoreTableMode, numberOfBlobs);
this.repTable = new ReplicatedTable(tableName, this.configurationService);
this.repTable.CreateIfNotExists();
this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);
this.cloudTableClients = new List <CloudTableClient>();
this.cloudTables = new List <CloudTable>();
for (int i = 0; i < this.actualStorageAccountsUsed.Count; i++)
{
CloudTableClient cloudBloblClient = repTable.GetReplicaTableClient(i);
this.cloudTableClients.Add(cloudBloblClient);
this.cloudTables.Add(cloudBloblClient.GetTableReference(repTable.TableName));
}
}
public void B00TamperSkipDeleteRowInnerTest()
{
this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);
string entityPartitionKey = "jobType-TamperSkipDeleteRowInnerTest";
string entityRowKey = "jobId-TamperSkipDeleteRowInnerTest";
this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);
int targetStorageAccount = 1;
TargetRTableWrapperApi <SampleRTableEntity> targetApi = this.rtableWrapper.DeleteRow;
bool targetApiExpectedToFail = true;
int skipInitialiSessions = 2;
bool checkOriginalEntityUnchanged = false;
bool checkStorageAccountsConsistent = false;
DateTime httpManglerStartTime;
this.SetupAndRunTamperTableBehavior(
entityPartitionKey,
entityRowKey,
targetStorageAccount,
targetApi,
targetApiExpectedToFail,
checkOriginalEntityUnchanged,
checkStorageAccountsConsistent,
out httpManglerStartTime,
skipInitialiSessions);
if (targetApiExpectedToFail)
{
// After recovery from outage, sleep some time to wait for entity to be unlocked, then confirm that we can delete the row.
this.SleepUntilRowLockHasExpired(httpManglerStartTime);
this.ExecuteDeleteRowAndValidate(entityPartitionKey, entityRowKey);
}
else
{
// After recovery from outage, confirm the entity is gone.
this.ExecuteReadRowAndValidateNotExist(entityPartitionKey, entityRowKey);
}
// Confirm we can create entity using the same partition and row keys again.
this.ExecuteCreateRowAndValidate(entityPartitionKey, entityRowKey);
}
public void A00TamperDeleteRowHeadTest()
{
this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);
string entityPartitionKey = "jobType-TamperDeleteRowHeadTest";
string entityRowKey = "jobId-TamperDeleteRowHeadTest";
this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);
int targetStorageAccount = 0;
TargetRTableWrapperApi <SampleRTableEntity> targetApi = this.rtableWrapper.DeleteRow;
bool targetApiExpectedToFail = true;
bool checkOriginalEntityUnchanged = true;
bool checkStorageAccountsConsistent = true;
this.SetupAndRunTamperTableBehavior(
entityPartitionKey,
entityRowKey,
targetStorageAccount,
targetApi,
targetApiExpectedToFail,
checkOriginalEntityUnchanged,
checkStorageAccountsConsistent);
if (targetApiExpectedToFail)
{
// After recovery from outage, confirm that we can delete the row.
this.ExecuteDeleteRowAndValidate(entityPartitionKey, entityRowKey);
}
else
{
// After recovery from outage, confirm the entity is gone.
this.ExecuteReadRowAndValidateNotExist(entityPartitionKey, entityRowKey);
}
// Confirm we can create entity using the same partition and row keys again.
this.ExecuteCreateRowAndValidate(entityPartitionKey, entityRowKey);
}
public void RepairMidAnUpdate()
{
this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);
// Not-Stable chain
// Reconfigure RTable so Head is WriteOnly.
View view = this.configurationService.GetTableView(this.repTable.TableName);
ReplicatedTableConfiguration config;
ReplicatedTableQuorumReadResult readStatus = this.configurationService.RetrieveConfiguration(out config);
Assert.IsTrue(readStatus.Code == ReplicatedTableQuorumReadCode.Success);
// Set Head as WriteOnly mode
ReplicatedTableConfigurationStore viewConfg = config.GetView(view.Name);
viewConfg.ReplicaChain[0].Status = ReplicaStatus.WriteOnly;
config.SetView(view.Name, viewConfg);
// Upload RTable config back
this.configurationService.UpdateConfiguration(config);
// Sanity: Replicated mode and chain Not-Stable
view = this.configurationService.GetTableView(this.repTable.TableName);
Assert.IsTrue(view != null && view.Chain.Count > 1, "Two replicas should be used.");
Assert.IsFalse(view.IsStable);
// Insert one entry
Console.WriteLine("Inserting entry ...");
string entityPartitionKey = "jobType-RepairMidAnUpdate-Replace";
string entityRowKey = "jobId-RepairMidAnUpdate-Replace";
var entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, "message");
this.rtableWrapper.InsertRow(entry);
// 1 - Launch a RepairRow task in wait mode ...
bool triggerRepair = false;
bool repaireDone = false;
bool headAfterRepairWasLocked = false;
TableResult repairResult = null;
Task.Run(() =>
{
ReplicatedTable repairTable = new ReplicatedTable(this.repTable.TableName, this.configurationService);
while (!triggerRepair)
{
Thread.Sleep(5);
}
Console.WriteLine("RepairRow started ...");
repairResult = repairTable.RepairRow(entry.PartitionKey, entry.RowKey, null);
Console.WriteLine("RepairRow completed with HttpStatus={0}", repairResult == null ? "NULL" : repairResult.HttpStatusCode.ToString());
// Check the entry at the Head is still locked i.e. RepairRow was NOP
headAfterRepairWasLocked = HeadIsLocked(entry);
Console.WriteLine("Signal the commit to Head job");
repaireDone = true;
});
// 2 - Configure Mangler ...
string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0];
Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper);
ProxyBehavior[] behaviors = new[]
{
TamperBehaviors.TamperAllRequestsIf(
(session =>
{
Console.WriteLine("Delaying commit to the Head ... => signal RepairRow job");
// Let RepairRow task go through
triggerRepair = true;
int iter = 0;
while (!repaireDone)
{
// TODO: break the loop after couple of iteration ...
Thread.Sleep(100);
Console.WriteLine("Waiting on RepairRow to finish ({0}) ...", ++iter);
}
Console.WriteLine("Request a commit to the head");
}),
(session =>
{
// Commit on head i.e. a PUT with RowLock == false
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.HTTPMethodIs("PUT") &&
session.GetRequestBodyAsString().Contains("\"_rtable_RowLock\":false"))
{
return(true);
}
return(false);
}))
};
using (new HttpMangler(false, behaviors))
{
Console.WriteLine("Updating entry ...");
entry = this.rtableWrapper.FindRow(entry.PartitionKey, entry.RowKey);
entry.Message = "updated message";
this.rtableWrapper.ReplaceRow(entry);
}
Assert.IsTrue(triggerRepair);
Assert.IsTrue(repairResult != null && repairResult.HttpStatusCode == (int)HttpStatusCode.OK, "Repair failed.");
Assert.IsTrue(repaireDone);
Assert.IsTrue(headAfterRepairWasLocked);
Console.WriteLine("DONE. Test passed.");
}
public void A00InsertOrReplaceCallConflictingWithDeleteOnTheHead()
{
this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);
string entityPartitionKey = "jobType-DelayInsertOrReplaceWhileDelete";
string entityRowKey = "jobType-DelayInsertOrReplaceWhileDelete";
this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);
// Insert one entry
Console.WriteLine("Inserting entry ...");
var entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, "insert message");
this.rtableWrapper.InsertRow(entry);
// 1 - Launch an Upsert task in wait mode ...
bool deleteLockedHead = false;
TableResult upsertResult = null;
var upsertTask = Task.Run(() =>
{
while (!deleteLockedHead)
{
Thread.Sleep(5);
}
try
{
entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, "upsert message");
var table = new ReplicatedTable(this.repTable.TableName, this.configurationService);
TableOperation upserOperation = TableOperation.InsertOrReplace(entry);
Console.WriteLine("Upsert started ...");
upsertResult = table.Execute(upserOperation);
Console.WriteLine("Upsert completed with HttpStatus={0}", upsertResult == null ? "NULL" : upsertResult.HttpStatusCode.ToString());
}
catch (AggregateException ex)
{
Console.WriteLine(ex);
}
});
string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0];
Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper);
int delayInMs = 5;
// Delay behavior
ProxyBehavior[] behaviors = new[]
{
DelayBehaviors.DelayAllResponsesIf(
delayInMs,
(session =>
{
var body = session.GetRequestBodyAsString();
Console.WriteLine(body);
// Delete locking the head response
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.HTTPMethodIs("PUT") &&
body.Contains("\"_rtable_Operation\":\"Replace\"") &&
body.Contains("\"_rtable_RowLock\":true") &&
body.Contains("\"_rtable_Tombstone\":true"))
{
// Signal upsert we locked the head, so it can continue ...
deleteLockedHead = true;
return(true);
}
return(false);
})),
};
// Launch a delete
using (new HttpMangler(false, behaviors))
{
try
{
var table = new ReplicatedTable(this.repTable.TableName, this.configurationService);
// Retrieve entity
TableOperation retrieveOperation = TableOperation.Retrieve <SampleRTableEntity>(entry.PartitionKey, entry.RowKey);
TableResult retrieveResult = table.Execute(retrieveOperation);
Assert.IsNotNull(retrieveResult, "retrieveResult = null");
Assert.AreEqual((int)HttpStatusCode.OK, retrieveResult.HttpStatusCode, "retrieveResult.HttpStatusCode mismatch");
Assert.IsNotNull((SampleRTableEntity)retrieveResult.Result, "Retrieve: customer = null");
// Delete entity
TableOperation deleteOperation = TableOperation.Delete((SampleRTableEntity)retrieveResult.Result);
TableResult deleteResult = table.Execute(deleteOperation);
Assert.IsNotNull(deleteResult, "deleteResult = null");
Assert.AreEqual((int)HttpStatusCode.NoContent, deleteResult.HttpStatusCode, "deleteResult.HttpStatusCode mismatch");
}
catch (AggregateException ex)
{
Console.WriteLine(ex);
}
}
// wait on upsert to finish ...
upsertTask.Wait();
// Upsert suceeded?
Assert.AreEqual((int)HttpStatusCode.NoContent, upsertResult.HttpStatusCode, "Upsert expected to suceed!");
SampleRTableEntity upsertedEntity = this.rtableWrapper.ReadEntity(entityPartitionKey, entityRowKey);
Assert.NotNull(upsertedEntity, "upsert should succeed");
Assert.AreEqual(upsertedEntity.Message, "upsert message", "upsert should succeeded");
}
public void A00TwoInsertOrReplaceCallsConflictingOnTheHead()
{
this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);
string entityPartitionKey = "jobType-DelayInsertOrReplaceRowHeadTest";
string entityRowKey = "jobId-DelayInsertOrReplaceRowHeadTest";
this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);
// Insert one entry
Console.WriteLine("Inserting entry ...");
var entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, "insert message");
this.rtableWrapper.InsertRow(entry);
string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0];
Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper);
int delayInMs = 3000;
bool firstWritterInitiatingCommit = false;
bool secondWritterTriedLockingHead = false;
// Delay behavior
ProxyBehavior[] behaviors = new[]
{
// Writter-1 tampering
TamperBehaviors.TamperAllRequestsIf(
(session =>
{
int iter = 0;
// Signal Writter-2
firstWritterInitiatingCommit = true;
// Blobk commit to head ... until Writter-2 try to lock the head
while (!secondWritterTriedLockingHead)
{
Console.WriteLine("Writter-1 waiting on Writter-2 to try to lock the Head (#{0})", iter);
Thread.Sleep(delayInMs);
if (++iter > 10)
{
break;
}
}
}),
(session =>
{
var body = session.GetRequestBodyAsString();
// Writter-1 committing to head
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.HTTPMethodIs("PUT") &&
body.Contains("\"_rtable_Operation\":\"Replace\"") &&
body.Contains("\"_rtable_RowLock\":false") &&
body.Contains("\"Message\":\"upsert message 0\"")
)
{
return(true);
}
return(false);
})),
// Writter-2 tampering
TamperBehaviors.TamperAllRequestsIf(
(session =>
{
// Block till Writter-1 issues a commit to head
while (!firstWritterInitiatingCommit)
{
Console.WriteLine("Writter-2 waiting on Writter-1 to issue a commit to head");
Thread.Sleep(delayInMs);
}
}),
(session =>
{
var body = session.GetRequestBodyAsString();
// Writter-2 locking the head
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.HTTPMethodIs("PUT") &&
body.Contains("\"_rtable_Operation\":\"Replace\"") &&
body.Contains("\"_rtable_RowLock\":true") &&
body.Contains("\"Message\":\"upsert message 1\"")
)
{
return(true);
}
return(false);
})),
// Delay Writter-2 lock-to-the-head's response, so Writter-1 can continue with its commit.
DelayBehaviors.DelayAllResponsesIf(
delayInMs,
(session =>
{
var body = session.GetRequestBodyAsString();
// Writter-2 locking the head response
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.HTTPMethodIs("PUT") &&
body.Contains("\"_rtable_Operation\":\"Replace\"") &&
body.Contains("\"_rtable_RowLock\":true") &&
body.Contains("\"Message\":\"upsert message 1\"")
)
{
// Signal Writter-1 so it can continue with commit to head
secondWritterTriedLockingHead = true;
return(true);
}
return(false);
})),
};
// Launch 2 concurrent Upserts
var results = new TableResult[2];
using (new HttpMangler(false, behaviors))
{
Parallel.For(0, 2, (index) =>
{
entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, string.Format("upsert message {0}", index));
try
{
var table = new ReplicatedTable(this.repTable.TableName, this.configurationService);
TableOperation operation = TableOperation.InsertOrReplace(entry);
results[index] = table.Execute(operation);
}
catch (AggregateException ex)
{
Console.WriteLine(ex);
}
});
}
// Writter-1 suceed?
Assert.AreEqual((int)HttpStatusCode.NoContent, results[0].HttpStatusCode, "Writter-1 expected to suceed!");
// Writter-2 suceeded?
Assert.AreEqual((int)HttpStatusCode.NoContent, results[1].HttpStatusCode, "Writter-2 expected to suceed!");
// Writter-2 upsert succeeded
SampleRTableEntity upsertedEntity = this.rtableWrapper.ReadEntity(entityPartitionKey, entityRowKey);
Assert.NotNull(upsertedEntity, "Writter-2 upsert succeeded");
Assert.AreEqual(upsertedEntity.Message, string.Format("upsert message {0}", 1), "Writter-2 upsert succeeded");
}
public void A00DelayTwoConflictingInsertOrReplaceCalls()
{
this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);
string entityPartitionKey = "jobType-DelayInsertOrReplaceRowHeadTest";
string entityRowKey = "jobId-DelayInsertOrReplaceRowHeadTest";
this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);
string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0];
Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper);
int delayInMs = 3000;
int insertRequestCount = 0;
int conflictResponseCount = 0;
bool secondUpsertConflicted = false;
int failedCallIndex = -1;
// Delay behavior
ProxyBehavior[] behaviors = new[]
{
// Delay Insert calls so they end up conflicting
TamperBehaviors.TamperAllRequestsIf(
(session =>
{
Interlocked.Increment(ref insertRequestCount);
while (insertRequestCount != 2)
{
Console.WriteLine("insertRequestCount={0}. Waiting on count to reach 2 ...", insertRequestCount);
Thread.Sleep(delayInMs);
}
}),
(session =>
{
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.HTTPMethodIs("POST") &&
session.GetRequestBodyAsString().Contains("\"_rtable_Operation\":\"Insert\""))
{
return(true);
}
return(false);
})),
// Delay conflict response
DelayBehaviors.DelayAllResponsesIf(
delayInMs,
(session =>
{
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.GetRequestBodyAsString().Contains("\"_rtable_Operation\":\"Insert\""))
{
if (session.responseCode == (int)HttpStatusCode.Conflict)
{
Interlocked.Increment(ref conflictResponseCount);
return(true);
}
}
return(false);
})),
};
using (new HttpMangler(false, behaviors))
{
Parallel.For(0, 2, (index) =>
{
var entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, string.Format("upsert message {0}", index));
try
{
this.rtableWrapper.InsertOrReplaceRow(entry);
}
catch (RTableConflictException)
{
if (secondUpsertConflicted)
{
// should never reach here
throw;
}
// That's possible, but that's the Replace step of upsert which conflicted with ongoing write
// can't do anything, client should retry on conflict
secondUpsertConflicted = true;
}
});
}
// got 2 inserts?
Assert.AreEqual(2, insertRequestCount, "Two insert calls expected!");
// got one conflict?
Assert.AreEqual(1, conflictResponseCount, "One conflict response expected!");
// at least one upsert would have succeeded
SampleRTableEntity upsertedEntity = this.rtableWrapper.ReadEntity(entityPartitionKey, entityRowKey);
Assert.NotNull(upsertedEntity, "at least one upsert should have succeeded");
// second upsert failed?
if (secondUpsertConflicted)
{
Assert.AreEqual(1, upsertedEntity._rtable_Version, "one upsert succeeded so version should be = 1");
Assert.AreEqual(upsertedEntity.Message, string.Format("upsert message {0}", (1 - failedCallIndex)));
}
else
{
Assert.AreEqual(2, upsertedEntity._rtable_Version, "both upserts succeeded so version should be = 2");
}
// After recovery from delay, confirm that we can update the row.
//this.ExecuteReplaceRowAndValidate(entityPartitionKey, entityRowKey);
}
public void TableQueryableCreateQueryNoPartitionKey()
{
Thread.Sleep(10000);
string tableName = this.GenerateRandomTableName();
ReplicatedTable localRTable = new ReplicatedTable(tableName, this.configurationService);
localRTable.CreateIfNotExists();
RTableWrapperForSampleRTableEntity localRTableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(localRTable);
CloudTableClient tableClient = localRTable.GetTailTableClient();
CloudTable table = tableClient.GetTableReference(localRTable.TableName);
string pk = "0";
try
{
try
{
TableBatchOperation batch = new TableBatchOperation();
for (int j = 0; j < 10; j++)
{
BaseEntity ent = GenerateRandomEntity(pk);
ent.RowKey = string.Format("{0:0000}", j);
batch.Insert(ent);
}
localRTable.ExecuteBatch(batch);
}
catch (Exception ex)
{
Console.WriteLine("Exception during test case init {0}", ex.ToString());
throw;
}
try
{
pk = "1";
TableBatchOperation batch = new TableBatchOperation();
for (int j = 0; j < 10; j++)
{
BaseEntity ent = GenerateRandomEntity(pk);
ent.RowKey = string.Format("{0:0000}", j);
batch.Insert(ent);
}
localRTable.ExecuteBatch(batch);
}
catch (Exception ex)
{
Console.WriteLine("Exception during test case init {0}", ex.ToString());
throw;
}
IQueryable <BaseEntity> tableQuery = table.CreateQuery <BaseEntity>();
IQueryable <BaseEntity> rtableQuery = localRTable.CreateQuery <BaseEntity>();
var list = tableQuery.AsEnumerable();
int tableCount = 0;
int rtableCount = 0;
foreach (BaseEntity ent in list)
{
tableCount++;
}
foreach (BaseEntity ent in rtableQuery.ToList())
{
rtableCount++;
Assert.IsTrue(ent.ETag != ent._rtable_Version.ToString(), "ETag is not virtualized when using CreateQuery()");
}
Assert.IsTrue(tableCount == rtableCount, "Query counts are different");
Assert.IsTrue(tableCount == 20, "Query counts are different");
// But, with "CreateReplicatedQuery" ETag is virtualized
IQueryable <BaseEntity> virtualizedRtableQuery = localRTable.CreateReplicatedQuery <BaseEntity>();
foreach (BaseEntity ent in virtualizedRtableQuery.ToList())
{
Assert.IsTrue(ent._rtable_Version == 0);
Assert.IsTrue(ent.ETag == ent._rtable_Version.ToString(), "ETag is virtualized when using CreateReplicatedQuery()");
ent.A += "`";
// Update should go fine since ETag is virtualized
TableOperation operation = TableOperation.Replace(ent);
TableResult result = localRTable.Execute(operation);
Assert.IsTrue(result != null && result.HttpStatusCode == (int)HttpStatusCode.NoContent);
}
virtualizedRtableQuery = localRTable.CreateReplicatedQuery <BaseEntity>();
foreach (BaseEntity ent in virtualizedRtableQuery.ToList())
{
Assert.IsTrue(ent._rtable_Version == 1);
Assert.IsTrue(ent.ETag == ent._rtable_Version.ToString(), "ETag is virtualized when using CreateReplicatedQuery()");
}
}
catch (Exception e)
{
Console.WriteLine("Error during query processing: {0}", e.ToString());
}
finally
{
localRTable.DeleteIfExists();
}
}
public void TableQueryableCreateQueryNoPartitionKey()
{
Thread.Sleep(10000);
string tableName = this.GenerateRandomTableName();
ReplicatedTable localRTable = new ReplicatedTable(tableName, this.configurationService);
localRTable.CreateIfNotExists();
RTableWrapperForSampleRTableEntity localRTableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(localRTable);
CloudTableClient tableClient = localRTable.GetTailTableClient();
CloudTable table = tableClient.GetTableReference(localRTable.TableName);
string pk = "0";
try
{
try
{
TableBatchOperation batch = new TableBatchOperation();
for (int j = 0; j < 10; j++)
{
BaseEntity ent = GenerateRandomEntity(pk);
ent.RowKey = string.Format("{0:0000}", j);
batch.Insert(ent);
}
localRTable.ExecuteBatch(batch);
}
catch (Exception ex)
{
Console.WriteLine("Exception during test case init {0}", ex.ToString());
throw;
}
try
{
pk = "1";
TableBatchOperation batch = new TableBatchOperation();
for (int j = 0; j < 10; j++)
{
BaseEntity ent = GenerateRandomEntity(pk);
ent.RowKey = string.Format("{0:0000}", j);
batch.Insert(ent);
}
localRTable.ExecuteBatch(batch);
}
catch (Exception ex)
{
Console.WriteLine("Exception during test case init {0}", ex.ToString());
throw;
}
IQueryable <BaseEntity> tableQuery = table.CreateQuery <BaseEntity>();
IQueryable <BaseEntity> rtableQuery = localRTable.CreateQuery <BaseEntity>();
var list = tableQuery.AsEnumerable();
int tableCount = 0;
int rtableCount = 0;
foreach (BaseEntity ent in list)
{
tableCount++;
}
foreach (BaseEntity ent in rtableQuery.ToList())
{
rtableCount++;
}
Assert.IsTrue(tableCount == rtableCount, "Query counts are different");
Assert.IsTrue(tableCount == 20, "Query counts are different");
}
catch (Exception e)
{
Console.WriteLine("Error during query processing: {0}", e.ToString());
}
finally
{
localRTable.DeleteIfExists();
}
}