public void A00XStoreThrottleTest()
{
string entityPartitionKey = "jobType-XStoreThrottleTest";
string entityRowKey = "jobId-XStoreThrottleTest";
this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);
int targetStorageAccount = 0;
bool targetApiExpectedToFail = true;
Assert.IsTrue(0 <= targetStorageAccount && targetStorageAccount < this.actualStorageAccountsUsed.Count,
"targetStorageAccount={0} is out-of-range", targetStorageAccount);
int index = this.actualStorageAccountsUsed[targetStorageAccount];
string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[index];
Console.WriteLine("accountNameToTamper={0}", accountNameToTamper);
// Throttle behavior
ProxyBehavior[] behaviors = new[]
{
TamperBehaviors.TamperAllRequestsIf(
Actions.ThrottleTableRequest,
AzureStorageSelectors.TableTraffic().IfHostNameContains(accountNameToTamper))
};
this.SetupAndRunXStoreHttpManglerTest(
entityPartitionKey,
entityRowKey,
targetStorageAccount,
targetApiExpectedToFail,
behaviors);
}
public void RetrieveThrowsBadRequest()
{
string jobType = "jobType//RTableWrapperCRUDTest";
string jobId = "jobId//RTableWrapperCRUDTest";
int getCallCounts = 0;
var manglingBehaviors = new[]
{
TamperBehaviors.TamperAllRequestsIf(
(session) =>
{
getCallCounts++;
},
(session) =>
{
if (session.HTTPMethodIs("GET"))
{
return(true);
}
return(false);
})
};
using (new HttpMangler(false, manglingBehaviors))
{
Assert.Throws <StorageException>(() =>
{
try
{
this.rtableWrapper.ReadEntity(jobType, jobId);
}
catch (StorageException se)
{
Assert.IsNotNull(se.InnerException);
var webException = se.InnerException as WebException;
Assert.IsNotNull(webException);
var response = (HttpWebResponse)webException.Response;
Assert.IsNotNull(response);
Assert.AreEqual(HttpStatusCode.BadRequest, response.StatusCode);
throw;
}
});
}
Assert.AreEqual(1, getCallCounts);
}
/// <summary>
/// EchoTableEntry echos back the contents that were sent to the server.
/// </summary>
/// <returns>A new behavior.</returns>
public static ProxyBehavior EchoTableEntry()
{
return(TamperBehaviors.TamperAllRequestsIf(EchoEntry, Selectors.IfPost().ForTableTraffic()));
}
/// <summary>
/// GetTableNotFound returns a behavior saying the request failed with 404 (not found)
/// </summary>
/// <returns>A new behavior.</returns>
public static ProxyBehavior GetTableNotFound()
{
return(TamperBehaviors.TamperAllRequestsIf(
session => CreateTableError(session, 404, "ResourceNotFound", "The specified resource does not exist."),
Selectors.IfGet().ForTableTraffic()));
}
/// <summary>
/// CreateTableErrorTableBeingDeleted returns a behavior saying the request failed with 409 (conflict) and substatus of "being deleted"
/// </summary>
/// <returns>A new behavior.</returns>
public static ProxyBehavior CreateTableErrorTableBeingDeleted()
{
return(TamperBehaviors.TamperAllRequestsIf(
session => CreateTableError(session, 409, "TableBeingDeleted", "The table specified is in the process of being deleted."),
Selectors.IfPost().ForTableTraffic()));
}
/// <summary>
/// CreateTableErrorTableAlreadyExists returns a behavior saying the request failed with 409 (conflict) and substatus of "already exists"
/// </summary>
/// <returns>A new behavior.</returns>
public static ProxyBehavior CreateTableErrorTableAlreadyExists()
{
return(TamperBehaviors.TamperAllRequestsIf(
session => CreateTableError(session, 409, "TableAlreadyExists", "The table specified already exists."),
Selectors.IfPost().ForTableTraffic()));
}
/// <summary>
/// CreateTableOk returns a behavior which causes all GET requests for table creation traffic to return successfully.
/// </summary>
/// <returns>A new behavior.</returns>
public static ProxyBehavior GetTableOk()
{
return(TamperBehaviors.TamperAllRequestsIf(session => GetTableWithCode(session, 200), Selectors.IfGet().ForTableTraffic()));
}
public void ReadingUncommittedDataFromReplicaIdentifiedByReadTailIndexWontThrow()
{
ReplicatedTableConfigurationServiceV2 configServiceOne, configServiceTwo;
// setup: Acc0 Acc1
// stale view = [H] -> [T]
// new view = [H]
SetupStaleViewAndNewView(out configServiceOne, out configServiceTwo);
long staleViewId = configServiceOne.GetTableView(this.repTable.TableName).ViewId;
long latestViewId = configServiceTwo.GetTableView(this.repTable.TableName).ViewId;
string firstName = "FirstName01";
string lastName = "LastName01";
/*
* 1 - WorkerOne => inserts an entry in stale View
*/
var workerOne = new ReplicatedTable(this.repTable.TableName, configServiceOne);
var workerTwo = new ReplicatedTable(this.repTable.TableName, configServiceTwo);
var customer = new CustomerEntity(firstName, lastName);
customer.Email = "***";
InsertCustormer(customer, workerOne);
string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[1];
Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper);
TableResult oldUpdateResult = null;
CustomerEntity customerSeenByNewView = null;
// Delay behavior
ProxyBehavior[] behaviors =
{
TamperBehaviors.TamperAllRequestsIf(
(session =>
{
session.oRequest.FailSession((int)HttpStatusCode.ServiceUnavailable, "ServerBusy", "");
}),
(session =>
{
var body = session.GetRequestBodyAsString();
// Fail Lock to Tail by stale view
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.HTTPMethodIs("PUT") &&
body.Contains("\"Email\":\"workerOne\"") &&
body.Contains(string.Format("\"_rtable_ViewId\":\"{0}\"", staleViewId)) &&
body.Contains("\"_rtable_RowLock\":true"))
{
return(true);
}
return(false);
})),
};
/*
* 2 - WorkerOne => update an entry using the stale View
*/
using (new HttpMangler(false, behaviors))
{
customer = RetrieveCustomer(firstName, lastName, workerOne);
customer.Email = "workerOne";
TableOperation operation = TableOperation.Replace(customer);
oldUpdateResult = workerOne.Execute(operation);
}
// Expected behavior:
// Thread_1 (stale ViewId) fails to commit to Tail
Assert.IsNotNull(oldUpdateResult, "oldUpdateResult = null");
Assert.AreEqual((int)HttpStatusCode.ServiceUnavailable, oldUpdateResult.HttpStatusCode, "oldUpdateResult.HttpStatusCode mismatch");
Console.WriteLine("Update in stale View Succeeded with HttpStatus={0}", oldUpdateResult.HttpStatusCode);
// Thread_1 (stale ViewId): Reads the entry => Succeeds eventhough the row is "uncommitted".
// "ReadTailIndex = [Head]" means [Head] is treated as [Tail] for Reads.
// => row is assumed "committed"
customer = RetrieveCustomer(firstName, lastName, workerOne);
Assert.IsNotNull(customer, "customer = null");
Assert.AreEqual("workerOne", customer.Email, "customer.Email mismatch");
// Thread_2 (new ViewId): Reads the entry => Succeed since reading from Head
customerSeenByNewView = RetrieveCustomer(firstName, lastName, workerTwo);
Assert.IsNotNull(customerSeenByNewView, "customerSeenByNewView = null");
Assert.AreEqual("workerOne", customer.Email, "customer.Email mismatch");
}
public void WhenReadFromTailFailsWithServiceUnavailableWeReadFromHeadAndSucceed()
{
View view = this.configurationWrapper.GetWriteView();
Assert.IsTrue(view.Chain.Count > 1, "expects at least 2 replicas!");
// Insert one entry
var rtable = new ReplicatedTable(this.repTable.TableName, this.configurationService);
string firstName = "FirstName01";
string lastName = "LastName01";
var customer = new CustomerEntity(firstName, lastName);
TableOperation operation = TableOperation.Insert(customer);
rtable.Execute(operation);
// Using xstore modify the row in each replica individually ... so we know, later, which replica RTable will retrieve from
for (int replicaIndex = 0; replicaIndex < this.cloudTableClients.Count; replicaIndex++)
{
CloudTableClient tableClient = this.cloudTableClients[replicaIndex];
CloudTable table = tableClient.GetTableReference(this.repTable.TableName);
TableOperation retrieveOperation = TableOperation.Retrieve <CustomerEntity>(firstName, lastName);
TableResult retrieveResult = table.Execute(retrieveOperation);
customer = (CustomerEntity)retrieveResult.Result;
customer.Email = replicaIndex.ToString(); // Head = 0
// ... = 1
// Tail = 2
TableOperation updateOperation = TableOperation.Replace(customer);
TableResult updateResult = table.Execute(updateOperation);
Assert.IsNotNull(updateResult, "updateResult = null");
Console.WriteLine("updateResult.HttpStatusCode = {0}", updateResult.HttpStatusCode);
Assert.AreEqual((int)HttpStatusCode.NoContent, updateResult.HttpStatusCode, "updateResult.HttpStatusCode mismatch");
}
string accountNameToNotTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0];
Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToNotTamper={0}", accountNameToNotTamper);
// Delay behavior
ProxyBehavior[] behaviors =
{
TamperBehaviors.TamperAllRequestsIf(
(session =>
{
session.oRequest.FailSession((int)HttpStatusCode.ServiceUnavailable, "ServerBusy", "");
}),
(session =>
{
if (session.HTTPMethodIs("Get"))
{
// Fail Get from all replicas, except Head
if (!session.hostname.Contains(accountNameToNotTamper + "."))
{
return(true);
}
}
return(false);
})),
};
using (new HttpMangler(false, behaviors))
{
operation = TableOperation.Retrieve <CustomerEntity>(firstName, lastName);
TableResult retrievedResult = repTable.Execute(operation);
Assert.AreNotEqual(null, retrievedResult, "retrievedResult = null");
Assert.AreEqual((int)HttpStatusCode.OK, retrievedResult.HttpStatusCode, "retrievedResult.HttpStatusCode mismatch");
Assert.AreNotEqual(null, retrievedResult.Result, "retrievedResult.Result = null");
customer = (CustomerEntity)retrievedResult.Result;
Assert.AreEqual(customer.Email, (0).ToString(), "we should have read the row from Head");
}
}
public void LinqQueriesAreServedFromHeadWhenReadViewTailIndexIsSet()
{
var rtable = new ReplicatedTable(this.repTable.TableName, this.configurationService);
string firstName = "FirstName";
string lastName = "LastName";
int dataSize = 5;
/*
* 1 - insert entries
*/
for (int i = 0; i < dataSize; i++)
{
var customer = new CustomerEntity(firstName + i, lastName + i);
customer.Email = "***";
TableOperation operation = TableOperation.Insert(customer);
rtable.Execute(operation);
}
// Identify the Tail account
View view = this.configurationWrapper.GetWriteView();
Assert.IsTrue(view.Chain.Count > 1, "expects at least one replica!");
string accountNameToTamper = view.Chain.Last().Item1.StorageAccountName;
Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper);
// We will fail requests to update a row in Tail
ProxyBehavior[] behaviors =
{
TamperBehaviors.TamperAllRequestsIf(
(session =>
{
session.oRequest.FailSession((int)HttpStatusCode.ServiceUnavailable, "ServerBusy", "");
}),
(session =>
{
var body = session.GetRequestBodyAsString();
// Fail Lock to Tail by stale view
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.HTTPMethodIs("PUT") &&
body.Contains("\"_rtable_RowLock\":true"))
{
return(true);
}
return(false);
})),
};
using (new HttpMangler(false, behaviors))
{
/*
* 2 - update entries
*/
for (int i = 0; i < dataSize; i++)
{
TableOperation operation = TableOperation.Retrieve <CustomerEntity>(firstName + i, lastName + i);
TableResult retrievedResult = rtable.Execute(operation);
var customer = (CustomerEntity)retrievedResult.Result;
customer.Email = "updated";
TableOperation replaceOperation = TableOperation.Replace(customer);
TableResult replaceResult = rtable.Execute(replaceOperation);
Assert.IsNotNull(replaceResult, "replaceResult = null");
Assert.AreEqual((int)HttpStatusCode.ServiceUnavailable, replaceResult.HttpStatusCode, "replaceResult.HttpStatusCode mismatch");
}
}
SetReadViewTailIndex(0);
Assert.AreEqual(0, this.configurationService.GetTableView(this.repTable.TableName).ReadTailIndex, "ReadTailIndex should be 0!!!");
/*
* 3 - CreateQuery is served from [Head], data should be new
*/
foreach (var customer in rtable.CreateReplicatedQuery <CustomerEntity>().AsEnumerable())
{
Assert.AreEqual(customer.Email, "updated", "expected new data");
}
/*
* 4 - ExecuteQuery is served from [Head], data should be new
*/
foreach (var customer in rtable.ExecuteQuery <CustomerEntity>(new TableQuery <CustomerEntity>()))
{
Assert.AreEqual(customer.Email, "updated", "expected new data");
}
}
public void LinqQueriesDontReturnEntriesWithTombstone()
{
var rtable = new ReplicatedTable(this.repTable.TableName, this.configurationService);
string firstName = "FirstName";
string lastName = "LastName";
/*
* 1 - insert entries
*/
for (int i = 0; i < 10; i++)
{
var customer = new CustomerEntity(firstName + i, lastName + i);
TableOperation operation = TableOperation.Insert(customer);
rtable.Execute(operation);
}
// Identify the Tail account
View view = this.configurationWrapper.GetWriteView();
Assert.IsTrue(view.Chain.Count > 1, "expects at least one replica!");
string accountNameToTamper = view.Chain.Last().Item1.StorageAccountName;
Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper);
// We will fail requests to delete a row in Tail
ProxyBehavior[] behaviors =
{
TamperBehaviors.TamperAllRequestsIf(
(session =>
{
session.oRequest.FailSession((int)HttpStatusCode.ServiceUnavailable, "ServerBusy", "");
}),
(session =>
{
// Commit to Tail i.e. physically delete the row
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.HTTPMethodIs("DELETE"))
{
return(true);
}
return(false);
})),
};
using (new HttpMangler(false, behaviors))
{
/*
* 2 - delete entries #2 and #4
*/
foreach (var i in new int[] { 2, 4 })
{
TableOperation operation = TableOperation.Retrieve <CustomerEntity>(firstName + i, lastName + i);
TableResult retrievedResult = rtable.Execute(operation);
var customer = (CustomerEntity)retrievedResult.Result;
TableOperation deleteOperation = TableOperation.Delete(customer);
TableResult deleteResult = rtable.Execute(deleteOperation);
Assert.IsNotNull(deleteResult, "deleteResult = null");
Assert.AreEqual((int)HttpStatusCode.ServiceUnavailable, deleteResult.HttpStatusCode, "deleteResult.HttpStatusCode mismatch");
}
}
// Verify, Retrieve doesn't return entries #2 and #4
int deleteId = 2;
var result = rtable.Execute(TableOperation.Retrieve <CustomerEntity>(firstName + deleteId, lastName + deleteId));
Assert.IsNotNull(result, "result = null");
Assert.AreEqual((int)HttpStatusCode.NotFound, result.HttpStatusCode, "result.HttpStatusCode mismatch");
deleteId = 4;
result = rtable.Execute(TableOperation.Retrieve <CustomerEntity>(firstName + deleteId, lastName + deleteId));
Assert.IsNotNull(result, "result = null");
Assert.AreEqual((int)HttpStatusCode.NotFound, result.HttpStatusCode, "result.HttpStatusCode mismatch");
/*
* 3 - CreateQuery doesn't return entries #2 and #4
*/
foreach (var customer in rtable.CreateReplicatedQuery <CustomerEntity>().AsEnumerable())
{
int id = int.Parse(customer.PartitionKey.Replace(firstName, ""));
Assert.AreNotEqual(id, 2, "entry #2 should have been deleted");
Assert.AreNotEqual(id, 4, "entry #4 should have been deleted");
}
/*
* 4 - ExecuteQuery doesn't return entries #2 and #4
*/
foreach (var customer in rtable.ExecuteQuery <CustomerEntity>(new TableQuery <CustomerEntity>()))
{
int id = int.Parse(customer.PartitionKey.Replace(firstName, ""));
Assert.AreNotEqual(id, 2, "entry #2 should have been deleted");
Assert.AreNotEqual(id, 4, "entry #4 should have been deleted");
}
}
public void RepairMidAnUpdate()
{
this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);
// Not-Stable chain
// Reconfigure RTable so Head is WriteOnly.
View view = this.configurationService.GetTableView(this.repTable.TableName);
ReplicatedTableConfiguration config;
ReplicatedTableQuorumReadResult readStatus = this.configurationService.RetrieveConfiguration(out config);
Assert.IsTrue(readStatus.Code == ReplicatedTableQuorumReadCode.Success);
// Set Head as WriteOnly mode
ReplicatedTableConfigurationStore viewConfg = config.GetView(view.Name);
viewConfg.ReplicaChain[0].Status = ReplicaStatus.WriteOnly;
config.SetView(view.Name, viewConfg);
// Upload RTable config back
this.configurationService.UpdateConfiguration(config);
// Sanity: Replicated mode and chain Not-Stable
view = this.configurationService.GetTableView(this.repTable.TableName);
Assert.IsTrue(view != null && view.Chain.Count > 1, "Two replicas should be used.");
Assert.IsFalse(view.IsStable);
// Insert one entry
Console.WriteLine("Inserting entry ...");
string entityPartitionKey = "jobType-RepairMidAnUpdate-Replace";
string entityRowKey = "jobId-RepairMidAnUpdate-Replace";
var entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, "message");
this.rtableWrapper.InsertRow(entry);
// 1 - Launch a RepairRow task in wait mode ...
bool triggerRepair = false;
bool repaireDone = false;
bool headAfterRepairWasLocked = false;
TableResult repairResult = null;
Task.Run(() =>
{
ReplicatedTable repairTable = new ReplicatedTable(this.repTable.TableName, this.configurationService);
while (!triggerRepair)
{
Thread.Sleep(5);
}
Console.WriteLine("RepairRow started ...");
repairResult = repairTable.RepairRow(entry.PartitionKey, entry.RowKey, null);
Console.WriteLine("RepairRow completed with HttpStatus={0}", repairResult == null ? "NULL" : repairResult.HttpStatusCode.ToString());
// Check the entry at the Head is still locked i.e. RepairRow was NOP
headAfterRepairWasLocked = HeadIsLocked(entry);
Console.WriteLine("Signal the commit to Head job");
repaireDone = true;
});
// 2 - Configure Mangler ...
string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0];
Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper);
ProxyBehavior[] behaviors = new[]
{
TamperBehaviors.TamperAllRequestsIf(
(session =>
{
Console.WriteLine("Delaying commit to the Head ... => signal RepairRow job");
// Let RepairRow task go through
triggerRepair = true;
int iter = 0;
while (!repaireDone)
{
// TODO: break the loop after couple of iteration ...
Thread.Sleep(100);
Console.WriteLine("Waiting on RepairRow to finish ({0}) ...", ++iter);
}
Console.WriteLine("Request a commit to the head");
}),
(session =>
{
// Commit on head i.e. a PUT with RowLock == false
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.HTTPMethodIs("PUT") &&
session.GetRequestBodyAsString().Contains("\"_rtable_RowLock\":false"))
{
return(true);
}
return(false);
}))
};
using (new HttpMangler(false, behaviors))
{
Console.WriteLine("Updating entry ...");
entry = this.rtableWrapper.FindRow(entry.PartitionKey, entry.RowKey);
entry.Message = "updated message";
this.rtableWrapper.ReplaceRow(entry);
}
Assert.IsTrue(triggerRepair);
Assert.IsTrue(repairResult != null && repairResult.HttpStatusCode == (int)HttpStatusCode.OK, "Repair failed.");
Assert.IsTrue(repaireDone);
Assert.IsTrue(headAfterRepairWasLocked);
Console.WriteLine("DONE. Test passed.");
}
public void A00TwoInsertOrReplaceCallsConflictingOnTheHead()
{
this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);
string entityPartitionKey = "jobType-DelayInsertOrReplaceRowHeadTest";
string entityRowKey = "jobId-DelayInsertOrReplaceRowHeadTest";
this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);
// Insert one entry
Console.WriteLine("Inserting entry ...");
var entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, "insert message");
this.rtableWrapper.InsertRow(entry);
string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0];
Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper);
int delayInMs = 3000;
bool firstWritterInitiatingCommit = false;
bool secondWritterTriedLockingHead = false;
// Delay behavior
ProxyBehavior[] behaviors = new[]
{
// Writter-1 tampering
TamperBehaviors.TamperAllRequestsIf(
(session =>
{
int iter = 0;
// Signal Writter-2
firstWritterInitiatingCommit = true;
// Blobk commit to head ... until Writter-2 try to lock the head
while (!secondWritterTriedLockingHead)
{
Console.WriteLine("Writter-1 waiting on Writter-2 to try to lock the Head (#{0})", iter);
Thread.Sleep(delayInMs);
if (++iter > 10)
{
break;
}
}
}),
(session =>
{
var body = session.GetRequestBodyAsString();
// Writter-1 committing to head
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.HTTPMethodIs("PUT") &&
body.Contains("\"_rtable_Operation\":\"Replace\"") &&
body.Contains("\"_rtable_RowLock\":false") &&
body.Contains("\"Message\":\"upsert message 0\"")
)
{
return(true);
}
return(false);
})),
// Writter-2 tampering
TamperBehaviors.TamperAllRequestsIf(
(session =>
{
// Block till Writter-1 issues a commit to head
while (!firstWritterInitiatingCommit)
{
Console.WriteLine("Writter-2 waiting on Writter-1 to issue a commit to head");
Thread.Sleep(delayInMs);
}
}),
(session =>
{
var body = session.GetRequestBodyAsString();
// Writter-2 locking the head
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.HTTPMethodIs("PUT") &&
body.Contains("\"_rtable_Operation\":\"Replace\"") &&
body.Contains("\"_rtable_RowLock\":true") &&
body.Contains("\"Message\":\"upsert message 1\"")
)
{
return(true);
}
return(false);
})),
// Delay Writter-2 lock-to-the-head's response, so Writter-1 can continue with its commit.
DelayBehaviors.DelayAllResponsesIf(
delayInMs,
(session =>
{
var body = session.GetRequestBodyAsString();
// Writter-2 locking the head response
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.HTTPMethodIs("PUT") &&
body.Contains("\"_rtable_Operation\":\"Replace\"") &&
body.Contains("\"_rtable_RowLock\":true") &&
body.Contains("\"Message\":\"upsert message 1\"")
)
{
// Signal Writter-1 so it can continue with commit to head
secondWritterTriedLockingHead = true;
return(true);
}
return(false);
})),
};
// Launch 2 concurrent Upserts
var results = new TableResult[2];
using (new HttpMangler(false, behaviors))
{
Parallel.For(0, 2, (index) =>
{
entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, string.Format("upsert message {0}", index));
try
{
var table = new ReplicatedTable(this.repTable.TableName, this.configurationService);
TableOperation operation = TableOperation.InsertOrReplace(entry);
results[index] = table.Execute(operation);
}
catch (AggregateException ex)
{
Console.WriteLine(ex);
}
});
}
// Writter-1 suceed?
Assert.AreEqual((int)HttpStatusCode.NoContent, results[0].HttpStatusCode, "Writter-1 expected to suceed!");
// Writter-2 suceeded?
Assert.AreEqual((int)HttpStatusCode.NoContent, results[1].HttpStatusCode, "Writter-2 expected to suceed!");
// Writter-2 upsert succeeded
SampleRTableEntity upsertedEntity = this.rtableWrapper.ReadEntity(entityPartitionKey, entityRowKey);
Assert.NotNull(upsertedEntity, "Writter-2 upsert succeeded");
Assert.AreEqual(upsertedEntity.Message, string.Format("upsert message {0}", 1), "Writter-2 upsert succeeded");
}
public void A00DelayTwoConflictingInsertOrReplaceCalls()
{
this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);
string entityPartitionKey = "jobType-DelayInsertOrReplaceRowHeadTest";
string entityRowKey = "jobId-DelayInsertOrReplaceRowHeadTest";
this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);
string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0];
Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper);
int delayInMs = 3000;
int insertRequestCount = 0;
int conflictResponseCount = 0;
bool secondUpsertConflicted = false;
int failedCallIndex = -1;
// Delay behavior
ProxyBehavior[] behaviors = new[]
{
// Delay Insert calls so they end up conflicting
TamperBehaviors.TamperAllRequestsIf(
(session =>
{
Interlocked.Increment(ref insertRequestCount);
while (insertRequestCount != 2)
{
Console.WriteLine("insertRequestCount={0}. Waiting on count to reach 2 ...", insertRequestCount);
Thread.Sleep(delayInMs);
}
}),
(session =>
{
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.HTTPMethodIs("POST") &&
session.GetRequestBodyAsString().Contains("\"_rtable_Operation\":\"Insert\""))
{
return(true);
}
return(false);
})),
// Delay conflict response
DelayBehaviors.DelayAllResponsesIf(
delayInMs,
(session =>
{
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.GetRequestBodyAsString().Contains("\"_rtable_Operation\":\"Insert\""))
{
if (session.responseCode == (int)HttpStatusCode.Conflict)
{
Interlocked.Increment(ref conflictResponseCount);
return(true);
}
}
return(false);
})),
};
using (new HttpMangler(false, behaviors))
{
Parallel.For(0, 2, (index) =>
{
var entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, string.Format("upsert message {0}", index));
try
{
this.rtableWrapper.InsertOrReplaceRow(entry);
}
catch (RTableConflictException)
{
if (secondUpsertConflicted)
{
// should never reach here
throw;
}
// That's possible, but that's the Replace step of upsert which conflicted with ongoing write
// can't do anything, client should retry on conflict
secondUpsertConflicted = true;
}
});
}
// got 2 inserts?
Assert.AreEqual(2, insertRequestCount, "Two insert calls expected!");
// got one conflict?
Assert.AreEqual(1, conflictResponseCount, "One conflict response expected!");
// at least one upsert would have succeeded
SampleRTableEntity upsertedEntity = this.rtableWrapper.ReadEntity(entityPartitionKey, entityRowKey);
Assert.NotNull(upsertedEntity, "at least one upsert should have succeeded");
// second upsert failed?
if (secondUpsertConflicted)
{
Assert.AreEqual(1, upsertedEntity._rtable_Version, "one upsert succeeded so version should be = 1");
Assert.AreEqual(upsertedEntity.Message, string.Format("upsert message {0}", (1 - failedCallIndex)));
}
else
{
Assert.AreEqual(2, upsertedEntity._rtable_Version, "both upserts succeeded so version should be = 2");
}
// After recovery from delay, confirm that we can update the row.
//this.ExecuteReplaceRowAndValidate(entityPartitionKey, entityRowKey);
}
public void InsertInStaleViewConflictingWithInsertInNewView()
{
ReplicatedTableConfigurationServiceV2 configServiceOne, configServiceTwo;
// setup: Acc0 Acc1
// stale view = [None] -> [RW]
// new view = [WO] -> [RW]
SetupStaleViewAndNewView(out configServiceOne, out configServiceTwo);
long staleViewId = configServiceOne.GetTableView(this.repTable.TableName).ViewId;
long latestViewId = configServiceTwo.GetTableView(this.repTable.TableName).ViewId;
string firstName = "FirstName01";
string lastName = "LastName01";
var workerOne = new ReplicatedTable(this.repTable.TableName, configServiceOne);
var workerTwo = new ReplicatedTable(this.repTable.TableName, configServiceTwo);
string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[1];
Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper);
TableResult oldInsertResult = null;
TableResult newInsertResult = null;
bool triggerInsertWithNewView = false;
bool oldInsertResume = false;
// Start new newInsertTask in wait
var newInsertTask = Task.Run(() =>
{
while (!triggerInsertWithNewView)
{
Thread.Sleep(100);
}
/*
* 2 - Executes after step 1 below:
* WorkerTwo => Insert a new row using new view
*/
var customer = new CustomerEntity(firstName, lastName);
customer.Email = "workerTwo";
TableOperation operation = TableOperation.Insert(customer);
newInsertResult = workerTwo.Execute(operation);
// Signal old Insert to resume
oldInsertResume = true;
});
// Delay behavior
ProxyBehavior[] behaviors =
{
TamperBehaviors.TamperAllRequestsIf(
(session =>
{
// => trigger new insert to start
triggerInsertWithNewView = true;
// Delaying commit to the Tail by stale view Update
while (!oldInsertResume)
{
Thread.Sleep(100);
}
}),
(session =>
{
var body = session.GetRequestBodyAsString();
// Commit to Tail by stale view
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.HTTPMethodIs("PUT") &&
body.Contains("\"Email\":\"workerOne\"") &&
body.Contains(string.Format("\"_rtable_ViewId\":\"{0}\"", staleViewId)) &&
body.Contains("\"_rtable_RowLock\":false"))
{
return(true);
}
return(false);
})),
};
/*
* 1 - WorkerOne => update an entry using the stale View
*/
using (new HttpMangler(false, behaviors))
{
var customer = new CustomerEntity(firstName, lastName);
customer.Email = "workerOne";
TableOperation operation = TableOperation.Insert(customer);
oldInsertResult = workerOne.Execute(operation);
}
// Wait on new Insert to finish
newInsertTask.Wait();
// Expected behavior:
// Thread_1 (stale ViewId)
Assert.IsNotNull(oldInsertResult, "oldInsertResult = null");
Assert.AreEqual((int)HttpStatusCode.NoContent, oldInsertResult.HttpStatusCode, "oldInsertResult.HttpStatusCode mismatch");
Console.WriteLine("Insert in stale View Succeeded with HttpStatus={0}", oldInsertResult.HttpStatusCode);
// Thread_2 (new ViewId)
Assert.IsNotNull(newInsertResult, "newInsertResult = null");
Assert.AreEqual((int)HttpStatusCode.Conflict, newInsertResult.HttpStatusCode, "newUpdateResult.HttpStatusCode mismatch");
Console.WriteLine("Insert in new View failed with HttpStatus={0}", newInsertResult.HttpStatusCode);
var currCustomer = RetrieveCustomer(firstName, lastName, workerTwo);
Assert.AreEqual(staleViewId, currCustomer._rtable_ViewId, "customer._rtable_ViewId mismatch");
Assert.AreEqual("workerOne", currCustomer.Email, "customer.Email mismatch");
Console.WriteLine("workerOne write Succeeded");
Console.WriteLine("workerTwo got Conflict because row was lock.");
}