public void TableServiceContextTimeoutDuringSaveChangesNonBatchSync()
{
CloudTableClient tableClient = GenerateCloudTableClient();
TableServiceContext ctx = tableClient.GetTableServiceContext();
for (int m = 0; m < 100; m++)
{
BaseEntity ent = new BaseEntity("testpartition", m.ToString());
ent.Randomize();
ent.A = ent.RowKey;
ctx.AddObject(currentTable.Name, ent);
}
OperationContext opContext = new OperationContext();
TableRequestOptions requestOptions = new TableRequestOptions()
{
MaximumExecutionTime = TimeSpan.FromSeconds(5)
};
using (HttpMangler proxy = new HttpMangler(false,
new[] { DelayBehaviors.DelayAllRequestsIf(4000 * 3, XStoreSelectors.TableTraffic().IfHostNameContains(tableClient.Credentials.AccountName).SkipNSessions(10)) }))
{
try
{
ctx.SaveChangesWithRetries(SaveChangesOptions.None, requestOptions, opContext);
}
catch (StorageException ex)
{
Assert.AreEqual(ex.RequestInformation.HttpStatusCode, (int)HttpStatusCode.RequestTimeout);
Assert.AreEqual("The client could not finish the operation within specified timeout.", ex.Message);
Assert.IsTrue(ex.InnerException is TimeoutException);
}
}
}
public void QueueGetACLCancellation()
{
CloudQueue queue = DefaultQueueClient.GetQueueReference(GenerateNewQueueName());
TestHelper.ExecuteAPMMethodWithCancellation(4000,
new[] { DelayBehaviors.DelayAllRequestsIf(4000 * 3, XStoreSelectors.QueueTraffic().IfHostNameContains(DefaultQueueClient.Credentials.AccountName)) },
(options, opContext, callback, state) => queue.BeginGetPermissions((QueueRequestOptions)options, opContext, callback, state),
(res) => queue.EndGetPermissions(res));
}
public void TableGetACLCancellation()
{
CloudTableClient tableClient = GenerateCloudTableClient();
CloudTable tbl = tableClient.GetTableReference(GenerateRandomTableName());
TestHelper.ExecuteAPMMethodWithCancellation(4000,
new[] { DelayBehaviors.DelayAllRequestsIf(4000 * 3, XStoreSelectors.TableTraffic().IfHostNameContains(tableClient.Credentials.AccountName)) },
(options, opContext, callback, state) => tbl.BeginGetPermissions((TableRequestOptions)options, opContext, callback, state),
(res) => tbl.EndGetPermissions(res));
}
public void TableServiceContextTimeoutDuringSaveChangesNonBatchAPM()
{
CloudTableClient tableClient = GenerateCloudTableClient();
TableServiceContext ctx = tableClient.GetTableServiceContext();
for (int m = 0; m < 100; m++)
{
BaseEntity ent = new BaseEntity("testpartition", m.ToString());
ent.Randomize();
ent.A = ent.RowKey;
ctx.AddObject(currentTable.Name, ent);
}
OperationContext opContext = new OperationContext();
TableRequestOptions requestOptions = new TableRequestOptions()
{
MaximumExecutionTime = TimeSpan.FromSeconds(5)
};
using (HttpMangler proxy = new HttpMangler(false,
new[] { DelayBehaviors.DelayAllRequestsIf(2000, XStoreSelectors.TableTraffic().IfHostNameContains(tableClient.Credentials.AccountName).SkipNSessions(10)) }))
{
try
{
using (ManualResetEvent evt = new ManualResetEvent(false))
{
IAsyncResult result = ctx.BeginSaveChangesWithRetries(SaveChangesOptions.None, requestOptions, opContext,
(res) =>
{
result = res;
evt.Set();
}, null);
evt.WaitOne();
ctx.EndSaveChangesWithRetries(result);
}
ctx.SaveChangesWithRetries(SaveChangesOptions.None, requestOptions, opContext);
}
catch (StorageException ex)
{
Assert.AreEqual(ex.RequestInformation.HttpStatusCode, (int)HttpStatusCode.RequestTimeout);
Assert.AreEqual(ex.Message, "The operation timed out.");
Assert.IsTrue(ex.InnerException is TimeoutException);
}
}
}
public void TableOperationCancellation()
{
CloudTableClient tableClient = GenerateCloudTableClient();
DynamicTableEntity insertEntity = new DynamicTableEntity("insert test", "foo");
for (int m = 0; m < 20; m++)
{
insertEntity.Properties.Add("prop" + m.ToString(), new EntityProperty(new byte[50 * 1024]));
}
TestHelper.ExecuteAPMMethodWithCancellation(4000,
new[] { DelayBehaviors.DelayAllRequestsIf(4000 * 3, XStoreSelectors.TableTraffic().IfHostNameContains(tableClient.Credentials.AccountName)) },
(options, opContext, callback, state) => currentTable.BeginExecute(TableOperation.Insert(insertEntity), (TableRequestOptions)options, opContext, callback, state),
(res) => currentTable.EndExecute(res));
}
public void TableTestSaveChangesCancellationNonBatch()
{
CloudTableClient tableClient = GenerateCloudTableClient();
TableServiceContext ctx = tableClient.GetTableServiceContext();
for (int m = 0; m < 100; m++)
{
// Insert Entity
ComplexEntity insertEntity = new ComplexEntity("insert test", m.ToString());
ctx.AddObject(currentTable.Name, insertEntity);
}
TestHelper.ExecuteAPMMethodWithCancellation(4000,
new[] { DelayBehaviors.DelayAllRequestsIf(4000 * 3, XStoreSelectors.TableTraffic().IfHostNameContains(tableClient.Credentials.AccountName)) },
(options, opContext, callback, state) => ctx.BeginSaveChangesWithRetries(SaveChangesOptions.None, (TableRequestOptions)options, opContext, callback, state),
(res) => ctx.EndSaveChangesWithRetries(res));
}
public void QueueSetACLCancellation()
{
CloudQueue queue = DefaultQueueClient.GetQueueReference(GenerateNewQueueName());
QueuePermissions permissions = new QueuePermissions();
permissions.SharedAccessPolicies.Add(Guid.NewGuid().ToString(), new SharedAccessQueuePolicy()
{
SharedAccessStartTime = DateTimeOffset.Now - TimeSpan.FromHours(1),
SharedAccessExpiryTime = DateTimeOffset.Now + TimeSpan.FromHours(1),
Permissions = SharedAccessQueuePermissions.Add | SharedAccessQueuePermissions.ProcessMessages | SharedAccessQueuePermissions.Read | SharedAccessQueuePermissions.Update
});
TestHelper.ExecuteAPMMethodWithCancellation(4000,
new[] { DelayBehaviors.DelayAllRequestsIf(4000 * 3, XStoreSelectors.QueueTraffic().IfHostNameContains(DefaultQueueClient.Credentials.AccountName)) },
(options, opContext, callback, state) => queue.BeginSetPermissions(permissions, (QueueRequestOptions)options, opContext, callback, state),
queue.EndSetPermissions);
}
public void CloudBlobContainerSetMetadataAPMCancel()
{
CloudBlobContainer container = GetRandomContainerReference();
try
{
container.Create();
container.Metadata.Add("key1", "value1");
TestHelper.ExecuteAPMMethodWithCancellation(4000,
new[] { DelayBehaviors.DelayAllRequestsIf(4000 * 3, AzureStorageSelectors.BlobTraffic().IfHostNameContains(container.ServiceClient.Credentials.AccountName)) },
(options, opContext, callback, state) => container.BeginSetMetadata(null, (BlobRequestOptions)options, opContext, callback, state),
container.EndSetMetadata);
}
finally
{
container.DeleteIfExists();
}
}
public void TableSetACLCancellation()
{
CloudTableClient tableClient = GenerateCloudTableClient();
CloudTable tbl = tableClient.GetTableReference(GenerateRandomTableName());
TablePermissions perms = new TablePermissions();
// Add a policy, check setting and getting.
perms.SharedAccessPolicies.Add(Guid.NewGuid().ToString(), new SharedAccessTablePolicy
{
Permissions = SharedAccessTablePermissions.Query,
SharedAccessStartTime = DateTimeOffset.Now - TimeSpan.FromHours(1),
SharedAccessExpiryTime = DateTimeOffset.Now + TimeSpan.FromHours(1)
});
TestHelper.ExecuteAPMMethodWithCancellation(4000,
new[] { DelayBehaviors.DelayAllRequestsIf(4000 * 3, XStoreSelectors.TableTraffic().IfHostNameContains(tableClient.Credentials.AccountName)) },
(options, opContext, callback, state) => tbl.BeginSetPermissions(perms, (TableRequestOptions)options, opContext, callback, state),
tbl.EndSetPermissions);
}
public void TableTestTableQueryCancellation()
{
CloudTableClient tableClient = GenerateCloudTableClient();
TableBatchOperation batch = new TableBatchOperation();
for (int m = 0; m < 100; m++)
{
// Insert Entity
DynamicTableEntity insertEntity = new DynamicTableEntity("insert test", m.ToString());
insertEntity.Properties.Add("prop" + m.ToString(), new EntityProperty(new byte[30 * 1024]));
batch.Insert(insertEntity);
}
currentTable.ExecuteBatch(batch);
TableQuery query = new TableQuery().Where(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, "insert test"));
TestHelper.ExecuteAPMMethodWithCancellation(4000,
new[] { DelayBehaviors.DelayAllRequestsIf(4000 * 3, XStoreSelectors.TableTraffic().IfHostNameContains(tableClient.Credentials.AccountName)) },
(options, opContext, callback, state) => currentTable.BeginExecuteQuerySegmented(query, null, (TableRequestOptions)options, opContext, callback, state),
(res) => currentTable.EndExecuteQuerySegmented(res));
}
public void TableTestSegmentedQueryCancellation()
{
CloudTableClient tableClient = GenerateCloudTableClient();
TableServiceContext ctx = tableClient.GetTableServiceContext();
for (int m = 0; m < 100; m++)
{
// Insert Entity
ComplexEntity insertEntity = new ComplexEntity("insert test", m.ToString());
ctx.AddObject(currentTable.Name, insertEntity);
}
ctx.SaveChangesWithRetries();
TableServiceQuery <BaseEntity> query = (from ent in ctx.CreateQuery <BaseEntity>(currentTable.Name)
select ent).AsTableServiceQuery(ctx);
TestHelper.ExecuteAPMMethodWithCancellation(4000,
new[] { DelayBehaviors.DelayAllRequestsIf(4000 * 3, XStoreSelectors.TableTraffic().IfHostNameContains(tableClient.Credentials.AccountName)) },
(options, opContext, callback, state) => query.BeginExecuteSegmented(null, (TableRequestOptions)options, opContext, callback, state),
(res) => query.EndExecuteSegmented(res));
}
public void TxnViewExpiresAndViewIdHasChanged()
{
// Insert an entry
string entityPartitionKey = "jobType-ReplaceWhenTxnViewExpires";
string entityRowKey = "jobType-ReplaceWhenTxnViewExpires";
this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);
Console.WriteLine("Inserting entry ...");
var entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, "insert message");
this.rtableWrapper.InsertRow(entry);
// Retrieve the config ...
ReplicatedTableConfiguration config;
ReplicatedTableQuorumReadResult readStatus = this.configurationService.RetrieveConfiguration(out config);
Assert.IsTrue(readStatus.Code == ReplicatedTableQuorumReadCode.Success);
// Reconfigure RTable config with a short LeaseDuration
int leaseDurationInSec = 3;
config.LeaseDuration = leaseDurationInSec;
this.configurationService.UpdateConfiguration(config);
string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0];
Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper);
// Delay behavior
ProxyBehavior[] behaviors = new[]
{
DelayBehaviors.DelayAllResponsesIf(
1,
(session =>
{
var body = session.GetRequestBodyAsString();
// Delay lock of the Head response enough so the txView expires ...
// And, upload a new config with a new viewId
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.HTTPMethodIs("PUT") &&
body.Contains("\"_rtable_Operation\":\"Replace\"") &&
body.Contains("\"_rtable_RowLock\":true"))
{
// Upload a new config with a new ViewId
View view = this.configurationService.GetTableView(this.repTable.TableName);
config.GetView(view.Name).ViewId++;
this.configurationService.UpdateConfiguration(config, false);
// ensure txView expires
Thread.Sleep(2 * leaseDurationInSec * 1000);
return(true);
}
return(false);
})),
};
// Launch an Update
using (new HttpMangler(false, behaviors))
{
try
{
entry = this.rtableWrapper.ReadEntity(entityPartitionKey, entityRowKey);
entry.Message = "update message";
var table = new ReplicatedTable(this.repTable.TableName, this.configurationService);
TableOperation replaceOperation = TableOperation.Replace(entry);
// Get a snapshot of txView before transaction starts ...
View viewBeforeTx = this.configurationService.GetTableView(this.repTable.TableName);
// Transaction will get delayed 2*LeaseDuration sec. and Config/ViewId will be changed
TableResult replaceResult = table.Execute(replaceOperation);
Assert.IsTrue(viewBeforeTx.IsExpired(), "txView expected to expire!");
Assert.AreNotEqual(this.configurationService.GetTableView(this.repTable.TableName).ViewId, viewBeforeTx.ViewId, "ViewId should have changed!");
Assert.IsNotNull(replaceResult, "upsertResult = null");
Assert.AreNotEqual((int)HttpStatusCode.NoContent, replaceResult.HttpStatusCode, "upsertResult.HttpStatusCode mismatch");
}
catch (ReplicatedTableStaleViewException ex)
{
// ValidateTxnView is called form everywhere ... we may endup here too
switch (ex.ErrorCode)
{
case ReplicatedTableViewErrorCodes.ViewIdChanged:
break;
default:
Console.WriteLine(ex);
Assert.IsTrue(false);
break;
}
}
}
}
public void A00InsertOrReplaceCallConflictingWithDeleteOnTheHead()
{
this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);
string entityPartitionKey = "jobType-DelayInsertOrReplaceWhileDelete";
string entityRowKey = "jobType-DelayInsertOrReplaceWhileDelete";
this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);
// Insert one entry
Console.WriteLine("Inserting entry ...");
var entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, "insert message");
this.rtableWrapper.InsertRow(entry);
// 1 - Launch an Upsert task in wait mode ...
bool deleteLockedHead = false;
TableResult upsertResult = null;
var upsertTask = Task.Run(() =>
{
while (!deleteLockedHead)
{
Thread.Sleep(5);
}
try
{
entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, "upsert message");
var table = new ReplicatedTable(this.repTable.TableName, this.configurationService);
TableOperation upserOperation = TableOperation.InsertOrReplace(entry);
Console.WriteLine("Upsert started ...");
upsertResult = table.Execute(upserOperation);
Console.WriteLine("Upsert completed with HttpStatus={0}", upsertResult == null ? "NULL" : upsertResult.HttpStatusCode.ToString());
}
catch (AggregateException ex)
{
Console.WriteLine(ex);
}
});
string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0];
Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper);
int delayInMs = 5;
// Delay behavior
ProxyBehavior[] behaviors = new[]
{
DelayBehaviors.DelayAllResponsesIf(
delayInMs,
(session =>
{
var body = session.GetRequestBodyAsString();
Console.WriteLine(body);
// Delete locking the head response
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.HTTPMethodIs("PUT") &&
body.Contains("\"_rtable_Operation\":\"Replace\"") &&
body.Contains("\"_rtable_RowLock\":true") &&
body.Contains("\"_rtable_Tombstone\":true"))
{
// Signal upsert we locked the head, so it can continue ...
deleteLockedHead = true;
return(true);
}
return(false);
})),
};
// Launch a delete
using (new HttpMangler(false, behaviors))
{
try
{
var table = new ReplicatedTable(this.repTable.TableName, this.configurationService);
// Retrieve entity
TableOperation retrieveOperation = TableOperation.Retrieve <SampleRTableEntity>(entry.PartitionKey, entry.RowKey);
TableResult retrieveResult = table.Execute(retrieveOperation);
Assert.IsNotNull(retrieveResult, "retrieveResult = null");
Assert.AreEqual((int)HttpStatusCode.OK, retrieveResult.HttpStatusCode, "retrieveResult.HttpStatusCode mismatch");
Assert.IsNotNull((SampleRTableEntity)retrieveResult.Result, "Retrieve: customer = null");
// Delete entity
TableOperation deleteOperation = TableOperation.Delete((SampleRTableEntity)retrieveResult.Result);
TableResult deleteResult = table.Execute(deleteOperation);
Assert.IsNotNull(deleteResult, "deleteResult = null");
Assert.AreEqual((int)HttpStatusCode.NoContent, deleteResult.HttpStatusCode, "deleteResult.HttpStatusCode mismatch");
}
catch (AggregateException ex)
{
Console.WriteLine(ex);
}
}
// wait on upsert to finish ...
upsertTask.Wait();
// Upsert suceeded?
Assert.AreEqual((int)HttpStatusCode.NoContent, upsertResult.HttpStatusCode, "Upsert expected to suceed!");
SampleRTableEntity upsertedEntity = this.rtableWrapper.ReadEntity(entityPartitionKey, entityRowKey);
Assert.NotNull(upsertedEntity, "upsert should succeed");
Assert.AreEqual(upsertedEntity.Message, "upsert message", "upsert should succeeded");
}
public void A00TwoInsertOrReplaceCallsConflictingOnTheHead()
{
this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);
string entityPartitionKey = "jobType-DelayInsertOrReplaceRowHeadTest";
string entityRowKey = "jobId-DelayInsertOrReplaceRowHeadTest";
this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);
// Insert one entry
Console.WriteLine("Inserting entry ...");
var entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, "insert message");
this.rtableWrapper.InsertRow(entry);
string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0];
Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper);
int delayInMs = 3000;
bool firstWritterInitiatingCommit = false;
bool secondWritterTriedLockingHead = false;
// Delay behavior
ProxyBehavior[] behaviors = new[]
{
// Writter-1 tampering
TamperBehaviors.TamperAllRequestsIf(
(session =>
{
int iter = 0;
// Signal Writter-2
firstWritterInitiatingCommit = true;
// Blobk commit to head ... until Writter-2 try to lock the head
while (!secondWritterTriedLockingHead)
{
Console.WriteLine("Writter-1 waiting on Writter-2 to try to lock the Head (#{0})", iter);
Thread.Sleep(delayInMs);
if (++iter > 10)
{
break;
}
}
}),
(session =>
{
var body = session.GetRequestBodyAsString();
// Writter-1 committing to head
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.HTTPMethodIs("PUT") &&
body.Contains("\"_rtable_Operation\":\"Replace\"") &&
body.Contains("\"_rtable_RowLock\":false") &&
body.Contains("\"Message\":\"upsert message 0\"")
)
{
return(true);
}
return(false);
})),
// Writter-2 tampering
TamperBehaviors.TamperAllRequestsIf(
(session =>
{
// Block till Writter-1 issues a commit to head
while (!firstWritterInitiatingCommit)
{
Console.WriteLine("Writter-2 waiting on Writter-1 to issue a commit to head");
Thread.Sleep(delayInMs);
}
}),
(session =>
{
var body = session.GetRequestBodyAsString();
// Writter-2 locking the head
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.HTTPMethodIs("PUT") &&
body.Contains("\"_rtable_Operation\":\"Replace\"") &&
body.Contains("\"_rtable_RowLock\":true") &&
body.Contains("\"Message\":\"upsert message 1\"")
)
{
return(true);
}
return(false);
})),
// Delay Writter-2 lock-to-the-head's response, so Writter-1 can continue with its commit.
DelayBehaviors.DelayAllResponsesIf(
delayInMs,
(session =>
{
var body = session.GetRequestBodyAsString();
// Writter-2 locking the head response
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.HTTPMethodIs("PUT") &&
body.Contains("\"_rtable_Operation\":\"Replace\"") &&
body.Contains("\"_rtable_RowLock\":true") &&
body.Contains("\"Message\":\"upsert message 1\"")
)
{
// Signal Writter-1 so it can continue with commit to head
secondWritterTriedLockingHead = true;
return(true);
}
return(false);
})),
};
// Launch 2 concurrent Upserts
var results = new TableResult[2];
using (new HttpMangler(false, behaviors))
{
Parallel.For(0, 2, (index) =>
{
entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, string.Format("upsert message {0}", index));
try
{
var table = new ReplicatedTable(this.repTable.TableName, this.configurationService);
TableOperation operation = TableOperation.InsertOrReplace(entry);
results[index] = table.Execute(operation);
}
catch (AggregateException ex)
{
Console.WriteLine(ex);
}
});
}
// Writter-1 suceed?
Assert.AreEqual((int)HttpStatusCode.NoContent, results[0].HttpStatusCode, "Writter-1 expected to suceed!");
// Writter-2 suceeded?
Assert.AreEqual((int)HttpStatusCode.NoContent, results[1].HttpStatusCode, "Writter-2 expected to suceed!");
// Writter-2 upsert succeeded
SampleRTableEntity upsertedEntity = this.rtableWrapper.ReadEntity(entityPartitionKey, entityRowKey);
Assert.NotNull(upsertedEntity, "Writter-2 upsert succeeded");
Assert.AreEqual(upsertedEntity.Message, string.Format("upsert message {0}", 1), "Writter-2 upsert succeeded");
}
public void A00DelayTwoConflictingInsertOrReplaceCalls()
{
this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);
string entityPartitionKey = "jobType-DelayInsertOrReplaceRowHeadTest";
string entityRowKey = "jobId-DelayInsertOrReplaceRowHeadTest";
this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);
string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0];
Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper);
int delayInMs = 3000;
int insertRequestCount = 0;
int conflictResponseCount = 0;
bool secondUpsertConflicted = false;
int failedCallIndex = -1;
// Delay behavior
ProxyBehavior[] behaviors = new[]
{
// Delay Insert calls so they end up conflicting
TamperBehaviors.TamperAllRequestsIf(
(session =>
{
Interlocked.Increment(ref insertRequestCount);
while (insertRequestCount != 2)
{
Console.WriteLine("insertRequestCount={0}. Waiting on count to reach 2 ...", insertRequestCount);
Thread.Sleep(delayInMs);
}
}),
(session =>
{
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.HTTPMethodIs("POST") &&
session.GetRequestBodyAsString().Contains("\"_rtable_Operation\":\"Insert\""))
{
return(true);
}
return(false);
})),
// Delay conflict response
DelayBehaviors.DelayAllResponsesIf(
delayInMs,
(session =>
{
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.GetRequestBodyAsString().Contains("\"_rtable_Operation\":\"Insert\""))
{
if (session.responseCode == (int)HttpStatusCode.Conflict)
{
Interlocked.Increment(ref conflictResponseCount);
return(true);
}
}
return(false);
})),
};
using (new HttpMangler(false, behaviors))
{
Parallel.For(0, 2, (index) =>
{
var entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, string.Format("upsert message {0}", index));
try
{
this.rtableWrapper.InsertOrReplaceRow(entry);
}
catch (RTableConflictException)
{
if (secondUpsertConflicted)
{
// should never reach here
throw;
}
// That's possible, but that's the Replace step of upsert which conflicted with ongoing write
// can't do anything, client should retry on conflict
secondUpsertConflicted = true;
}
});
}
// got 2 inserts?
Assert.AreEqual(2, insertRequestCount, "Two insert calls expected!");
// got one conflict?
Assert.AreEqual(1, conflictResponseCount, "One conflict response expected!");
// at least one upsert would have succeeded
SampleRTableEntity upsertedEntity = this.rtableWrapper.ReadEntity(entityPartitionKey, entityRowKey);
Assert.NotNull(upsertedEntity, "at least one upsert should have succeeded");
// second upsert failed?
if (secondUpsertConflicted)
{
Assert.AreEqual(1, upsertedEntity._rtable_Version, "one upsert succeeded so version should be = 1");
Assert.AreEqual(upsertedEntity.Message, string.Format("upsert message {0}", (1 - failedCallIndex)));
}
else
{
Assert.AreEqual(2, upsertedEntity._rtable_Version, "both upserts succeeded so version should be = 2");
}
// After recovery from delay, confirm that we can update the row.
//this.ExecuteReplaceRowAndValidate(entityPartitionKey, entityRowKey);
}