private DataSampling DeleteEntriesPerf(ReplicatedTable rtable, List <CustomerEntity> entries)
{
var stats = new DataSampling("[D]elete");
foreach (var entry in entries)
{
TableOperation retrieveOperation = TableOperation.Retrieve <CustomerEntity>(entry.PartitionKey, entry.RowKey);
TableResult retrieveResult = rtable.Execute(retrieveOperation);
Assert.IsNotNull(retrieveResult, "retrieveResult = null");
Assert.AreEqual((int)HttpStatusCode.OK, retrieveResult.HttpStatusCode, "retrieveResult.HttpStatusCode mismatch");
Assert.IsNotNull((CustomerEntity)retrieveResult.Result, "Retrieve: customer = null");
// Delete entity
var customer = (CustomerEntity)retrieveResult.Result;
Assert.IsTrue(customer._rtable_Version == 2, "entry was updated once, version should be 2");
TableOperation deleteOperation = TableOperation.Delete(customer);
TableResult deleteResult;
var watch = System.Diagnostics.Stopwatch.StartNew();
{
deleteResult = rtable.Execute(deleteOperation);
}
watch.Stop();
stats.AddPoint(watch.ElapsedMilliseconds);
Assert.IsNotNull(deleteResult, "deleteResult = null");
Assert.AreEqual((int)HttpStatusCode.NoContent, deleteResult.HttpStatusCode, "deleteResult.HttpStatusCode mismatch");
}
return(stats);
}
private DataSampling RetrieveEntriesPerf(ReplicatedTable rtable, List <CustomerEntity> entries)
{
var stats = new DataSampling("[R]etrieve");
foreach (var entry in entries)
{
TableOperation retrieveOperation = TableOperation.Retrieve <CustomerEntity>(entry.PartitionKey, entry.RowKey);
TableResult retrieveResult = null;
var watch = System.Diagnostics.Stopwatch.StartNew();
{
retrieveResult = rtable.Execute(retrieveOperation);
}
watch.Stop();
stats.AddPoint(watch.ElapsedMilliseconds);
Assert.IsNotNull(retrieveResult, "retrieveResult = null");
Assert.AreEqual((int)HttpStatusCode.OK, retrieveResult.HttpStatusCode, "retrieveResult.HttpStatusCode mismatch");
var customer = (CustomerEntity)retrieveResult.Result;
Assert.IsNotNull(customer, "Retrieve: customer = null");
Assert.IsTrue(customer._rtable_Version == 1, "new entry should have version 1");
}
return(stats);
}
public void RTableStableTwoReplicaCRUDTest()
{
try
{
string tableName = this.GenerateRandomTableName("RTable2Perf");
this.SetupRTableEnv(tableName, true, "", new List <int> {
0, 1
});
Console.WriteLine("tableName = {0}", tableName);
// two-replica and stable?
View view = this.configurationService.GetTableView(tableName);
Assert.IsTrue(view != null && view.Chain != null && view.Chain.Count == 2 && view.IsStable);
// convert mode is Off ? yes, because we can't have more than 1 replica while in convert mode!
ReplicatedTable rtable = this.repTable;
Console.WriteLine("[C]reate stats ...");
entries = ShuffleList(entries);
DataSampling createStats = CreateEntriesStats(rtable, entries);
Console.WriteLine("[R]etrieve stats ...");
entries = ShuffleList(entries);
DataSampling retrieveStats = RetrieveEntriesPerf(rtable, entries);
Console.WriteLine("[U]pdate stats ...");
entries = ShuffleList(entries);
DataSampling updateStats = UpdateEntriesPerf(rtable, entries);
Console.WriteLine("[D]elete stats ...");
entries = ShuffleList(entries);
DataSampling deleteStats = DeleteEntriesPerf(rtable, entries);
//// TODO: Add perf. tests for LINQ query ...
IEnumerable <CustomerEntity> customers = GetCustomerEntities(rtable);
Assert.AreEqual(customers.Count(), 0);
var report = new StringBuilder();
report.AppendLine("RTable Stable Two-Replicas CRUD perf (ms)");
report.AppendFormat("{0}\n", createStats.ToString());
report.AppendFormat("{0}\n", retrieveStats.ToString());
report.AppendFormat("{0}\n", updateStats.ToString());
report.AppendFormat("{0}\n", deleteStats.ToString());
report.AppendLine("End report.");
Console.WriteLine(report);
}
catch (Exception ex)
{
Console.WriteLine("{0}", ex);
Assert.Fail();
}
finally
{
base.DeleteAllRtableResources();
}
}
/// <summary>
/// Call this function to modify the contents of the RTable configuration blob to use the specified viewId
/// and convertXStoreTableMode and update the appropriate wrapper.
/// </summary>
/// <param name="updatedViewId"></param>
protected void RefreshRTableEnvJsonConfigBlob(long updatedViewId, bool convertXStoreTableMode = false,
int readViewHeadIndex = 0, List <int> blobIndexes = null, bool waitForConfig = true)
{
if (blobIndexes == null)
{
blobIndexes = new List <int> {
0
};
}
Console.WriteLine(
"Calling RefreshRTableEnvJsonConfigBlob() with updatedViewId={0} convertXStoreTableMode={1} readViewHeadIndex={2}",
updatedViewId, convertXStoreTableMode, readViewHeadIndex);
this.ModifyConfigurationBlob(updatedViewId, convertXStoreTableMode, readViewHeadIndex, blobIndexes);
this.repTable = new ReplicatedTable(this.repTable.TableName, this.configurationService);
this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);
if (!waitForConfig)
{
return;
}
while (this.configurationWrapper.GetWriteView().ViewId != updatedViewId)
{
Console.WriteLine("Sleeping {0} seconds for new viewId to take effect...",
Constants.LeaseDurationInSec / 2);
Thread.Sleep((Constants.LeaseDurationInSec / 2) * 1000);
}
}
private DataSampling UpdateEntriesPerf(ReplicatedTable rtable, List <CustomerEntity> entries)
{
var stats = new DataSampling("[U]pdate");
foreach (var entry in entries)
{
TableOperation retrieveOperation = TableOperation.Retrieve <CustomerEntity>(entry.PartitionKey, entry.RowKey);
TableResult retrieveResult = rtable.Execute(retrieveOperation);
Assert.IsNotNull(retrieveResult, "retrieveResult = null");
Assert.AreEqual((int)HttpStatusCode.OK, retrieveResult.HttpStatusCode, "retrieveResult.HttpStatusCode mismatch");
Assert.IsNotNull((CustomerEntity)retrieveResult.Result, "Retrieve: customer = null");
// Update entity
var customer = (CustomerEntity)retrieveResult.Result;
customer.Email = string.Format("{0}.{1}@email.com", entry.PartitionKey, entry.RowKey);
TableOperation updateOperation = TableOperation.Replace(customer);
TableResult updateResult;
var watch = System.Diagnostics.Stopwatch.StartNew();
{
updateResult = rtable.Execute(updateOperation);
}
watch.Stop();
stats.AddPoint(watch.ElapsedMilliseconds);
Assert.IsNotNull(updateResult, "updateResult = null");
Assert.AreEqual((int)HttpStatusCode.NoContent, updateResult.HttpStatusCode, "updateResult.HttpStatusCode mismatch");
}
return(stats);
}
public ReplicatedTableV2(string name, ReplicatedTableConfigurationServiceV2 replicatedTableConfigurationAgent)
{
this._configurationWrapper = new ReplicatedTableConfigurationV2Wrapper(name, replicatedTableConfigurationAgent);
TableName = name;
_replicatedTableInstance = new ReplicatedTable(name, replicatedTableConfigurationAgent);
}
private void UpdateCustomer(CustomerEntity customer, ReplicatedTable repTable)
{
TableOperation operation = TableOperation.Replace(customer);
TableResult updateResult = repTable.Execute(operation);
Assert.IsNotNull(updateResult, "updateResult = null");
Assert.AreEqual((int)HttpStatusCode.NoContent, updateResult.HttpStatusCode, "updateResult.HttpStatusCode mismatch");
}
/// <summary>
/// Setup the RTable Env in order to run unit tests.
/// </summary>
/// <param name="tableName">Name of the RTable. If string.Empty or null, then use this.rtableTestConfiguration.RTableInformation.RTableName</param>
/// <param name="useHttps">When using HttpMangler, set this to false</param>
/// <param name="viewIdString">_rtable_ViewId used in the RTable config blob. Set to string.Empty or null to use the _rtable_ViewId value in the xml config</param>
/// <param name="actualStorageAccountsToBeUsed">Specify how/which Storage accounts in the config.xml will be used to construct the RTable. Set to null to use all accounts in xml. E.g., {0,1}</param>
protected void SetupRTableEnv(
string tableName = "",
bool useHttps = true,
string viewIdString = "",
List <int> actualStorageAccountsToBeUsed = null,
bool convertXStoreTableMode = false,
int numberOfBlobs = 0)
{
int numberOfStorageAccounts = this.rtableTestConfiguration.StorageInformation.AccountNames.Count();
// If user does not specify which storage accounts to construct the RTable, then use all the storage accounts in the xml.
this.actualStorageAccountsUsed = actualStorageAccountsToBeUsed;
if (this.actualStorageAccountsUsed == null || this.actualStorageAccountsUsed.Count == 0)
{
this.actualStorageAccountsUsed = new List <int>();
for (int i = 0; i < numberOfStorageAccounts; i++)
{
this.actualStorageAccountsUsed.Add(i);
}
}
this.UpdateConfigurationConstants();
// Upload RTable configuration to blob
int viewId = this.rtableTestConfiguration.RTableInformation.ViewId;
if (!string.IsNullOrEmpty(viewIdString))
{
viewId = int.Parse(viewIdString);
}
numberOfBlobs = numberOfBlobs == 0
? this.rtableTestConfiguration.StorageInformation.AccountKeys.Count()
: numberOfBlobs;
if (string.IsNullOrEmpty(tableName))
{
tableName = this.rtableTestConfiguration.RTableInformation.RTableName;
}
this.InitConnectionStringMap(useHttps);
InitialSetup(tableName, useHttps, viewId, convertXStoreTableMode, numberOfBlobs);
this.repTable = new ReplicatedTable(tableName, this.configurationService);
this.repTable.CreateIfNotExists();
this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);
this.cloudTableClients = new List <CloudTableClient>();
this.cloudTables = new List <CloudTable>();
for (int i = 0; i < this.actualStorageAccountsUsed.Count; i++)
{
CloudTableClient cloudBloblClient = repTable.GetReplicaTableClient(i);
this.cloudTableClients.Add(cloudBloblClient);
this.cloudTables.Add(cloudBloblClient.GetTableReference(repTable.TableName));
}
}
private void InsertCustormer(CustomerEntity customer, ReplicatedTable repTable)
{
TableOperation operation = TableOperation.Insert(customer);
TableResult result = repTable.Execute(operation);
Assert.AreNotEqual(null, result, "result = null");
Assert.AreEqual((int)HttpStatusCode.NoContent, result.HttpStatusCode, "result.HttpStatusCode mismatch");
Assert.AreNotEqual(null, result.Result, "result.Result = null");
}
public void LinqQueriesDontReturnPhysicallyDeletedEntries()
{
var rtable = new ReplicatedTable(this.repTable.TableName, this.configurationService);
string firstName = "FirstName";
string lastName = "LastName";
/*
* 1 - insert entries
*/
for (int i = 0; i < 10; i++)
{
var customer = new CustomerEntity(firstName + i, lastName + i);
TableOperation operation = TableOperation.Insert(customer);
rtable.Execute(operation);
}
/*
* 2 - delete entries #2 and #4
*/
foreach (var i in new int[] { 2, 4 })
{
TableOperation operation = TableOperation.Retrieve <CustomerEntity>(firstName + i, lastName + i);
TableResult retrievedResult = rtable.Execute(operation);
var customer = (CustomerEntity)retrievedResult.Result;
TableOperation deleteOperation = TableOperation.Delete(customer);
TableResult deleteResult = rtable.Execute(deleteOperation);
Assert.IsNotNull(deleteResult, "deleteResult = null");
Assert.AreEqual((int)HttpStatusCode.NoContent, deleteResult.HttpStatusCode, "deleteResult.HttpStatusCode mismatch");
}
/*
* 3 - CreateQuery doesn't return entries #2 and #4
*/
foreach (var customer in rtable.CreateReplicatedQuery <CustomerEntity>().AsEnumerable())
{
int id = int.Parse(customer.PartitionKey.Replace(firstName, ""));
Assert.AreNotEqual(id, 2, "entry #2 should have been deleted");
Assert.AreNotEqual(id, 4, "entry #4 should have been deleted");
}
/*
* 4 - ExecuteQuery doesn't return entries #2 and #4
*/
foreach (var customer in rtable.ExecuteQuery <CustomerEntity>(new TableQuery <CustomerEntity>()))
{
int id = int.Parse(customer.PartitionKey.Replace(firstName, ""));
Assert.AreNotEqual(id, 2, "entry #2 should have been deleted");
Assert.AreNotEqual(id, 4, "entry #4 should have been deleted");
}
}
private CustomerEntity RetrieveCustomer(string firstName, string lastName, ReplicatedTable repTable)
{
TableOperation operation = TableOperation.Retrieve <CustomerEntity>(firstName, lastName);
TableResult retrievedResult = repTable.Execute(operation);
Assert.AreNotEqual(null, retrievedResult, "retrievedResult = null");
Assert.AreEqual((int)HttpStatusCode.OK, retrievedResult.HttpStatusCode, "retrievedResult.HttpStatusCode mismatch");
Assert.AreNotEqual(null, retrievedResult.Result, "retrievedResult.Result = null");
return((CustomerEntity)retrievedResult.Result);
}
public IChainTable GetChainTable(string tableId)
{
if (tableId.StartsWith("__RTable_"))
{
var name = tableId.Substring(9);
ReplicatedTableConfigurationService rtableConfig = new ReplicatedTableConfigurationService(rTabConfLocs, true);
ReplicatedTable rTable = new ReplicatedTable(name, rtableConfig);
return(new RTableAdapter(rTable));
}
else
{
return(new ATableAdapter(client.GetTableReference(tableId)));
}
}
public void RTableCreateTableAlreadyExistsSync()
{
try
{
string tableName = this.GenerateRandomTableName();
this.SetupRTableEnv(tableName);
Assert.IsTrue(this.repTable.Exists(), "RTable does not exist");
// Try to create the same RTable again.
ReplicatedTable curTable = new ReplicatedTable(this.repTable.TableName, this.configurationService);
Assert.IsFalse(curTable.CreateIfNotExists(), "Calling CreateIfNotExists() again returned true. Should be false.");
int replicasCreated = curTable.replicasCreated;
Assert.AreEqual(replicasCreated, 0, "Calling CreateIfNotExists() again returned replicasCreated={0} > 0. Should be 0.", replicasCreated);
}
finally
{
base.DeleteAllRtableResources();
}
}
public void RTableCreateTableAlreadyExistsSync()
{
try
{
string tableName = this.GenerateRandomTableName();
this.SetupRTableEnv(tableName);
Assert.IsTrue(this.repTable.Exists(), "RTable does not exist");
// Try to create the same RTable again.
ReplicatedTable curTable = new ReplicatedTable(this.repTable.TableName, this.configurationService);
Assert.IsFalse(curTable.CreateIfNotExists(), "Calling CreateIfNotExists() again returned true. Should be false.");
int replicasCreated = curTable.replicasCreated;
Assert.AreEqual(replicasCreated, 0, "Calling CreateIfNotExists() again returned replicasCreated={0} > 0. Should be 0.", replicasCreated);
}
finally
{
base.DeleteAllRtableResources();
}
}
private DataSampling CreateEntriesStats(ReplicatedTable rtable, List <CustomerEntity> entries)
{
var stats = new DataSampling("[C]reate");
foreach (var entry in entries)
{
TableOperation insertOperation = TableOperation.Insert(entry);
TableResult insertResult = null;
var watch = System.Diagnostics.Stopwatch.StartNew();
{
insertResult = rtable.Execute(insertOperation);
}
watch.Stop();
stats.AddPoint(watch.ElapsedMilliseconds);
Assert.IsNotNull(insertResult, "insertResult = null");
Assert.AreEqual((int)HttpStatusCode.NoContent, insertResult.HttpStatusCode, "insertResult.HttpStatusCode mismatch");
}
return(stats);
}
public void TxnViewExpiresAndViewIdHasChanged()
{
// Insert an entry
string entityPartitionKey = "jobType-ReplaceWhenTxnViewExpires";
string entityRowKey = "jobType-ReplaceWhenTxnViewExpires";
this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);
Console.WriteLine("Inserting entry ...");
var entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, "insert message");
this.rtableWrapper.InsertRow(entry);
// Retrieve the config ...
ReplicatedTableConfiguration config;
ReplicatedTableQuorumReadResult readStatus = this.configurationService.RetrieveConfiguration(out config);
Assert.IsTrue(readStatus.Code == ReplicatedTableQuorumReadCode.Success);
// Reconfigure RTable config with a short LeaseDuration
int leaseDurationInSec = 3;
config.LeaseDuration = leaseDurationInSec;
this.configurationService.UpdateConfiguration(config);
string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0];
Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper);
// Delay behavior
ProxyBehavior[] behaviors = new[]
{
DelayBehaviors.DelayAllResponsesIf(
1,
(session =>
{
var body = session.GetRequestBodyAsString();
// Delay lock of the Head response enough so the txView expires ...
// And, upload a new config with a new viewId
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.HTTPMethodIs("PUT") &&
body.Contains("\"_rtable_Operation\":\"Replace\"") &&
body.Contains("\"_rtable_RowLock\":true"))
{
// Upload a new config with a new ViewId
View view = this.configurationService.GetTableView(this.repTable.TableName);
config.GetView(view.Name).ViewId++;
this.configurationService.UpdateConfiguration(config, false);
// ensure txView expires
Thread.Sleep(2 * leaseDurationInSec * 1000);
return(true);
}
return(false);
})),
};
// Launch an Update
using (new HttpMangler(false, behaviors))
{
try
{
entry = this.rtableWrapper.ReadEntity(entityPartitionKey, entityRowKey);
entry.Message = "update message";
var table = new ReplicatedTable(this.repTable.TableName, this.configurationService);
TableOperation replaceOperation = TableOperation.Replace(entry);
// Get a snapshot of txView before transaction starts ...
View viewBeforeTx = this.configurationService.GetTableView(this.repTable.TableName);
// Transaction will get delayed 2*LeaseDuration sec. and Config/ViewId will be changed
TableResult replaceResult = table.Execute(replaceOperation);
Assert.IsTrue(viewBeforeTx.IsExpired(), "txView expected to expire!");
Assert.AreNotEqual(this.configurationService.GetTableView(this.repTable.TableName).ViewId, viewBeforeTx.ViewId, "ViewId should have changed!");
Assert.IsNotNull(replaceResult, "upsertResult = null");
Assert.AreNotEqual((int)HttpStatusCode.NoContent, replaceResult.HttpStatusCode, "upsertResult.HttpStatusCode mismatch");
}
catch (ReplicatedTableStaleViewException ex)
{
// ValidateTxnView is called form everywhere ... we may endup here too
switch (ex.ErrorCode)
{
case ReplicatedTableViewErrorCodes.ViewIdChanged:
break;
default:
Console.WriteLine(ex);
Assert.IsTrue(false);
break;
}
}
}
}
public void TableQueryableCreateQueryNoPartitionKey()
{
Thread.Sleep(10000);
string tableName = this.GenerateRandomTableName();
ReplicatedTable localRTable = new ReplicatedTable(tableName, this.configurationService);
localRTable.CreateIfNotExists();
RTableWrapperForSampleRTableEntity localRTableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(localRTable);
CloudTableClient tableClient = localRTable.GetTailTableClient();
CloudTable table = tableClient.GetTableReference(localRTable.TableName);
string pk = "0";
try
{
try
{
TableBatchOperation batch = new TableBatchOperation();
for (int j = 0; j < 10; j++)
{
BaseEntity ent = GenerateRandomEntity(pk);
ent.RowKey = string.Format("{0:0000}", j);
batch.Insert(ent);
}
localRTable.ExecuteBatch(batch);
}
catch (Exception ex)
{
Console.WriteLine("Exception during test case init {0}", ex.ToString());
throw;
}
try
{
pk = "1";
TableBatchOperation batch = new TableBatchOperation();
for (int j = 0; j < 10; j++)
{
BaseEntity ent = GenerateRandomEntity(pk);
ent.RowKey = string.Format("{0:0000}", j);
batch.Insert(ent);
}
localRTable.ExecuteBatch(batch);
}
catch (Exception ex)
{
Console.WriteLine("Exception during test case init {0}", ex.ToString());
throw;
}
IQueryable<BaseEntity> tableQuery = table.CreateQuery<BaseEntity>();
IQueryable<BaseEntity> rtableQuery = localRTable.CreateQuery<BaseEntity>();
var list = tableQuery.AsEnumerable();
int tableCount = 0;
int rtableCount = 0;
foreach (BaseEntity ent in list)
{
tableCount++;
}
foreach (BaseEntity ent in rtableQuery.ToList())
{
rtableCount++;
}
Assert.IsTrue(tableCount == rtableCount, "Query counts are different");
Assert.IsTrue(tableCount == 20, "Query counts are different");
}
catch (Exception e)
{
Console.WriteLine("Error during query processing: {0}", e.ToString());
}
finally
{
localRTable.DeleteIfExists();
}
}
private ReplicatedTableRepairResult RepairTable(string tableName, string storageAccountName, ReplicatedTableConfiguration configuration)
{
string viewName = "";
try
{
ReplicatedTableConfiguredTable tableConfig;
if (!configuration.IsConfiguredTable(tableName, out tableConfig))
{
return(new ReplicatedTableRepairResult(ReplicatedTableRepairCode.NotConfiguredTable, tableName));
}
viewName = tableConfig.ViewName;
List <ReplicaInfo> list = configuration.GetView(viewName).ReplicaChain;
if (!list.Any() ||
list[0].StorageAccountName != storageAccountName)
{
return(new ReplicatedTableRepairResult(ReplicatedTableRepairCode.NotImpactedTable, tableName, viewName, storageAccountName));
}
ReplicaInfo head = list[0];
if (head.Status != ReplicaStatus.WriteOnly)
{
return(new ReplicatedTableRepairResult(ReplicatedTableRepairCode.StableTable, tableName, viewName, storageAccountName));
}
int viewIdToRecoverFrom = (int)head.ViewWhenTurnedOff;
ReplicatedTableLogger.LogInformational("RepairTable={0}, View={1}, StorageAccountName={2}, from viewId={3} ...",
tableName,
viewName,
storageAccountName,
viewIdToRecoverFrom);
// Repairing ...
ReconfigurationStatus status = new ReplicatedTable(tableName, this).RepairTable(viewIdToRecoverFrom, null);
ReplicatedTableLogger.LogInformational("RepairTable={0}, View={1}, StorageAccountName={2}, from viewId={3} => Status={4}",
tableName,
viewName,
storageAccountName,
viewIdToRecoverFrom,
status);
if (status == ReconfigurationStatus.SUCCESS)
{
return(new ReplicatedTableRepairResult(ReplicatedTableRepairCode.Success, tableName, viewName, storageAccountName));
}
// Failure!
return(new ReplicatedTableRepairResult(ReplicatedTableRepairCode.Error, tableName, viewName, storageAccountName)
{
Status = status,
});
}
catch (Exception ex)
{
return(new ReplicatedTableRepairResult(ReplicatedTableRepairCode.Error, tableName, viewName, storageAccountName)
{
Status = ReconfigurationStatus.FAILURE,
Message = ex.ToString(),
});
}
}
public void TableQueryableCreateQueryNoPartitionKey()
{
Thread.Sleep(10000);
string tableName = this.GenerateRandomTableName();
ReplicatedTable localRTable = new ReplicatedTable(tableName, this.configurationService);
localRTable.CreateIfNotExists();
RTableWrapperForSampleRTableEntity localRTableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(localRTable);
CloudTableClient tableClient = localRTable.GetTailTableClient();
CloudTable table = tableClient.GetTableReference(localRTable.TableName);
string pk = "0";
try
{
try
{
TableBatchOperation batch = new TableBatchOperation();
for (int j = 0; j < 10; j++)
{
BaseEntity ent = GenerateRandomEntity(pk);
ent.RowKey = string.Format("{0:0000}", j);
batch.Insert(ent);
}
localRTable.ExecuteBatch(batch);
}
catch (Exception ex)
{
Console.WriteLine("Exception during test case init {0}", ex.ToString());
throw;
}
try
{
pk = "1";
TableBatchOperation batch = new TableBatchOperation();
for (int j = 0; j < 10; j++)
{
BaseEntity ent = GenerateRandomEntity(pk);
ent.RowKey = string.Format("{0:0000}", j);
batch.Insert(ent);
}
localRTable.ExecuteBatch(batch);
}
catch (Exception ex)
{
Console.WriteLine("Exception during test case init {0}", ex.ToString());
throw;
}
IQueryable <BaseEntity> tableQuery = table.CreateQuery <BaseEntity>();
IQueryable <BaseEntity> rtableQuery = localRTable.CreateQuery <BaseEntity>();
var list = tableQuery.AsEnumerable();
int tableCount = 0;
int rtableCount = 0;
foreach (BaseEntity ent in list)
{
tableCount++;
}
foreach (BaseEntity ent in rtableQuery.ToList())
{
rtableCount++;
Assert.IsTrue(ent.ETag != ent._rtable_Version.ToString(), "ETag is not virtualized when using CreateQuery()");
}
Assert.IsTrue(tableCount == rtableCount, "Query counts are different");
Assert.IsTrue(tableCount == 20, "Query counts are different");
// But, with "CreateReplicatedQuery" ETag is virtualized
IQueryable <BaseEntity> virtualizedRtableQuery = localRTable.CreateReplicatedQuery <BaseEntity>();
foreach (BaseEntity ent in virtualizedRtableQuery.ToList())
{
Assert.IsTrue(ent._rtable_Version == 0);
Assert.IsTrue(ent.ETag == ent._rtable_Version.ToString(), "ETag is virtualized when using CreateReplicatedQuery()");
ent.A += "`";
// Update should go fine since ETag is virtualized
TableOperation operation = TableOperation.Replace(ent);
TableResult result = localRTable.Execute(operation);
Assert.IsTrue(result != null && result.HttpStatusCode == (int)HttpStatusCode.NoContent);
}
virtualizedRtableQuery = localRTable.CreateReplicatedQuery <BaseEntity>();
foreach (BaseEntity ent in virtualizedRtableQuery.ToList())
{
Assert.IsTrue(ent._rtable_Version == 1);
Assert.IsTrue(ent.ETag == ent._rtable_Version.ToString(), "ETag is virtualized when using CreateReplicatedQuery()");
}
}
catch (Exception e)
{
Console.WriteLine("Error during query processing: {0}", e.ToString());
}
finally
{
localRTable.DeleteIfExists();
}
}
/// <summary>
/// Initializes a new instance of the <see cref="DynamicReplicatedTableEntity"/> class.
/// </summary>
public DynamicReplicatedTableEntity()
: base()
{
this.Properties = new Dictionary <string, EntityProperty>();
this._rtable_Operation = ReplicatedTable.GetTableOperation(TableOperationType.Insert);
}
public void ConfigurationChangeNotificationAfterDetectingStaleView()
{
ReplicatedTableConfigurationServiceV2 configServiceOne, configServiceTwo;
// setup:
// stale view = [H] -> [T]
// new view = [H]
SetupStaleViewAndNewView(out configServiceOne, out configServiceTwo);
long staleViewId = configServiceOne.GetTableView(this.repTable.TableName).ViewId;
long latestViewId = configServiceTwo.GetTableView(this.repTable.TableName).ViewId;
string firstName = "FirstName01";
string lastName = "LastName01";
/*
* 1 - WorkerOne => inserts an entry using the stale View
*/
var workerOne = new ReplicatedTable(this.repTable.TableName, configServiceOne);
var customer = new CustomerEntity(firstName, lastName);
customer.Email = "*****@*****.**";
InsertCustormer(customer, workerOne);
customer = RetrieveCustomer(firstName, lastName, workerOne);
Assert.AreEqual(staleViewId, customer._rtable_ViewId, "customer._rtable_ViewId mismatch");
Assert.AreEqual("*****@*****.**", customer.Email, "customer.Email mismatch");
Console.WriteLine("workerOne inserted a customer in ViewId={0}", staleViewId);
/*
* 2 - WorkerTwo => updates the entry using latest View
* row is updated in [Head] replica
*/
var workerTwo = new ReplicatedTable(this.repTable.TableName, configServiceTwo);
customer = RetrieveCustomer(firstName, lastName, workerTwo);
customer.Email = "*****@*****.**";
UpdateCustomer(customer, workerTwo);
customer = RetrieveCustomer(firstName, lastName, workerTwo);
Assert.AreEqual(latestViewId, customer._rtable_ViewId, "customer._rtable_ViewId mismatch");
Assert.AreEqual("*****@*****.**", customer.Email, "customer.Email mismatch");
Console.WriteLine("workerTwo updated the customer in ViewId={0}", latestViewId);
/*
* 3 - WorkerOne => access existing entry using "the stale View"
* Read is served from [Head] i.e. we are not reading stale data from [Tail]
*/
try
{
customer = RetrieveCustomer(firstName, lastName, workerOne);
Assert.Fail("Retrieve() is expected to get an RTableStaleViewException but did not get it.");
}
catch (ReplicatedTableStaleViewException ex)
{
Console.WriteLine("workerOne got exception: {0}", ex.Message);
Assert.IsTrue(ex.ErrorCode == ReplicatedTableViewErrorCodes.ViewIdSmallerThanEntryViewId);
Assert.IsTrue(ex.Message.Contains(string.Format("current _rtable_ViewId {0} is smaller than _rtable_ViewId of existing row {1}", staleViewId, latestViewId)), "Got unexpected exception message");
}
// Note:
// No need to test all table APIs since already covered by UT:
// => "Microsoft.Azure.Toolkit.Replication.Test.RTableViewIdTests.ExceptionWhenUsingSmallerViewId()"
/*
* 4 - Notify "configServiceOne" instance of config change
*/
Assert.AreEqual(configServiceOne.GetTableView(this.repTable.TableName).ViewId, staleViewId, "we should see old View!!!");
{
configServiceOne.ConfigurationChangeNotification();
}
Assert.AreEqual(configServiceOne.GetTableView(this.repTable.TableName).ViewId, latestViewId, "we should see latest View!!!");
/*
* 5 - WorkerOne => updates the entry using latest View
*/
customer = RetrieveCustomer(firstName, lastName, workerTwo);
customer.Email = "*****@*****.**";
UpdateCustomer(customer, workerTwo);
customer = RetrieveCustomer(firstName, lastName, workerTwo);
Assert.AreEqual(latestViewId, customer._rtable_ViewId, "customer._rtable_ViewId mismatch");
Assert.AreEqual("*****@*****.**", customer.Email, "customer.Email mismatch");
Console.WriteLine("workerOne updated the customer in ViewId={0}", latestViewId);
}
public void ReadingUncommittedDataFromReplicaIdentifiedByReadTailIndexWontThrow()
{
ReplicatedTableConfigurationServiceV2 configServiceOne, configServiceTwo;
// setup: Acc0 Acc1
// stale view = [H] -> [T]
// new view = [H]
SetupStaleViewAndNewView(out configServiceOne, out configServiceTwo);
long staleViewId = configServiceOne.GetTableView(this.repTable.TableName).ViewId;
long latestViewId = configServiceTwo.GetTableView(this.repTable.TableName).ViewId;
string firstName = "FirstName01";
string lastName = "LastName01";
/*
* 1 - WorkerOne => inserts an entry in stale View
*/
var workerOne = new ReplicatedTable(this.repTable.TableName, configServiceOne);
var workerTwo = new ReplicatedTable(this.repTable.TableName, configServiceTwo);
var customer = new CustomerEntity(firstName, lastName);
customer.Email = "***";
InsertCustormer(customer, workerOne);
string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[1];
Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper);
TableResult oldUpdateResult = null;
CustomerEntity customerSeenByNewView = null;
// Delay behavior
ProxyBehavior[] behaviors =
{
TamperBehaviors.TamperAllRequestsIf(
(session =>
{
session.oRequest.FailSession((int)HttpStatusCode.ServiceUnavailable, "ServerBusy", "");
}),
(session =>
{
var body = session.GetRequestBodyAsString();
// Fail Lock to Tail by stale view
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.HTTPMethodIs("PUT") &&
body.Contains("\"Email\":\"workerOne\"") &&
body.Contains(string.Format("\"_rtable_ViewId\":\"{0}\"", staleViewId)) &&
body.Contains("\"_rtable_RowLock\":true"))
{
return(true);
}
return(false);
})),
};
/*
* 2 - WorkerOne => update an entry using the stale View
*/
using (new HttpMangler(false, behaviors))
{
customer = RetrieveCustomer(firstName, lastName, workerOne);
customer.Email = "workerOne";
TableOperation operation = TableOperation.Replace(customer);
oldUpdateResult = workerOne.Execute(operation);
}
// Expected behavior:
// Thread_1 (stale ViewId) fails to commit to Tail
Assert.IsNotNull(oldUpdateResult, "oldUpdateResult = null");
Assert.AreEqual((int)HttpStatusCode.ServiceUnavailable, oldUpdateResult.HttpStatusCode, "oldUpdateResult.HttpStatusCode mismatch");
Console.WriteLine("Update in stale View Succeeded with HttpStatus={0}", oldUpdateResult.HttpStatusCode);
// Thread_1 (stale ViewId): Reads the entry => Succeeds eventhough the row is "uncommitted".
// "ReadTailIndex = [Head]" means [Head] is treated as [Tail] for Reads.
// => row is assumed "committed"
customer = RetrieveCustomer(firstName, lastName, workerOne);
Assert.IsNotNull(customer, "customer = null");
Assert.AreEqual("workerOne", customer.Email, "customer.Email mismatch");
// Thread_2 (new ViewId): Reads the entry => Succeed since reading from Head
customerSeenByNewView = RetrieveCustomer(firstName, lastName, workerTwo);
Assert.IsNotNull(customerSeenByNewView, "customerSeenByNewView = null");
Assert.AreEqual("workerOne", customer.Email, "customer.Email mismatch");
}
public void A00TwoInsertOrReplaceCallsConflictingOnTheHead()
{
this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);
string entityPartitionKey = "jobType-DelayInsertOrReplaceRowHeadTest";
string entityRowKey = "jobId-DelayInsertOrReplaceRowHeadTest";
this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);
// Insert one entry
Console.WriteLine("Inserting entry ...");
var entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, "insert message");
this.rtableWrapper.InsertRow(entry);
string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0];
Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper);
int delayInMs = 3000;
bool firstWritterInitiatingCommit = false;
bool secondWritterTriedLockingHead = false;
// Delay behavior
ProxyBehavior[] behaviors = new[]
{
// Writter-1 tampering
TamperBehaviors.TamperAllRequestsIf(
(session =>
{
int iter = 0;
// Signal Writter-2
firstWritterInitiatingCommit = true;
// Blobk commit to head ... until Writter-2 try to lock the head
while (!secondWritterTriedLockingHead)
{
Console.WriteLine("Writter-1 waiting on Writter-2 to try to lock the Head (#{0})", iter);
Thread.Sleep(delayInMs);
if (++iter > 10)
{
break;
}
}
}),
(session =>
{
var body = session.GetRequestBodyAsString();
// Writter-1 committing to head
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.HTTPMethodIs("PUT") &&
body.Contains("\"_rtable_Operation\":\"Replace\"") &&
body.Contains("\"_rtable_RowLock\":false") &&
body.Contains("\"Message\":\"upsert message 0\"")
)
{
return(true);
}
return(false);
})),
// Writter-2 tampering
TamperBehaviors.TamperAllRequestsIf(
(session =>
{
// Block till Writter-1 issues a commit to head
while (!firstWritterInitiatingCommit)
{
Console.WriteLine("Writter-2 waiting on Writter-1 to issue a commit to head");
Thread.Sleep(delayInMs);
}
}),
(session =>
{
var body = session.GetRequestBodyAsString();
// Writter-2 locking the head
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.HTTPMethodIs("PUT") &&
body.Contains("\"_rtable_Operation\":\"Replace\"") &&
body.Contains("\"_rtable_RowLock\":true") &&
body.Contains("\"Message\":\"upsert message 1\"")
)
{
return(true);
}
return(false);
})),
// Delay Writter-2 lock-to-the-head's response, so Writter-1 can continue with its commit.
DelayBehaviors.DelayAllResponsesIf(
delayInMs,
(session =>
{
var body = session.GetRequestBodyAsString();
// Writter-2 locking the head response
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.HTTPMethodIs("PUT") &&
body.Contains("\"_rtable_Operation\":\"Replace\"") &&
body.Contains("\"_rtable_RowLock\":true") &&
body.Contains("\"Message\":\"upsert message 1\"")
)
{
// Signal Writter-1 so it can continue with commit to head
secondWritterTriedLockingHead = true;
return(true);
}
return(false);
})),
};
// Launch 2 concurrent Upserts
var results = new TableResult[2];
using (new HttpMangler(false, behaviors))
{
Parallel.For(0, 2, (index) =>
{
entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, string.Format("upsert message {0}", index));
try
{
var table = new ReplicatedTable(this.repTable.TableName, this.configurationService);
TableOperation operation = TableOperation.InsertOrReplace(entry);
results[index] = table.Execute(operation);
}
catch (AggregateException ex)
{
Console.WriteLine(ex);
}
});
}
// Writter-1 suceed?
Assert.AreEqual((int)HttpStatusCode.NoContent, results[0].HttpStatusCode, "Writter-1 expected to suceed!");
// Writter-2 suceeded?
Assert.AreEqual((int)HttpStatusCode.NoContent, results[1].HttpStatusCode, "Writter-2 expected to suceed!");
// Writter-2 upsert succeeded
SampleRTableEntity upsertedEntity = this.rtableWrapper.ReadEntity(entityPartitionKey, entityRowKey);
Assert.NotNull(upsertedEntity, "Writter-2 upsert succeeded");
Assert.AreEqual(upsertedEntity.Message, string.Format("upsert message {0}", 1), "Writter-2 upsert succeeded");
}
public void WhenReadFromTailFailsWithServiceUnavailableWeReadFromHeadAndSucceed()
{
View view = this.configurationWrapper.GetWriteView();
Assert.IsTrue(view.Chain.Count > 1, "expects at least 2 replicas!");
// Insert one entry
var rtable = new ReplicatedTable(this.repTable.TableName, this.configurationService);
string firstName = "FirstName01";
string lastName = "LastName01";
var customer = new CustomerEntity(firstName, lastName);
TableOperation operation = TableOperation.Insert(customer);
rtable.Execute(operation);
// Using xstore modify the row in each replica individually ... so we know, later, which replica RTable will retrieve from
for (int replicaIndex = 0; replicaIndex < this.cloudTableClients.Count; replicaIndex++)
{
CloudTableClient tableClient = this.cloudTableClients[replicaIndex];
CloudTable table = tableClient.GetTableReference(this.repTable.TableName);
TableOperation retrieveOperation = TableOperation.Retrieve <CustomerEntity>(firstName, lastName);
TableResult retrieveResult = table.Execute(retrieveOperation);
customer = (CustomerEntity)retrieveResult.Result;
customer.Email = replicaIndex.ToString(); // Head = 0
// ... = 1
// Tail = 2
TableOperation updateOperation = TableOperation.Replace(customer);
TableResult updateResult = table.Execute(updateOperation);
Assert.IsNotNull(updateResult, "updateResult = null");
Console.WriteLine("updateResult.HttpStatusCode = {0}", updateResult.HttpStatusCode);
Assert.AreEqual((int)HttpStatusCode.NoContent, updateResult.HttpStatusCode, "updateResult.HttpStatusCode mismatch");
}
string accountNameToNotTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0];
Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToNotTamper={0}", accountNameToNotTamper);
// Delay behavior
ProxyBehavior[] behaviors =
{
TamperBehaviors.TamperAllRequestsIf(
(session =>
{
session.oRequest.FailSession((int)HttpStatusCode.ServiceUnavailable, "ServerBusy", "");
}),
(session =>
{
if (session.HTTPMethodIs("Get"))
{
// Fail Get from all replicas, except Head
if (!session.hostname.Contains(accountNameToNotTamper + "."))
{
return(true);
}
}
return(false);
})),
};
using (new HttpMangler(false, behaviors))
{
operation = TableOperation.Retrieve <CustomerEntity>(firstName, lastName);
TableResult retrievedResult = repTable.Execute(operation);
Assert.AreNotEqual(null, retrievedResult, "retrievedResult = null");
Assert.AreEqual((int)HttpStatusCode.OK, retrievedResult.HttpStatusCode, "retrievedResult.HttpStatusCode mismatch");
Assert.AreNotEqual(null, retrievedResult.Result, "retrievedResult.Result = null");
customer = (CustomerEntity)retrievedResult.Result;
Assert.AreEqual(customer.Email, (0).ToString(), "we should have read the row from Head");
}
}
public void LinqQueriesAreServedFromHeadWhenReadViewTailIndexIsSet()
{
var rtable = new ReplicatedTable(this.repTable.TableName, this.configurationService);
string firstName = "FirstName";
string lastName = "LastName";
int dataSize = 5;
/*
* 1 - insert entries
*/
for (int i = 0; i < dataSize; i++)
{
var customer = new CustomerEntity(firstName + i, lastName + i);
customer.Email = "***";
TableOperation operation = TableOperation.Insert(customer);
rtable.Execute(operation);
}
// Identify the Tail account
View view = this.configurationWrapper.GetWriteView();
Assert.IsTrue(view.Chain.Count > 1, "expects at least one replica!");
string accountNameToTamper = view.Chain.Last().Item1.StorageAccountName;
Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper);
// We will fail requests to update a row in Tail
ProxyBehavior[] behaviors =
{
TamperBehaviors.TamperAllRequestsIf(
(session =>
{
session.oRequest.FailSession((int)HttpStatusCode.ServiceUnavailable, "ServerBusy", "");
}),
(session =>
{
var body = session.GetRequestBodyAsString();
// Fail Lock to Tail by stale view
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.HTTPMethodIs("PUT") &&
body.Contains("\"_rtable_RowLock\":true"))
{
return(true);
}
return(false);
})),
};
using (new HttpMangler(false, behaviors))
{
/*
* 2 - update entries
*/
for (int i = 0; i < dataSize; i++)
{
TableOperation operation = TableOperation.Retrieve <CustomerEntity>(firstName + i, lastName + i);
TableResult retrievedResult = rtable.Execute(operation);
var customer = (CustomerEntity)retrievedResult.Result;
customer.Email = "updated";
TableOperation replaceOperation = TableOperation.Replace(customer);
TableResult replaceResult = rtable.Execute(replaceOperation);
Assert.IsNotNull(replaceResult, "replaceResult = null");
Assert.AreEqual((int)HttpStatusCode.ServiceUnavailable, replaceResult.HttpStatusCode, "replaceResult.HttpStatusCode mismatch");
}
}
SetReadViewTailIndex(0);
Assert.AreEqual(0, this.configurationService.GetTableView(this.repTable.TableName).ReadTailIndex, "ReadTailIndex should be 0!!!");
/*
* 3 - CreateQuery is served from [Head], data should be new
*/
foreach (var customer in rtable.CreateReplicatedQuery <CustomerEntity>().AsEnumerable())
{
Assert.AreEqual(customer.Email, "updated", "expected new data");
}
/*
* 4 - ExecuteQuery is served from [Head], data should be new
*/
foreach (var customer in rtable.ExecuteQuery <CustomerEntity>(new TableQuery <CustomerEntity>()))
{
Assert.AreEqual(customer.Email, "updated", "expected new data");
}
}
public void A00InsertOrReplaceCallConflictingWithDeleteOnTheHead()
{
this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);
string entityPartitionKey = "jobType-DelayInsertOrReplaceWhileDelete";
string entityRowKey = "jobType-DelayInsertOrReplaceWhileDelete";
this.ForceDeleteEntryFromStorageTablesDirectly(entityPartitionKey, entityRowKey);
// Insert one entry
Console.WriteLine("Inserting entry ...");
var entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, "insert message");
this.rtableWrapper.InsertRow(entry);
// 1 - Launch an Upsert task in wait mode ...
bool deleteLockedHead = false;
TableResult upsertResult = null;
var upsertTask = Task.Run(() =>
{
while (!deleteLockedHead)
{
Thread.Sleep(5);
}
try
{
entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, "upsert message");
var table = new ReplicatedTable(this.repTable.TableName, this.configurationService);
TableOperation upserOperation = TableOperation.InsertOrReplace(entry);
Console.WriteLine("Upsert started ...");
upsertResult = table.Execute(upserOperation);
Console.WriteLine("Upsert completed with HttpStatus={0}", upsertResult == null ? "NULL" : upsertResult.HttpStatusCode.ToString());
}
catch (AggregateException ex)
{
Console.WriteLine(ex);
}
});
string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0];
Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper);
int delayInMs = 5;
// Delay behavior
ProxyBehavior[] behaviors = new[]
{
DelayBehaviors.DelayAllResponsesIf(
delayInMs,
(session =>
{
var body = session.GetRequestBodyAsString();
Console.WriteLine(body);
// Delete locking the head response
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.HTTPMethodIs("PUT") &&
body.Contains("\"_rtable_Operation\":\"Replace\"") &&
body.Contains("\"_rtable_RowLock\":true") &&
body.Contains("\"_rtable_Tombstone\":true"))
{
// Signal upsert we locked the head, so it can continue ...
deleteLockedHead = true;
return(true);
}
return(false);
})),
};
// Launch a delete
using (new HttpMangler(false, behaviors))
{
try
{
var table = new ReplicatedTable(this.repTable.TableName, this.configurationService);
// Retrieve entity
TableOperation retrieveOperation = TableOperation.Retrieve <SampleRTableEntity>(entry.PartitionKey, entry.RowKey);
TableResult retrieveResult = table.Execute(retrieveOperation);
Assert.IsNotNull(retrieveResult, "retrieveResult = null");
Assert.AreEqual((int)HttpStatusCode.OK, retrieveResult.HttpStatusCode, "retrieveResult.HttpStatusCode mismatch");
Assert.IsNotNull((SampleRTableEntity)retrieveResult.Result, "Retrieve: customer = null");
// Delete entity
TableOperation deleteOperation = TableOperation.Delete((SampleRTableEntity)retrieveResult.Result);
TableResult deleteResult = table.Execute(deleteOperation);
Assert.IsNotNull(deleteResult, "deleteResult = null");
Assert.AreEqual((int)HttpStatusCode.NoContent, deleteResult.HttpStatusCode, "deleteResult.HttpStatusCode mismatch");
}
catch (AggregateException ex)
{
Console.WriteLine(ex);
}
}
// wait on upsert to finish ...
upsertTask.Wait();
// Upsert suceeded?
Assert.AreEqual((int)HttpStatusCode.NoContent, upsertResult.HttpStatusCode, "Upsert expected to suceed!");
SampleRTableEntity upsertedEntity = this.rtableWrapper.ReadEntity(entityPartitionKey, entityRowKey);
Assert.NotNull(upsertedEntity, "upsert should succeed");
Assert.AreEqual(upsertedEntity.Message, "upsert message", "upsert should succeeded");
}
private IEnumerable <CustomerEntity> GetCustomerEntities(ReplicatedTable rtable)
{
ReplicatedTableQuery <CustomerEntity> query = rtable.CreateReplicatedQuery <CustomerEntity>();
return(query.AsEnumerable());
}
public void LinqQueriesDontReturnRowsWithHighViewIdWhenFlagIgnoreHigherViewIdRowsIsSet()
{
TableOperation operation;
TableResult result;
CustomerEntity customer;
/*
* Set config viewId to 5
*/
long staleViewId = 5;
SetConfigViewIdAndIgnoreHigherViewIdRowsFlag(staleViewId, false);
Assert.AreEqual(staleViewId, this.configurationService.GetTableView(this.repTable.TableName).ViewId, "View should be 5!!!");
// Insert entries in stale viewId 5
var rtable = new ReplicatedTable(this.repTable.TableName, this.configurationService);
string firstName = "FirstName";
string lastName = "LastName";
for (int i = 0; i < 10; i++)
{
customer = new CustomerEntity(firstName + i, lastName + i);
operation = TableOperation.Insert(customer);
rtable.Execute(operation);
}
/*
* Set config new viewId to 6
*/
long newViewId = 6;
SetConfigViewIdAndIgnoreHigherViewIdRowsFlag(newViewId, false);
Assert.AreEqual(newViewId, this.configurationService.GetTableView(this.repTable.TableName).ViewId, "View should be 6!!!");
// Update entry #5 and #8 in new viewId 6
foreach (int entryId in new int[] { 5, 8 })
{
operation = TableOperation.Retrieve <CustomerEntity>(firstName + entryId, lastName + entryId);
result = rtable.Execute(operation);
Assert.IsTrue(result != null && result.HttpStatusCode == (int)HttpStatusCode.OK && (CustomerEntity)result.Result != null, "Retrieve customer failed");
customer = (CustomerEntity)result.Result;
customer.Email = "*****@*****.**";
operation = TableOperation.Replace(customer);
result = rtable.Execute(operation);
Assert.IsTrue(result != null && result.HttpStatusCode == (int)HttpStatusCode.NoContent, "Update customer failed");
}
/*
* Simulate a stale client => Set config viewId back to 5
* and Set 'IgnoreHigherViewIdRows' flag so we ignore rows with higher viewIds
*/
SetConfigViewIdAndIgnoreHigherViewIdRowsFlag(staleViewId, true);
Assert.AreEqual(staleViewId, this.configurationService.GetTableView(this.repTable.TableName).ViewId, "View should be 5!!!");
try
{
// Check Retrieve of row #5 and #8 returns NotFound
foreach (int entryId in new int[] { 5, 8 })
{
operation = TableOperation.Retrieve <CustomerEntity>(firstName + entryId, lastName + entryId);
var retrievedResult = rtable.Execute(operation);
Assert.AreNotEqual(null, retrievedResult, "retrievedResult = null");
Assert.AreEqual((int)HttpStatusCode.NotFound, retrievedResult.HttpStatusCode, "retrievedResult.HttpStatusCode mismatch");
}
}
catch (ReplicatedTableStaleViewException)
{
Assert.Fail("Retrieve() is expected to NotFound the row, but got RTableStaleViewException !");
}
/*
* stale client using LINQ: CreateReplicatedQuery
*/
foreach (var entry in rtable.CreateReplicatedQuery <CustomerEntity>().AsEnumerable())
{
int id = int.Parse(entry.PartitionKey.Replace(firstName, ""));
Assert.AreNotEqual(id, 5, "row #5 should not be returned");
Assert.AreNotEqual(id, 8, "row #8 should not be returned");
Assert.AreEqual(entry._rtable_ViewId, staleViewId, "CreateReplicatedQuery: entry viewId should be '5'");
}
/*
* stale client using LINQ: ExecuteQuery
*/
foreach (var entry in rtable.ExecuteQuery <CustomerEntity>(new TableQuery <CustomerEntity>()))
{
int id = int.Parse(entry.PartitionKey.Replace(firstName, ""));
Assert.AreNotEqual(id, 5, "row #5 should not be returned");
Assert.AreNotEqual(id, 8, "row #8 should not be returned");
Assert.AreEqual(entry._rtable_ViewId, staleViewId, "CreateReplicatedQuery: entry viewId should be '5'");
}
}
public void LinqQueriesThrowAfterDetectingStaleViewWhenThrowOnStaleViewInLinqQueryFlagIsSet()
{
TableOperation operation;
TableResult result;
CustomerEntity customer;
/*
* Set config viewId to 5
*/
long staleViewId = 5;
SetConfigViewIdAndIgnoreHigherViewIdRowsFlag(staleViewId, false);
Assert.AreEqual(staleViewId, this.configurationService.GetTableView(this.repTable.TableName).ViewId, "View should be 5!!!");
// Insert entries in stale viewId 5
var rtable = new ReplicatedTable(this.repTable.TableName, this.configurationService);
string firstName = "FirstName";
string lastName = "LastName";
for (int i = 0; i < 10; i++)
{
customer = new CustomerEntity(firstName + i, lastName + i);
operation = TableOperation.Insert(customer);
rtable.Execute(operation);
}
/*
* Set config new viewId to 6
*/
long newViewId = 6;
SetConfigViewIdAndIgnoreHigherViewIdRowsFlag(newViewId, false);
Assert.AreEqual(newViewId, this.configurationService.GetTableView(this.repTable.TableName).ViewId, "View should be 6!!!");
// Update entry #5 in new viewId 6
int entryId = 5;
operation = TableOperation.Retrieve <CustomerEntity>(firstName + entryId, lastName + entryId);
result = rtable.Execute(operation);
Assert.IsTrue(result != null && result.HttpStatusCode == (int)HttpStatusCode.OK && (CustomerEntity)result.Result != null, "Retrieve customer failed");
customer = (CustomerEntity)result.Result;
customer.Email = "*****@*****.**";
operation = TableOperation.Replace(customer);
result = rtable.Execute(operation);
Assert.IsTrue(result != null && result.HttpStatusCode == (int)HttpStatusCode.NoContent, "Update customer failed");
/*
* Simulate a stale client => Set config viewId back to 5
*/
SetConfigViewIdAndIgnoreHigherViewIdRowsFlag(staleViewId, false);
Assert.AreEqual(staleViewId, this.configurationService.GetTableView(this.repTable.TableName).ViewId, "View should be 5!!!");
try
{
// Check Retrieve of row #5 throws stale view as expected
operation = TableOperation.Retrieve <CustomerEntity>(firstName + entryId, lastName + entryId);
rtable.Execute(operation);
Assert.Fail("Retrieve() is expected to get an RTableStaleViewException but did not get it.");
}
catch (ReplicatedTableStaleViewException ex)
{
Assert.IsTrue(ex.ErrorCode == ReplicatedTableViewErrorCodes.ViewIdSmallerThanEntryViewId);
Assert.IsTrue(ex.Message.Contains(string.Format("current _rtable_ViewId {0} is smaller than _rtable_ViewId of existing row {1}", staleViewId, newViewId)), "Got unexpected exception message");
}
// Enable throwing on stale view detection
rtable.ThrowOnStaleViewInLinqQueryFlag = true;
/*
* stale client using LINQ: CreateReplicatedQuery
*/
try
{
foreach (var entry in rtable.CreateReplicatedQuery <CustomerEntity>().AsEnumerable())
{
int id = int.Parse(entry.PartitionKey.Replace(firstName, ""));
Assert.IsTrue(id != entryId, "we should throw on entry #5");
Assert.AreEqual(entry._rtable_ViewId, staleViewId, "CreateReplicatedQuery: entry viewId should be '5'");
}
}
catch (ReplicatedTableStaleViewException ex)
{
Assert.IsTrue(ex.ErrorCode == ReplicatedTableViewErrorCodes.ViewIdSmallerThanEntryViewId);
Assert.IsTrue(ex.Message.Contains(string.Format("current _rtable_ViewId {0} is smaller than _rtable_ViewId of existing row {1}", staleViewId, newViewId)), "Got unexpected exception message");
}
/*
* stale client using LINQ: ExecuteQuery
*/
try
{
foreach (var entry in rtable.ExecuteQuery <CustomerEntity>(new TableQuery <CustomerEntity>()))
{
int id = int.Parse(entry.PartitionKey.Replace(firstName, ""));
Assert.IsTrue(id != entryId, "we should throw on entry #5");
Assert.AreEqual(entry._rtable_ViewId, staleViewId, "ExecuteQuery: entry viewId should be '5'");
}
}
catch (ReplicatedTableStaleViewException ex)
{
Assert.IsTrue(ex.ErrorCode == ReplicatedTableViewErrorCodes.ViewIdSmallerThanEntryViewId);
Assert.IsTrue(ex.Message.Contains(string.Format("current _rtable_ViewId {0} is smaller than _rtable_ViewId of existing row {1}", staleViewId, newViewId)), "Got unexpected exception message");
}
}
public RTableAdapter(ReplicatedTable rtable)
{
this.rtable = rtable;
}
public void RepairMidAnUpdate()
{
this.rtableWrapper = RTableWrapperForSampleRTableEntity.GetRTableWrapper(this.repTable);
// Not-Stable chain
// Reconfigure RTable so Head is WriteOnly.
View view = this.configurationService.GetTableView(this.repTable.TableName);
ReplicatedTableConfiguration config;
ReplicatedTableQuorumReadResult readStatus = this.configurationService.RetrieveConfiguration(out config);
Assert.IsTrue(readStatus.Code == ReplicatedTableQuorumReadCode.Success);
// Set Head as WriteOnly mode
ReplicatedTableConfigurationStore viewConfg = config.GetView(view.Name);
viewConfg.ReplicaChain[0].Status = ReplicaStatus.WriteOnly;
config.SetView(view.Name, viewConfg);
// Upload RTable config back
this.configurationService.UpdateConfiguration(config);
// Sanity: Replicated mode and chain Not-Stable
view = this.configurationService.GetTableView(this.repTable.TableName);
Assert.IsTrue(view != null && view.Chain.Count > 1, "Two replicas should be used.");
Assert.IsFalse(view.IsStable);
// Insert one entry
Console.WriteLine("Inserting entry ...");
string entityPartitionKey = "jobType-RepairMidAnUpdate-Replace";
string entityRowKey = "jobId-RepairMidAnUpdate-Replace";
var entry = new SampleRTableEntity(entityPartitionKey, entityRowKey, "message");
this.rtableWrapper.InsertRow(entry);
// 1 - Launch a RepairRow task in wait mode ...
bool triggerRepair = false;
bool repaireDone = false;
bool headAfterRepairWasLocked = false;
TableResult repairResult = null;
Task.Run(() =>
{
ReplicatedTable repairTable = new ReplicatedTable(this.repTable.TableName, this.configurationService);
while (!triggerRepair)
{
Thread.Sleep(5);
}
Console.WriteLine("RepairRow started ...");
repairResult = repairTable.RepairRow(entry.PartitionKey, entry.RowKey, null);
Console.WriteLine("RepairRow completed with HttpStatus={0}", repairResult == null ? "NULL" : repairResult.HttpStatusCode.ToString());
// Check the entry at the Head is still locked i.e. RepairRow was NOP
headAfterRepairWasLocked = HeadIsLocked(entry);
Console.WriteLine("Signal the commit to Head job");
repaireDone = true;
});
// 2 - Configure Mangler ...
string accountNameToTamper = this.rtableTestConfiguration.StorageInformation.AccountNames[0];
Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper);
ProxyBehavior[] behaviors = new[]
{
TamperBehaviors.TamperAllRequestsIf(
(session =>
{
Console.WriteLine("Delaying commit to the Head ... => signal RepairRow job");
// Let RepairRow task go through
triggerRepair = true;
int iter = 0;
while (!repaireDone)
{
// TODO: break the loop after couple of iteration ...
Thread.Sleep(100);
Console.WriteLine("Waiting on RepairRow to finish ({0}) ...", ++iter);
}
Console.WriteLine("Request a commit to the head");
}),
(session =>
{
// Commit on head i.e. a PUT with RowLock == false
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.HTTPMethodIs("PUT") &&
session.GetRequestBodyAsString().Contains("\"_rtable_RowLock\":false"))
{
return(true);
}
return(false);
}))
};
using (new HttpMangler(false, behaviors))
{
Console.WriteLine("Updating entry ...");
entry = this.rtableWrapper.FindRow(entry.PartitionKey, entry.RowKey);
entry.Message = "updated message";
this.rtableWrapper.ReplaceRow(entry);
}
Assert.IsTrue(triggerRepair);
Assert.IsTrue(repairResult != null && repairResult.HttpStatusCode == (int)HttpStatusCode.OK, "Repair failed.");
Assert.IsTrue(repaireDone);
Assert.IsTrue(headAfterRepairWasLocked);
Console.WriteLine("DONE. Test passed.");
}
private ReplicatedTableRepairResult RepairTable(string tableName, string storageAccountName, ReplicatedTableConfiguration configuration)
{
string viewName = "";
try
{
ReplicatedTableConfiguredTable tableConfig;
if (!configuration.IsConfiguredTable(tableName, out tableConfig))
{
return new ReplicatedTableRepairResult(ReplicatedTableRepairCode.NotConfiguredTable, tableName);
}
viewName = tableConfig.ViewName;
List<ReplicaInfo> list = configuration.GetView(viewName).ReplicaChain;
if (!list.Any() ||
list[0].StorageAccountName != storageAccountName)
{
return new ReplicatedTableRepairResult(ReplicatedTableRepairCode.NotImpactedTable, tableName, viewName, storageAccountName);
}
ReplicaInfo head = list[0];
if (head.Status != ReplicaStatus.WriteOnly)
{
return new ReplicatedTableRepairResult(ReplicatedTableRepairCode.StableTable, tableName, viewName, storageAccountName);
}
int viewIdToRecoverFrom = (int)head.ViewWhenTurnedOff;
ReplicatedTableLogger.LogInformational("RepairTable={0}, View={1}, StorageAccountName={2}, from viewId={3} ...",
tableName,
viewName,
storageAccountName,
viewIdToRecoverFrom);
// Repairing ...
ReconfigurationStatus status = new ReplicatedTable(tableName, this).RepairTable(viewIdToRecoverFrom, null);
ReplicatedTableLogger.LogInformational("RepairTable={0}, View={1}, StorageAccountName={2}, from viewId={3} => Status={4}",
tableName,
viewName,
storageAccountName,
viewIdToRecoverFrom,
status);
if (status == ReconfigurationStatus.SUCCESS)
{
return new ReplicatedTableRepairResult(ReplicatedTableRepairCode.Success, tableName, viewName, storageAccountName);
}
// Failure!
return new ReplicatedTableRepairResult(ReplicatedTableRepairCode.Error, tableName, viewName, storageAccountName)
{
Status = status,
};
}
catch (Exception ex)
{
return new ReplicatedTableRepairResult(ReplicatedTableRepairCode.Error, tableName, viewName, storageAccountName)
{
Status = ReconfigurationStatus.FAILURE,
Message = ex.ToString(),
};
}
}
public void LinqQueriesDontReturnEntriesWithTombstone()
{
var rtable = new ReplicatedTable(this.repTable.TableName, this.configurationService);
string firstName = "FirstName";
string lastName = "LastName";
/*
* 1 - insert entries
*/
for (int i = 0; i < 10; i++)
{
var customer = new CustomerEntity(firstName + i, lastName + i);
TableOperation operation = TableOperation.Insert(customer);
rtable.Execute(operation);
}
// Identify the Tail account
View view = this.configurationWrapper.GetWriteView();
Assert.IsTrue(view.Chain.Count > 1, "expects at least one replica!");
string accountNameToTamper = view.Chain.Last().Item1.StorageAccountName;
Console.WriteLine("RunHttpManglerBehaviorHelper(): accountNameToTamper={0}", accountNameToTamper);
// We will fail requests to delete a row in Tail
ProxyBehavior[] behaviors =
{
TamperBehaviors.TamperAllRequestsIf(
(session =>
{
session.oRequest.FailSession((int)HttpStatusCode.ServiceUnavailable, "ServerBusy", "");
}),
(session =>
{
// Commit to Tail i.e. physically delete the row
if (session.hostname.Contains(accountNameToTamper + ".") &&
session.HTTPMethodIs("DELETE"))
{
return(true);
}
return(false);
})),
};
using (new HttpMangler(false, behaviors))
{
/*
* 2 - delete entries #2 and #4
*/
foreach (var i in new int[] { 2, 4 })
{
TableOperation operation = TableOperation.Retrieve <CustomerEntity>(firstName + i, lastName + i);
TableResult retrievedResult = rtable.Execute(operation);
var customer = (CustomerEntity)retrievedResult.Result;
TableOperation deleteOperation = TableOperation.Delete(customer);
TableResult deleteResult = rtable.Execute(deleteOperation);
Assert.IsNotNull(deleteResult, "deleteResult = null");
Assert.AreEqual((int)HttpStatusCode.ServiceUnavailable, deleteResult.HttpStatusCode, "deleteResult.HttpStatusCode mismatch");
}
}
// Verify, Retrieve doesn't return entries #2 and #4
int deleteId = 2;
var result = rtable.Execute(TableOperation.Retrieve <CustomerEntity>(firstName + deleteId, lastName + deleteId));
Assert.IsNotNull(result, "result = null");
Assert.AreEqual((int)HttpStatusCode.NotFound, result.HttpStatusCode, "result.HttpStatusCode mismatch");
deleteId = 4;
result = rtable.Execute(TableOperation.Retrieve <CustomerEntity>(firstName + deleteId, lastName + deleteId));
Assert.IsNotNull(result, "result = null");
Assert.AreEqual((int)HttpStatusCode.NotFound, result.HttpStatusCode, "result.HttpStatusCode mismatch");
/*
* 3 - CreateQuery doesn't return entries #2 and #4
*/
foreach (var customer in rtable.CreateReplicatedQuery <CustomerEntity>().AsEnumerable())
{
int id = int.Parse(customer.PartitionKey.Replace(firstName, ""));
Assert.AreNotEqual(id, 2, "entry #2 should have been deleted");
Assert.AreNotEqual(id, 4, "entry #4 should have been deleted");
}
/*
* 4 - ExecuteQuery doesn't return entries #2 and #4
*/
foreach (var customer in rtable.ExecuteQuery <CustomerEntity>(new TableQuery <CustomerEntity>()))
{
int id = int.Parse(customer.PartitionKey.Replace(firstName, ""));
Assert.AreNotEqual(id, 2, "entry #2 should have been deleted");
Assert.AreNotEqual(id, 4, "entry #4 should have been deleted");
}
}
