private BackupParserManager SetupBackupParserManagerAndStartParsing()
{
var backupParser = new BackupParser(this.backupChainInfo.BackupChainPath, this.backupChainInfo.CodePackagePath);
var stateManager = backupParser.StateManager;
foreach (var serializer in this.backupChainInfo.Serializers)
{
var stateTypeName = serializer.StateFullyQualifiedTypeName;
var serializerTypeName = serializer.SerializerFullyQualifiedTypeName;
var stateType = Type.GetType(stateTypeName, true);
var serializerType = Type.GetType(serializerTypeName, true);
var serializerObject = new object();
if (serializerType.Name != "DataContractSerializer")
{
serializerObject = Activator.CreateInstance(serializerType);
stateManager.GetType()
.GetMethod("TryAddStateSerializer", BindingFlags.Instance | BindingFlags.Public)
.MakeGenericMethod(stateType)
.Invoke(stateManager, new object[] { serializerObject });
}
}
var backupParserManager = new BackupParserManager(backupParser);
backupParserManager.StartParsing();
return(backupParserManager);
}
public async Task BackupParser_CollectChangesInTransactionAppliedEvents()
{
using (var backupParser = new BackupParser(BackupFolderPath, ""))
{
int totalDictionaryAdds = 0;
backupParser.TransactionApplied += (sender, args) =>
{
foreach (var reliableStateChange in args.Changes)
{
if (reliableStateChange.Name == DictionaryName)
{
foreach (var change in reliableStateChange.Changes)
{
var addChange = change as NotifyDictionaryItemAddedEventArgs <long, long>;
if (null != addChange)
{
totalDictionaryAdds++;
}
}
}
}
};
await backupParser.ParseAsync(CancellationToken.None);
Assert.AreEqual(totalDictionaryAdds, TotalDictionaryInserts, "Not able to collect all Dictionary change events");
}
}
public async Task BackupParser_TakeBackupAfterParseFinish()
{
long initialCountKeysInDict = 0;
// read a back and then write more keys and take a full back.
using (var backupParser = new BackupParser(BackupFolderPath, ""))
{
await backupParser.ParseAsync(CancellationToken.None);
var stateManager = backupParser.StateManager;
var result = await stateManager.TryGetAsync <IReliableDictionary <long, long> >(DictionaryName);
Assert.IsTrue(result.HasValue, "Not able to find IReliableDictionary<long, long> dictionary");
var dictionary = result.Value;
using (var tx = stateManager.CreateTransaction())
{
initialCountKeysInDict = await dictionary.GetCountAsync(tx);
await tx.CommitAsync();
}
Assert.IsTrue(initialCountKeysInDict > 0, "No data seen in dictionary");
// Take backup
await backupParser.BackupAsync(Data.BackupOption.Full, TimeSpan.FromMinutes(30), CancellationToken.None, this.OnFullBackupCallback);
}
// Verify full backup
using (var backupParser = new BackupParser(this.FullBackupFolderPath, ""))
{
await backupParser.ParseAsync(CancellationToken.None);
var stateManager = backupParser.StateManager;
var result = await stateManager.TryGetAsync <IReliableDictionary <long, long> >(DictionaryName);
Assert.IsTrue(result.HasValue, "Not able to find IReliableDictionary<long, long> dictionary");
var dictionary = result.Value;
using (var tx = stateManager.CreateTransaction())
{
var countKeysInDict = await dictionary.GetCountAsync(tx);
Assert.AreEqual(initialCountKeysInDict, countKeysInDict, "Fullbackup does not have right number of keys.");
for (long i = 0; i < initialCountKeysInDict; ++i)
{
var valueResult = await dictionary.TryGetValueAsync(tx, i);
Assert.IsTrue(valueResult.HasValue, "Value not present in dictionary");
Assert.AreEqual(i, valueResult.Value, "Not able to get expected value");
}
await tx.CommitAsync();
}
}
}
public BackupParserManager(BackupParser backupParser)
{
this.BackupParser = backupParser;
this.BackupParser.TransactionApplied += (sender, args) =>
{
transactionsList = args;
Console.WriteLine("Transaction Id {0}, CommitSequenceNumber {1}, Changes {2}", args.TransactionId, args.CommitSequenceNumber, args.Changes.Count());
};
}
public async Task BackupParser_StateManagerFailsToWriteDuringParse()
{
using (var backupParser = new BackupParser(BackupFolderPath, ""))
{
long countValuesInDictionary = 0;
backupParser.TransactionApplied += async(sender, args) =>
{
var stateManager = backupParser.StateManager;
var result = await stateManager.TryGetAsync <IReliableDictionary <long, long> >(DictionaryName);
Assert.IsTrue(result.HasValue, "Not able to find IReliableDictionary<long, long> dictionary");
var dictionary = result.Value;
try
{
using (var tx = stateManager.CreateTransaction())
{
countValuesInDictionary = await dictionary.GetCountAsync(tx);
for (int i = 0; i < countValuesInDictionary; ++i)
{
var valueResult = await dictionary.TryGetValueAsync(tx, i);
Assert.IsTrue(valueResult.HasValue, "Value not present in dictionary");
Assert.AreEqual(i, valueResult.Value, "Not able to get expected value");
await dictionary.AddOrUpdateAsync(tx, i, i + 1, (k, v) => v + 1);
}
await tx.CommitAsync();
}
if (countValuesInDictionary > 0)
{
// If we called AddOrUpdateAsync, commit should fail.
Assert.Fail("This transaction should not have committed.");
}
}
catch (InvalidOperationException)
{ }
catch (FabricNotPrimaryException)
{ }
};
await backupParser.ParseAsync(CancellationToken.None);
Assert.IsTrue(countValuesInDictionary > 0, "No data read in dictionary.");
}
}
public BackupParserManager(BackupParser backupParser)
{
this.BackupParser = backupParser;
// todo : take NumMaxTransactionsInMemory from config.
this.transactionsQueue = new BlockingCollection <NotifyTransactionAppliedEventArgs>(NumMaxTransactionsInMemory);
this.BackupParser.TransactionApplied += (sender, args) =>
{
if (transactionsQueue.Count < NumMaxTransactionsInMemory)
{
transactionsQueue.Add(args);
Console.WriteLine("{0} : TransactionId {1} , CommitSequenceNumber {2}, Changes {3}", transactionsQueue.Count, args.TransactionId, args.CommitSequenceNumber, args.Changes.Count());
}
};
}
public async Task BackupParser_FireTransactionAppliedEvents()
{
using (var backupParser = new BackupParser(BackupFolderPath, ""))
{
bool transactionSeen = false;
backupParser.TransactionApplied += (sender, args) =>
{
transactionSeen = true;
};
await backupParser.ParseAsync(CancellationToken.None).ConfigureAwait(false);
Assert.IsTrue(transactionSeen, "Not able to parse any transaction from backup chain.");
}
}
public async Task BackupParser_EachTransactionHasRightChangesEvenWithBlockingTransactionAppliedEvents()
{
using (var backupParser = new BackupParser(BackupFolderPath, ""))
{
int totalDictionaryAdds = 0;
int totalTransactionsSeenForDictionary = 0;
int lastKeyAdded = -1;
var rand = new Random();
backupParser.TransactionApplied += (sender, args) =>
{
foreach (var reliableStateChange in args.Changes)
{
if (reliableStateChange.Name == DictionaryName)
{
foreach (var change in reliableStateChange.Changes)
{
var addChange = change as NotifyDictionaryItemAddedEventArgs <long, long>;
if (null != addChange)
{
totalDictionaryAdds++;
Assert.IsTrue(addChange.Key > lastKeyAdded,
string.Format("Found wrong changes within a transaction : False : key added {0} > lastKeyAdded {1}",
totalTransactionsSeenForDictionary, lastKeyAdded));
}
}
Assert.AreEqual(NumOperationsPerTransaction, reliableStateChange.Changes.Count(), "Wrong number of changes within a transaction");
totalTransactionsSeenForDictionary += 1;
}
}
// sleep for 10-200 ms.
Thread.Sleep(rand.Next(10, 200));
};
await backupParser.ParseAsync(CancellationToken.None);
Assert.AreEqual(TotalDictionaryInserts, totalDictionaryAdds, "Not able to collect all Dictionary change events");
Assert.AreEqual(TotalDictionaryTransactions, totalTransactionsSeenForDictionary, "Wrong number of transactions");
}
}
public async Task BackupParser_ComplexTypesWithSerializers()
{
var complexDataBackupFolderPath = Path.Combine(ClassTestPath, @"..\UserFullBackup");
// from this test run's bin\Debug\netstandard2.0\<testname> dir to parent of bin
var codePackagePath = Path.Combine(ClassTestPath, @"..\..\..\..\UserType\bin\");
using (var backupParser = new BackupParser(complexDataBackupFolderPath, codePackagePath))
{
backupParser.StateManager.TryAddStateSerializer <User>(new UserSerializer());
int totalUsers = 0;
backupParser.TransactionApplied += (sender, args) =>
{
foreach (var reliableStateChange in args.Changes)
{
if (reliableStateChange.Name == DictionaryName)
{
foreach (var change in reliableStateChange.Changes)
{
// we can't link against User project as that will make it easy to load the dll.
bool isAddChange = GenericUtils.IsSubClassOfGeneric(change.GetType(), typeof(NotifyDictionaryItemAddedEventArgs <,>));
if (isAddChange)
{
// make sure that this is Add event with User type.
if (change.GetType().ToString().Contains("UserType.User"))
{
totalUsers++;
}
}
}
}
}
};
await backupParser.ParseAsync(CancellationToken.None);
Assert.IsTrue(totalUsers > 0, "Could not parse any user from backup");
}
}
public async Task BackupParser_StateManagerAbleToReadQueuesInTransactionApplied()
{
using (var backupParser = new BackupParser(BackupFolderPath, ""))
{
long countValuesInQueue = 0;
backupParser.TransactionApplied += async(sender, args) =>
{
var stateManager = backupParser.StateManager;
// verify ReliableQueue
var result = await stateManager.TryGetAsync <IReliableQueue <long> >(QueueName);
Assert.IsTrue(result.HasValue, "Not able to find IReliableQueue<long> queue");
var queue = result.Value;
using (var tx = stateManager.CreateTransaction())
{
countValuesInQueue = await queue.GetCountAsync(tx);
if (countValuesInQueue > 0)
{
var valueResult = await queue.TryPeekAsync(tx);
Assert.IsTrue(valueResult.HasValue, "Value not present in queue");
Assert.AreEqual(0, valueResult.Value, "Queue head should be always first element 0");
}
await tx.CommitAsync();
}
// Don't verify ConcurrentQueue
// There is no read only api in ConcurrentQueue : only enque/deque.
// Reading Count on queue does not work because it checks for TransactionalReplicator's IsReadable which is not true.
};
await backupParser.ParseAsync(CancellationToken.None);
Assert.AreEqual(TotalQueueTransactions * NumOperationsPerTransaction, countValuesInQueue, "No data read in queue.");
}
}
public async Task BackupParser_VerifyQueuesTransaction()
{
using (var backupParser = new BackupParser(BackupFolderPath, ""))
{
int totalTransactionsSeenForQueue = 0, totalTransactionsSeenForConcQueue = 0, totalTransactionsSeen = 0;
var rand = new Random();
backupParser.TransactionApplied += (sender, args) =>
{
foreach (var reliableStateChange in args.Changes)
{
if (reliableStateChange.Name == QueueName)
{
// Uncomment once we fire events for queues.
// Assert.AreEqual(NumOperationsPerTransaction, reliableStateChange.Changes.Count(), "Wrong number of changes within a transaction.");
totalTransactionsSeenForQueue += 1;
}
else if (reliableStateChange.Name == ConcurrentQueueName)
{
// Assert.AreEqual(NumOperationsPerTransaction, reliableStateChange.Changes.Count(), "Wrong number of changes within a transaction.");
totalTransactionsSeenForConcQueue += 1;
}
}
// sleep for 10 - 200 ms.
Thread.Sleep(rand.Next(10, 200));
totalTransactionsSeen += 1;
};
await backupParser.ParseAsync(CancellationToken.None);
// Uncomment once we fire events for queues.
// Assert.AreEqual(TotalQueueTransactions, totalTransactionsSeenForQueue, "Wrong number of transactions for queue.");
// Assert.AreEqual(TotalQueueTransactions, totalTransactionsSeenForQueue, "Wrong number of transactions for concurrent queue.");
Assert.AreEqual(TotalTransactions, totalTransactionsSeen, "Wrong number of total Transactions");
}
}
public async Task BackupParser_StateManagerAbleToReadInTransactionApplied()
{
using (var backupParser = new BackupParser(BackupFolderPath, ""))
{
long countValuesInDictionary = 0;
backupParser.TransactionApplied += async(sender, args) =>
{
var stateManager = backupParser.StateManager;
var result = await stateManager.TryGetAsync <IReliableDictionary <long, long> >(DictionaryName);
Assert.IsTrue(result.HasValue, "Not able to find IReliableDictionary<long, long> dictionary");
var dictionary = result.Value;
using (var tx = stateManager.CreateTransaction())
{
var count = await dictionary.GetCountAsync(tx);
for (int i = 0; i < count; ++i)
{
var valueResult = await dictionary.TryGetValueAsync(tx, i);
Assert.IsTrue(valueResult.HasValue, "Value not present in dictionary");
Assert.AreEqual(i, valueResult.Value, "Not able to get expected value");
}
countValuesInDictionary = count;
await tx.CommitAsync();
}
};
await backupParser.ParseAsync(CancellationToken.None);
Assert.IsTrue(countValuesInDictionary > 0, "No data read in dictionary.");
}
}
public async Task BackupParser_TakeIncrementatlBackupAfterWritingToRCs()
{
long initialCountKeysInDict = 0;
const long numKeysToAdd = 100, numIncrementalKeysToAdd = 20;
// read a back and then write more keys and take a full back.
using (var backupParser = new BackupParser(BackupFolderPath, ""))
{
await backupParser.ParseAsync(CancellationToken.None);
var stateManager = backupParser.StateManager;
var result = await stateManager.TryGetAsync <IReliableDictionary <long, long> >(DictionaryName);
Assert.IsTrue(result.HasValue, "Not able to find IReliableDictionary<long, long> dictionary");
var dictionary = result.Value;
using (var tx = stateManager.CreateTransaction())
{
initialCountKeysInDict = await dictionary.GetCountAsync(tx);
var totalKeys = initialCountKeysInDict + numKeysToAdd;
for (long i = initialCountKeysInDict; i < totalKeys; ++i)
{
await dictionary.AddOrUpdateAsync(tx, i, i + 1, (k, v) => k + 1);
}
await tx.CommitAsync();
}
Assert.IsTrue(initialCountKeysInDict > 0, "No data seen in dictionary");
// Take backup
await backupParser.BackupAsync(Data.BackupOption.Full, TimeSpan.FromMinutes(30), CancellationToken.None, this.OnFullBackupCallback);
using (var tx = stateManager.CreateTransaction())
{
var totalKeys = initialCountKeysInDict + numKeysToAdd + numIncrementalKeysToAdd;
for (long i = initialCountKeysInDict + numKeysToAdd; i < totalKeys; ++i)
{
await dictionary.AddOrUpdateAsync(tx, i, i + 2, (k, v) => k + 2);
}
await tx.CommitAsync();
}
// Take incremental backup
await backupParser.BackupAsync(BackupOption.Incremental, TimeSpan.FromMinutes(3), CancellationToken.None, this.OnIncrementalBackupCallback);
}
// Verify full+incremental backups
using (var backupParser = new BackupParser(this.FullAndIncrementalBackupFolderPath, ""))
{
await backupParser.ParseAsync(CancellationToken.None);
var stateManager = backupParser.StateManager;
var result = await stateManager.TryGetAsync <IReliableDictionary <long, long> >(DictionaryName);
Assert.IsTrue(result.HasValue, "Not able to find IReliableDictionary<long, long> dictionary");
var dictionary = result.Value;
using (var tx = stateManager.CreateTransaction())
{
var countKeysInDict = await dictionary.GetCountAsync(tx);
Assert.AreEqual(initialCountKeysInDict + numKeysToAdd + numIncrementalKeysToAdd, countKeysInDict, "Fullbackup does not have right number of keys.");
for (long i = 0; i < initialCountKeysInDict; ++i)
{
var valueResult = await dictionary.TryGetValueAsync(tx, i);
Assert.IsTrue(valueResult.HasValue, "Value not present in dictionary");
Assert.AreEqual(i, valueResult.Value, "Not able to get expected value");
}
for (long i = initialCountKeysInDict; i < initialCountKeysInDict + numKeysToAdd; ++i)
{
var valueResult = await dictionary.TryGetValueAsync(tx, i);
Assert.IsTrue(valueResult.HasValue, "Value not present in dictionary");
Assert.AreEqual(i + 1, valueResult.Value, "Not able to get expected value");
}
for (long i = initialCountKeysInDict + numKeysToAdd; i < countKeysInDict; ++i)
{
var valueResult = await dictionary.TryGetValueAsync(tx, i);
Assert.IsTrue(valueResult.HasValue, "Value not present in dictionary");
Assert.AreEqual(i + 2, valueResult.Value, "Not able to get expected value");
}
await tx.CommitAsync();
}
}
}
public async Task BackupParser_StateManagerAbleToWriteAfterParseFinish()
{
using (var backupParser = new BackupParser(BackupFolderPath, ""))
{
bool transactionSeen = false;
var stateManager = backupParser.StateManager;
backupParser.TransactionApplied += (sender, args) =>
{
transactionSeen = true;
};
await backupParser.ParseAsync(CancellationToken.None);
Assert.IsTrue(transactionSeen, "No transaction seen.");
// Make writes..
{
var result = await stateManager.TryGetAsync <IReliableDictionary <long, long> >(DictionaryName);
Assert.IsTrue(result.HasValue, "Not able to find IReliableDictionary<long, long> dictionary");
long countKeysInDict = 0;
var dictionary = result.Value;
using (var tx = stateManager.CreateTransaction())
{
countKeysInDict = await dictionary.GetCountAsync(tx);
for (int i = 0; i < countKeysInDict; ++i)
{
var valueResult = await dictionary.TryGetValueAsync(tx, i);
Assert.IsTrue(valueResult.HasValue, "Value not present in dictionary");
Assert.AreEqual(i, valueResult.Value, "Not able to get expected value");
await dictionary.AddOrUpdateAsync(tx, i, i + 1, (k, v) => v + 1);
}
await tx.CommitAsync();
}
Assert.IsTrue(countKeysInDict > 0, "No data seen in dictionary");
}
// Verify writes
{
var result = await stateManager.TryGetAsync <IReliableDictionary <long, long> >(DictionaryName);
Assert.IsTrue(result.HasValue, "Not able to find IReliableDictionary<long, long> dictionary");
long countKeysInDict = 0;
var dictionary = result.Value;
using (var tx = stateManager.CreateTransaction())
{
countKeysInDict = await dictionary.GetCountAsync(tx);
for (int i = 0; i < countKeysInDict; ++i)
{
var valueResult = await dictionary.TryGetValueAsync(tx, i);
Assert.IsTrue(valueResult.HasValue, "Value not present in dictionary");
Assert.AreEqual(i + 1, valueResult.Value, "Not able to get expected value");
}
await tx.CommitAsync();
}
Assert.IsTrue(countKeysInDict > 0, "No data seen in dictionary");
}
}
}
