public async Task PersistenceProvider_Sharded_9_WriteRead()
{
const string testName = nameof(PersistenceProvider_Sharded_9_WriteRead);
IStorageProvider store1 = new MockStorageProvider(2);
IStorageProvider store2 = new MockStorageProvider(2);
IStorageProvider store3 = new MockStorageProvider(2);
IStorageProvider store4 = new MockStorageProvider(2);
IStorageProvider store5 = new MockStorageProvider(2);
IStorageProvider store6 = new MockStorageProvider(2);
IStorageProvider store7 = new MockStorageProvider(2);
IStorageProvider store8 = new MockStorageProvider(2);
IStorageProvider store9 = new MockStorageProvider(2);
await storageProviderManager.AddAndInitProvider("Store1", store1);
await storageProviderManager.AddAndInitProvider("Store2", store2);
await storageProviderManager.AddAndInitProvider("Store3", store3);
await storageProviderManager.AddAndInitProvider("Store4", store4);
await storageProviderManager.AddAndInitProvider("Store5", store5);
await storageProviderManager.AddAndInitProvider("Store6", store6);
await storageProviderManager.AddAndInitProvider("Store7", store7);
await storageProviderManager.AddAndInitProvider("Store8", store8);
await storageProviderManager.AddAndInitProvider("Store9", store9);
ShardedStorageProvider composite = await ConfigureShardedStorageProvider(testName, storageProviderManager);
await Test_PersistenceProvider_WriteRead(testName, composite);
}
public void ShouldSaveTagsIntoDatabase()
{
string startingUrl = @"http://www.samplesite.com/pages";
string link = @"pages/page1.htm";
IStorageProvider storageProvider = new MockStorageProvider();
IOutputProvider outputProvider = new MockOutputProvider();
int recursionLevel = 1;
Uri uri = new Uri(startingUrl);
string completeLink = uri.Scheme + @":\\" + new Uri(startingUrl).Authority + @"/" + link;
if (!storageProvider.Contains(completeLink))
{
outputProvider.WriteLine(completeLink + " " + (recursionLevel + 1).ToString());
storageProvider.TryAdd(completeLink, recursionLevel + 1);
}
Assert.IsTrue(storageProvider.GetRecords().Count == 1);
Assert.IsTrue(storageProvider.GetRecords().ContainsKey(completeLink));
if (!storageProvider.Contains(completeLink))
{
storageProvider.TryAdd(completeLink, recursionLevel + 1);
}
Assert.IsTrue(storageProvider.GetRecords().Count == 1);
}
public async Task PersistenceProvider_Mock_WriteRead()
{
const string testName = nameof(PersistenceProvider_Mock_WriteRead);
IStorageProvider store = new MockStorageProvider();
var cfg = new ProviderConfiguration(providerCfgProps, null);
await store.Init(testName, storageProviderManager, cfg);
await Test_PersistenceProvider_WriteRead(testName, store);
}
public async Task PersistenceProvider_Sharded_WriteRead()
{
const string testName = nameof(PersistenceProvider_Sharded_WriteRead);
IStorageProvider store1 = new MockStorageProvider(2);
IStorageProvider store2 = new MockStorageProvider(2);
await storageProviderManager.AddAndInitProvider("Store1", store1);
await storageProviderManager.AddAndInitProvider("Store2", store2);
var composite = await ConfigureShardedStorageProvider(testName, storageProviderManager);
await Test_PersistenceProvider_WriteRead(testName, composite);
}
public void ShouldCallParsingForEachPageFromPreviousIteration()
{
IInputProvider inputProvider = new MockInputProvider();
IOutputProvider outputProvider = new MockOutputProvider();
IStorageProvider storageProvider = new MockStorageProvider();
IClientProvider clientProvider = new MockClientProvider();
IPageProvider pageProvider = new MockPageProvider();
ILinkProcessorProvider linkProcessorProvider = new MockLinkProcessorProvider(outputProvider, storageProvider);
IPageParserProvider pageParserProvider = new MockPageParcerProvider();
WebPageProcessor wpp = new WebPageProcessor(storageProvider, inputProvider, outputProvider, pageProvider, linkProcessorProvider, pageParserProvider, 0);
wpp.ProcessWebPage("Test", 0);
Assert.IsTrue(storageProvider.Contains("Test1processed"));
Assert.IsTrue(storageProvider.Contains("Test2processed"));
Assert.IsTrue(storageProvider.Contains("Test3processed"));
Assert.IsTrue(storageProvider.Contains("Test4processed"));
Assert.IsTrue(storageProvider.Contains("Test5processed"));
Assert.IsTrue(storageProvider.Contains("Test6processed"));
Assert.IsTrue(storageProvider.Contains("Test7processed"));
Assert.IsTrue(storageProvider.Contains("Test8processed"));
}
