private void WriteReferencesRecursively(Dictionary <string, int> pages, int iterationId) { iterationId++; if (iterationId >= maxIterations || pages.Count < 1) { return; } Parallel.ForEach(pages, (e) => { Thread.Sleep(500); var page = pageDownloadService.DownloadPage(e.Key); if (string.IsNullOrEmpty(page.Result)) { return; } var filePath = fileSystemService.WriteDataToFile(page.Result, folderPath); var references = wikiReferencesParsingService.ParseRefsFromFileToDictionary(filePath, iterationId); fileSystemService.DeleteFile(filePath); lock (dbLock) { referencesDbService.WriteRefsToDb(ref references); } WriteReferencesRecursively(references, iterationId); } ); }
public void ShouldFilterReferencesIfTheyAreInDb() { //Arrange string reference1 = "test1"; string reference2 = "test2"; int iterationId = 0; var test = new Dictionary <string, int>() { { reference1, iterationId }, { reference2, iterationId } }; referenceTable.ContainsById(reference1).Returns(true); referenceTable.ContainsById(reference2).Returns(false); //Act referencesDbService.WriteRefsToDb(ref test); //Assert Assert.IsTrue(test.ContainsKey(reference2) && test.Count == 1); }