public void Indexes_Build_Time() { const int buildTimes = 3; var stopWatch = new Stopwatch(); var tokenizer = new DefaultTokenizer(); var lines = TestDataGenerator.GetRandomLines(50000, 20000); stopWatch.Start(); for (var i = 0; i < buildTimes; i++) { BuildIndex(new InvertedHashIndex(tokenizer), lines); } var hashIndexBuildTime = stopWatch.Elapsed; stopWatch.Restart(); for (var i = 0; i < buildTimes; i++) { BuildIndex(new InvertedIndex(tokenizer), lines); } var suffixIndeBuildTime = stopWatch.Elapsed; Console.WriteLine($"HashIndex build time: {hashIndexBuildTime}"); Console.WriteLine($"SuffixIndex build time: {suffixIndeBuildTime}"); }
public void Search_Result_Is_Same_For_InvertedIndex_And_SimpleSearching() { var tokenizer = new DefaultTokenizer(); var ticks = Environment.TickCount; Console.WriteLine($"Test ticks: {ticks}"); var lines = TestDataGenerator.GetRandomLines(ticks); var invertedIndex = new InvertedIndex(tokenizer); var phrase = TestDataGenerator.GetSearchPhrase(ticks); BuildIndex(invertedIndex, lines); var inmemoryResult = InmemorySimpleSearch.Find(lines, phrase); invertedIndex.Find(phrase) .Select(x => new { x.RowNumber, x.ColNumber }) .Should() .BeEquivalentTo(inmemoryResult.Select(x => new { x.RowNumber, x.ColNumber })); }
public void InvertedIndex_Should_Be_Faster_Than_Simple_Searching() { const int phrasesCount = 50; var phrases = new string[phrasesCount]; var tickCount = Environment.TickCount; Console.WriteLine($"TickCount: {tickCount}"); for (var i = 0; i < phrasesCount; i++) { phrases[i] = TestDataGenerator.GetSearchPhrase(tickCount + i); } var tokenizer = new DefaultTokenizer(); var stopWatch = new Stopwatch(); var lines = TestDataGenerator.GetRandomLines(tickCount, 50000); var invertedIndex = new InvertedIndex(tokenizer); BuildIndex(invertedIndex, lines); stopWatch.Start(); for (var i = 0; i < phrasesCount; i++) { var elapsedBefore = stopWatch.Elapsed; invertedIndex.Find(phrases[i]); var elapsed = stopWatch.Elapsed - elapsedBefore; Console.WriteLine($"Elapsed for phrase: {phrases[i]} {elapsed}"); } var indexSearchingTime = stopWatch.Elapsed; stopWatch.Restart(); for (var i = 0; i < phrasesCount; i++) { InmemorySimpleSearch.Find(lines, phrases[i]); } var simpleSearchingTime = stopWatch.Elapsed; Console.WriteLine($"InvertedIndex searching time: {indexSearchingTime}"); Console.WriteLine($"Simple searching time: {simpleSearchingTime}"); indexSearchingTime.Should().BeLessThan(simpleSearchingTime); }
public void OneTimeSetup() { const int fromTicks = 200; var firstFileLines = TestDataGenerator.GetRandomLines(fromTicks, 30000); var secondFileLines = TestDataGenerator.GetRandomLines(fromTicks, 30000); var includedFileLines = TestDataGenerator.GetRandomLines(fromTicks, 30000); var deepIncludedFileLines = TestDataGenerator.GetRandomLines(fromTicks, 30000); EnsureErasedFolder(this.PathToFiles); EnsureErasedFolder(this.IncludedDirPath); EnsureErasedFolder(this.DeepIncludedDirPath); File.WriteAllLines(this.FirstFilePath, firstFileLines); File.AppendAllLines(this.FirstFilePath, new[] { FirstFileLastLine }); File.WriteAllLines(this.SecondFilePath, secondFileLines); File.AppendAllLines(this.SecondFilePath, new[] { SecondFileLastLine }); File.WriteAllLines(this.IncludedFilePath, includedFileLines); File.AppendAllLines(this.IncludedFilePath, new[] { IncludedFileLastLine }); File.WriteAllLines(this.DeepIncludedFilePath, deepIncludedFileLines); File.AppendAllLines(this.DeepIncludedFilePath, new[] { DeepIncludedFileLastLine }); }
public void Add_And_Read_Simultaneously() { var exceptionsCount = 0; var index = this.GetNewIndex(); var ticks = 100; Action addAction = () => { try { for (var i = 0; i < 5; i++) { var lines = TestDataGenerator.GetRandomLines(ticks + i, 10000); index.Add(lines, "some doc" + i); } } catch (Exception e) { Console.WriteLine(e); Interlocked.Increment(ref exceptionsCount); throw; } }; var stopWatch = Stopwatch.StartNew(); var addTasks = new Task[2]; for (var i = 0; i < addTasks.Length; i++) { addTasks[i] = new Task(addAction); addTasks[i].Start(); } Action readAction = () => { try { for (var i = 0; i < 1000; i++) { var phrase = TestDataGenerator.GetSearchPhrase(ticks + i); index.Find(phrase); } } catch (Exception e) { Console.WriteLine(e); Interlocked.Increment(ref exceptionsCount); throw; } }; var readTasks = new Task[2]; for (var i = 0; i < readTasks.Length; i++) { readTasks[i] = new Task(readAction); readTasks[i].Start(); } Task.WaitAll(addTasks); Console.WriteLine($"Build elapsed: {stopWatch.Elapsed}"); Task.WaitAll(readTasks); Console.WriteLine($"Total elapsed: {stopWatch.Elapsed}"); exceptionsCount.Should().Be(0); }