//[TestCase(500)] //[TestCase(5000)] public void AsyncIndexer(int actionCount) { var world = CreateOneItem<PersistableItem>(2, "hello world", root); var universe = CreateOneItem<PersistableItem>(3, "hello universe", root); var generator = new SCG.General.MarkovNameGenerator(Words.Thousand, 3, 2); var words = Enumerable.Range(0, 1000).Select(i => generator.NextName).ToArray(); var items = new [] { root, world, universe }; var r = new Random(); for (int i = 0; i < actionCount; i++) { if (r.NextDouble() < .75) { var item = items[r.Next(0, items.Length)]; item.StringProperty = Enumerable.Range(0, 50).Select(index => words[r.Next(0, words.Length)]).Aggregate(new StringBuilder(), (sb, w) => sb.Append(w).Append(" ")).ToString(); asyncIndexer.Reindex(item.ID, false); } else asyncIndexer.Delete(items[r.Next(0, items.Length)].ID); Thread.Sleep(1); } int times = 0; while (asyncIndexer.GetCurrentStatus().QueueSize > 0) { Debug.WriteLine("Waiting " + times + " times on thread " + Thread.CurrentThread.ManagedThreadId); times++; Thread.Sleep(100); if (times > actionCount) throw new Exception("Waited for a long time for indexing to finish"); } }
//[TestCase(500)] //[TestCase(5000)] public void AsyncIndexer(int actionCount) { var world = CreateOneItem <PersistableItem>(2, "hello world", root); var universe = CreateOneItem <PersistableItem>(3, "hello universe", root); var generator = new SCG.General.MarkovNameGenerator(Words.Thousand, 3, 2); var words = Enumerable.Range(0, 1000).Select(i => generator.NextName).ToArray(); var items = new [] { root, world, universe }; var r = new Random(); for (int i = 0; i < actionCount; i++) { if (r.NextDouble() < .75) { var item = items[r.Next(0, items.Length)]; item.StringProperty = Enumerable.Range(0, 50).Select(index => words[r.Next(0, words.Length)]).Aggregate(new StringBuilder(), (sb, w) => sb.Append(w).Append(" ")).ToString(); asyncIndexer.Reindex(item.ID, false); } else { asyncIndexer.Delete(items[r.Next(0, items.Length)].ID); } Thread.Sleep(1); } int times = 0; while (asyncIndexer.GetCurrentStatus().QueueSize > 0) { Debug.WriteLine("Waiting " + times + " times on thread " + Thread.CurrentThread.ManagedThreadId); times++; Thread.Sleep(100); if (times > actionCount) { throw new Exception("Waited for a long time for indexing to finish"); } } }
//[TestCase(8, 2, 5000, 1000, 500, .5)] //[TestCase(16, 4, 30 * 1000, 10000, 1000, .5)] //[TestCase(8, 2, 300 * 1000, 20000, 2000, .5)] public void Multithreading(int readerCount, int indexerCount, int workMilliseconds, int dictionaryCount, int indexedWordsCount, double updateFrequency) { var threads = new List<Thread>(); var exceptions = new List<Exception>(); bool loop = true; var generator = new SCG.General.MarkovNameGenerator(Words.Thousand, 3, 2); var words = Enumerable.Range(0, dictionaryCount).Select(i => generator.NextName).ToArray(); int idCounter = 1; int creates = 0; int updates = 0; var indexFunction = new ThreadStart(() => { var r = new Random(); Trace.WriteLine(Thread.CurrentThread.ManagedThreadId + " Index start: " + DateTime.Now); while (loop) { bool isUpdate = r.NextDouble() < updateFrequency && idCounter > 2; int id = (isUpdate) ? r.Next(0, idCounter) : Interlocked.Increment(ref idCounter); var item = CreateOneItem<PersistableItem1>(id, "Item " + id, null); item.StringProperty = Enumerable.Range(0, indexedWordsCount).Select(i => words[r.Next(0, words.Length)]).Aggregate(new StringBuilder(), (sb, w) => sb.Append(w).Append(" ")).ToString(); try { indexer.Update(item); Console.Write(isUpdate ? 'U' : 'C'); if (isUpdate) Interlocked.Increment(ref updates); else Interlocked.Increment(ref creates); } catch (Exception ex) { Trace.WriteLine(ex); lock (exceptions) { exceptions.Add(ex); } } } Trace.WriteLine(Thread.CurrentThread.ManagedThreadId + " Index stop: " + DateTime.Now); }); var searcher = new LuceneSearcher(accessor, persister); int searches = 0; var searchFunction = new ThreadStart(() => { Trace.WriteLine(Thread.CurrentThread.ManagedThreadId + " Search start: " + DateTime.Now); var r = new Random(); while (loop) { try { var result = searcher.Search(Query.For(words[r.Next(words.Length)])); Interlocked.Increment(ref searches); Console.Write('?'); } catch (Exception ex) { Trace.WriteLine(ex); lock (exceptions) { exceptions.Add(ex); } } } Trace.WriteLine(Thread.CurrentThread.ManagedThreadId + " Search stop: " + DateTime.Now); }); for (int i = 0; i < indexerCount; i++) { threads.Add(new Thread(indexFunction)); } for (int i = 0; i < readerCount; i++) { threads.Add(new Thread(searchFunction)); } foreach (var t in threads) t.Start(); Thread.Sleep(workMilliseconds); loop = false; foreach (var t in threads) t.Join(); Trace.WriteLine("Creates: " + creates + ", Updates: " + updates + ", Searches: " + searches + " Exceptions: " + exceptions.Count); foreach (var ex in exceptions) Trace.WriteLine(ex.Message); exceptions.Count.ShouldBe(0); }
//[TestCase(8, 2, 5000, 1000, 500, .5)] //[TestCase(16, 4, 30 * 1000, 10000, 1000, .5)] //[TestCase(8, 2, 300 * 1000, 20000, 2000, .5)] public void Multithreading(int readerCount, int indexerCount, int workMilliseconds, int dictionaryCount, int indexedWordsCount, double updateFrequency) { var threads = new List <Thread>(); var exceptions = new List <Exception>(); bool loop = true; var generator = new SCG.General.MarkovNameGenerator(Words.Thousand, 3, 2); var words = Enumerable.Range(0, dictionaryCount).Select(i => generator.NextName).ToArray(); int idCounter = 1; int creates = 0; int updates = 0; var indexFunction = new ThreadStart(() => { var r = new Random(); Trace.WriteLine(Thread.CurrentThread.ManagedThreadId + " Index start: " + DateTime.Now); while (loop) { bool isUpdate = r.NextDouble() < updateFrequency && idCounter > 2; int id = (isUpdate) ? r.Next(0, idCounter) : Interlocked.Increment(ref idCounter); var item = CreateOneItem <PersistableItem>(id, "Item " + id, null); item.StringProperty = Enumerable.Range(0, indexedWordsCount).Select(i => words[r.Next(0, words.Length)]).Aggregate(new StringBuilder(), (sb, w) => sb.Append(w).Append(" ")).ToString(); try { indexer.Update(item); Console.Write(isUpdate ? 'U' : 'C'); if (isUpdate) { Interlocked.Increment(ref updates); } else { Interlocked.Increment(ref creates); } } catch (Exception ex) { Trace.WriteLine(ex); lock (exceptions) { exceptions.Add(ex); } } } Trace.WriteLine(Thread.CurrentThread.ManagedThreadId + " Index stop: " + DateTime.Now); }); int searches = 0; var searchFunction = new ThreadStart(() => { Trace.WriteLine(Thread.CurrentThread.ManagedThreadId + " Search start: " + DateTime.Now); var r = new Random(); while (loop) { try { var result = searcher.Search(Query.For(words[r.Next(words.Length)])); Interlocked.Increment(ref searches); Console.Write('?'); } catch (Exception ex) { Trace.WriteLine(ex); lock (exceptions) { exceptions.Add(ex); } } } Trace.WriteLine(Thread.CurrentThread.ManagedThreadId + " Search stop: " + DateTime.Now); }); for (int i = 0; i < indexerCount; i++) { threads.Add(new Thread(indexFunction)); } for (int i = 0; i < readerCount; i++) { threads.Add(new Thread(searchFunction)); } foreach (var t in threads) { t.Start(); } Thread.Sleep(workMilliseconds); loop = false; foreach (var t in threads) { t.Join(); } Trace.WriteLine("Creates: " + creates + ", Updates: " + updates + ", Searches: " + searches + " Exceptions: " + exceptions.Count); foreach (var ex in exceptions) { Trace.WriteLine(ex.Message); } exceptions.Count.ShouldBe(0); }