private async void StartIndexing(Interface.IIndexService indexservice, string path) { int pageCounter = 0; string line; var file = new StreamReader(path); var source = new List<Dictionary<string, string>>(3000000); while ((line = file.ReadLine()) != null) { pageCounter++; var document = new Dictionary<string, string>(); var id = pageCounter; document.Add("title", line.Substring(0, line.IndexOf('|') - 1)); document.Add("body", line.Substring(line.IndexOf('|') + 1)); source.Add(document); //indexservice.SendCommandToQueue( // "wikipedia", // IndexCommand.NewCreate(id.ToString(CultureInfo.InvariantCulture), document)); if (pageCounter % 100000 == 0) { Console.WriteLine(pageCounter); } } Console.WriteLine("Starting Test"); Stopwatch stopwatch = new Stopwatch(); stopwatch.Start(); pageCounter = 0; foreach (var item in source) { pageCounter++; await indexservice.CommandQueue() .SendAsync( new Tuple<string, IndexCommand>( "wikipedia", IndexCommand.NewCreate(pageCounter.ToString(CultureInfo.InvariantCulture), item))); } stopwatch.Stop(); FileInfo fileInfo = new FileInfo(path); double fileSize = fileInfo.Length / 1073741824.0; double time = stopwatch.ElapsedMilliseconds / 3600000.0; double indexingSpeed = fileSize / time; Console.WriteLine("Total Records indexed: " + pageCounter); Console.WriteLine("Total Elapsed time (ms): " + stopwatch.ElapsedMilliseconds); Console.WriteLine("Total Data Size (MB): " + fileInfo.Length / (1024.0 * 1024.0)); Console.WriteLine("Indexing Speed (GB/Hr): " + indexingSpeed); }