public void WaitForIndexesToBecomeNonStale()
{
MemoryUsage.Clear();
while (true)
{
process.Refresh();
MemoryUsage.Add(process.WorkingSet64);
var statistics = store.DatabaseCommands.GetStatistics();
if (statistics.StaleIndexes.Length == 0 && doneImporting)
{
return;
}
foreach (var staleIndex in statistics.StaleIndexes)
{
var indexStats = statistics.Indexes.Single(x => x.Name == staleIndex);
var latencyInTime = (DateTime.UtcNow - indexStats.LastIndexedTimestamp).TotalMilliseconds;
LatencyTimes.Add(new KeyValuePair <string, double>(staleIndex, latencyInTime));
var latencyInDocuments = statistics.CountOfDocuments - indexStats.IndexingAttempts;
LatencyInDocuments.Add(new KeyValuePair <string, long>(staleIndex, latencyInDocuments));
logger.Debug("Stale index {0} - {1:#,#}/{2:#,#} - latency: {3:#,#}, {4:#,#}ms", indexStats.Id, indexStats.IndexingAttempts, statistics.CountOfDocuments,
latencyInDocuments,
latencyInTime);
}
Thread.Sleep(1000);
}
}
private Stopwatch ParseDisks(Action <Disk> addToBatch)
{
int i = 0;
var parser = new Parser();
var buffer = new byte[1024 * 1024]; // more than big enough for all files
var sp = Stopwatch.StartNew();
using (var bz2 = new BZip2InputStream(File.Open(dataLocation, FileMode.Open)))
using (var tar = new TarInputStream(bz2))
{
TarEntry entry;
while ((entry = tar.GetNextEntry()) != null)
{
if (entry.Size == 0 || entry.Name == "README" || entry.Name == "COPYING")
{
continue;
}
var readSoFar = 0;
while (true)
{
var read = tar.Read(buffer, readSoFar, ((int)entry.Size) - readSoFar);
if (read == 0)
{
break;
}
readSoFar += read;
}
// we do it in this fashion to have the stream reader detect the BOM / unicode / other stuff
// so we can read the values properly
var fileText = new StreamReader(new MemoryStream(buffer, 0, readSoFar)).ReadToEnd();
try
{
var disk = parser.Parse(fileText);
addToBatch(disk);
if (i++ % BatchSize == 0)
{
process.Refresh();
MemoryUsage.Add(process.WorkingSet64);
logger.Info("\r{0} {1:#,#} {2} ", entry.Name, i, sp.Elapsed);
}
}
catch (Exception e)
{
logger.Error("");
logger.Error(entry.Name);
logger.Error(e);
return(sp);
}
}
}
return(sp);
}