/// <summary>
/// Merges the given index into the called index. The given index gets changed and should be discarded
/// </summary>
public WriteableIndex Merge(WriteableIndex other)
{
var offset = _documents.Count;
// add documents to list
_documents.AddRange(other._documents);
_documentLength.AddRange(other._documentLength);
// add dictionary (with added offset)
foreach (var item in other._store)
{
// add to dict (existing or new)
var index = _store.InitOrGetPosition(item.Key);
var itemOld = _store.GetAtPosition(index);
if (itemOld == null)
{
item.Value.IncreaseDocumentIndex(offset);
_store.StoreAtPosition(index, item.Value);
}
else
{
itemOld.Append(item.Value, offset);
}
}
return(this);
}
public static void SerializeIndexToDisk(WriteableIndex index, string filePath)
{
using (Stream stream = File.Open(filePath, FileMode.Create))
{
index.Serialize(stream);
}
}
public unsafe (WriteableIndex index, int files, int docs, long size) IndexAllParallel(IndexOptions options, string folder)
{
var timer = Stopwatch.StartNew();
var files = Directory.GetFiles(folder, "*", SearchOption.AllDirectories);
Console.WriteLine("Found files: " + files.Length + " - took: " + timer.ElapsedMilliseconds + "ms");
timer.Restart();
var tasks = new List <Task <WriteableIndex> >();
var parallel = Environment.ProcessorCount;
var docsCount = 0;
var fileCount = 0;
var sizeSum = 0L;
var sizeLocal = 0L;
for (var p = 0; p < parallel; p++)
{
var taskNumber = p;
tasks.Add(Task.Run(() =>
{
var localIndex = new WriteableIndex(options);
var localStemmer = new Stemmer();
var localParser = new Parser();
var localPart = (files.Length / parallel);
var from = taskNumber * localPart;
var to = taskNumber == parallel - 1 ? files.Length : from + localPart;
for (var i = from; i < to; i++)
{
using (var mmf = MemoryMappedFile.CreateFromFile(files[i], FileMode.Open))
using (var accessor = mmf.CreateViewAccessor())
{
byte *buffer = null;
accessor.SafeMemoryMappedViewHandle.AcquirePointer(ref buffer);
var len = new FileInfo(files[i]).Length;
var docs = localParser.ParseFileFast(buffer, len);
Interlocked.Add(ref docsCount, docs.Count);
Interlocked.Add(ref sizeSum, len);
Interlocked.Add(ref sizeLocal, len);
IndexDocuments(localIndex, localStemmer, buffer, docs);
accessor.SafeMemoryMappedViewHandle.ReleasePointer();
}
if (Interlocked.Increment(ref fileCount) % 100 == 0)
{
Console.WriteLine(
"Finished: " + fileCount + " - "
+ Math.Round((Interlocked.Read(ref sizeLocal) * 0.000001d)) + " mb - " +
+Math.Round((Interlocked.Read(ref sizeLocal) * 0.000001d) / (timer.ElapsedMilliseconds / 1000d), 2) + " mb/s");
timer.Restart();
Interlocked.Exchange(ref sizeLocal, 0);
}
}
Console.WriteLine("task finished: " + taskNumber);
return(localIndex);
}
));
}
var all = Task.WhenAll(tasks).Result;
Console.WriteLine("Index building completed. Merging indices ...");
var master = all[0];
var mergeTime = Stopwatch.StartNew();
for (var i = 1; i < all.Length; i++)
{
master.Merge(all[i]);
}
mergeTime.Stop();
Console.WriteLine("Merge complete after: " + mergeTime.ElapsedMilliseconds + " ms");
timer.Stop();
return(master, fileCount, docsCount, sizeSum);
}
private unsafe void IndexDocuments(WriteableIndex index, Stemmer stemmer, byte *buffer, List <(string id, int from, int length)> docs)