private static async Task Index(string dir, string collection, int skip, int take, int batchSize)
{
var timer = new Stopwatch();
timer.Start();
var files = Directory.GetFiles(dir, "*.docs");
var sessionFactory = new SessionFactory(
dir,
new LatinTokenizer(),
new IniConfiguration(Path.Combine(Directory.GetCurrentDirectory(), "sir.ini")));
var batchNo = 0;
foreach (var docFileName in files)
{
var name = Path.GetFileNameWithoutExtension(docFileName)
.Split(".", StringSplitOptions.RemoveEmptyEntries);
var collectionId = name[0];
if (collectionId == collection.ToHash().ToString())
{
using (var readSession = new DocumentStreamSession(collection, sessionFactory))
{
var docs = readSession.ReadDocs();
if (skip > 0)
{
docs = docs.Skip(skip);
}
if (take > 0)
{
docs = docs.Take(take);
}
var writeTimer = new Stopwatch();
foreach (var batch in docs.Batch(batchSize))
{
writeTimer.Restart();
var job = new IndexingJob(collection, batch);
using (var indexSession = sessionFactory.CreateIndexSession(collection))
{
await indexSession.Write(job);
}
_log.Log(string.Format("batch {0} done in {1}", batchNo++, writeTimer.Elapsed));
}
}
break;
}
}
_log.Log(string.Format("indexing took {0}", timer.Elapsed));
}
protected override JobHandle OnUpdate(JobHandle inputDependencies)
{
_lastAllocation++;
LastJob.Complete();
if (_batchQueue.Count > 0)
{
return(inputDependencies);
}
Profiler.BeginSample("Dependencies");
var voxelColorType = GetArchetypeChunkComponentType <VoxelColor>(true);
var matrixType = GetArchetypeChunkComponentType <LocalToWorld>(true);
var chunks = _query.CreateArchetypeChunkArrayAsync(Allocator.TempJob, out var chunksHandle);
var chunkCount = _query.CalculateChunkCount();
var spaceRequirement = _query.CalculateEntityCount();
var indexMappings = new NativeArray <int>(chunkCount, Allocator.TempJob);
var indexing = new IndexingJob
{
IndexMappings = indexMappings,
Chunks = chunks,
ChunksPerBatch = chunksPerBatch
}.Schedule(JobHandle.CombineDependencies(inputDependencies, chunksHandle));
Profiler.EndSample();
JobHandle deps = indexing;
Profiler.BeginSample("Prepare Space");
if (_lastColors.Length == spaceRequirement && _lastAllocation < 4)
{
var colorClear = new MemsetNativeArray <VoxelColor>
{
Source = _lastColors,
Value = default
}.Schedule(_lastColors.Length, 256, inputDependencies);