private async Task LoadDocumentsAsync()
{
if (_initialized == null)
{
throw new InvalidOperationException("InitializeAsync hasn't been called");
}
_logger.Display("Index documents");
_logger.WriteEvent("Index-Collection-Documents-Start");
var bufferSizes = GetDocumentBufferSizes();
using (var buffer = new IndexingBuffer(
bufferSizes.indexSize,
bufferSizes.contentSize,
_initialized.IndexFile.PushDocumentsAsync))
{
var batchCount = 0;
long lastTimeStamp = 0;
await foreach (var batch in _initialized.LogFile.ReadDocumentsFromLogFileAsync(
_initialized.IndexFile.LastDocumentTimeStamp,
_indexConstants.MaxLogBufferSize))
{
lastTimeStamp = batch.BatchTimeStamp;
++batchCount;
foreach (var item in batch.Items)
{
var(metaData, content) = SplitDocument(item, _initialized.LogFile.PartitionParts);
await buffer.WriteAsync(metaData, content);
}
}
await buffer.FlushAsync();
_initialized.IndexFile.UpdateLastDocumentTimeStamp(lastTimeStamp);
_logger.Display($"Indexed {buffer.ItemCount} documents in {batchCount} batches");
_logger
.AddContext("documentCount", buffer.ItemCount)
.AddContext("batchCount", batchCount)
.WriteEvent("Index-Collection-Documents-End");
}
}
private async Task LoadStoredProceduresAsync()
{
if (_initialized == null)
{
throw new InvalidOperationException("InitializeAsync hasn't been called");
}
_logger.Display("Index Stored Procedures");
_logger.WriteEvent("Index-Collection-Sprocs-Start");
var bufferSizes = GetStoredProcedureBufferSizes();
var sprocs = await LoadUntilNowSprocsIndexAsync();
var batchCount = 0;
long lastTimeStamp = 0;
using (var buffer = new IndexingBuffer(
bufferSizes.indexSize,
bufferSizes.contentSize,
_initialized.IndexFile.PushDocumentsAsync))
{
await foreach (var batch in _initialized.LogFile.ReadStoredProceduresAsync(
_initialized.IndexFile.LastStoredProcedureTimeStamp,
_indexConstants.MaxLogBufferSize))
{
var batchIds = ImmutableHashSet <string> .Empty;
lastTimeStamp = batch.BatchTimeStamp;
++batchCount;
foreach (var item in batch.Items)
{
batchIds = batchIds.Add(item.Id.Id);
if (!sprocs.ContainsKey(item.Id.Id) ||
sprocs[item.Id.Id].TimeStamp < item.Id.TimeStamp)
{
var(metaData, content) = item.Split();
sprocs = sprocs.SetItem(item.Id.Id, metaData.Id);
await buffer.WriteAsync(metaData, content);
}
}
var deletedIds = sprocs.Keys.ToImmutableHashSet().Except(batchIds);
foreach (var id in deletedIds)
{
sprocs = sprocs.Remove(id);
await buffer.WriteAsync(new ScriptMetaData(
new ScriptIdentifier(id, lastTimeStamp), 0),
EMPTY_CONTENT);
}
}
await buffer.FlushAsync();
_initialized.IndexFile.UpdateLastSprocTimeStamp(lastTimeStamp);
_logger.Display($"Indexed {buffer.ItemCount} stored procedures in {batchCount} batches");
_logger
.AddContext("documentCount", buffer.ItemCount)
.AddContext("batchCount", batchCount)
.WriteEvent("Index-Collection-Sprocs-End");
}
_logger.WriteEvent("Index-Collection-Sprocs-End");
}