コード例 #1
0
        private async Task LoadDocumentsAsync()
        {
            if (_initialized == null)
            {
                throw new InvalidOperationException("InitializeAsync hasn't been called");
            }

            _logger.Display("Index documents");
            _logger.WriteEvent("Index-Collection-Documents-Start");

            var bufferSizes = GetDocumentBufferSizes();

            using (var buffer = new IndexingBuffer(
                       bufferSizes.indexSize,
                       bufferSizes.contentSize,
                       _initialized.IndexFile.PushDocumentsAsync))
            {
                var  batchCount    = 0;
                long lastTimeStamp = 0;

                await foreach (var batch in _initialized.LogFile.ReadDocumentsFromLogFileAsync(
                                   _initialized.IndexFile.LastDocumentTimeStamp,
                                   _indexConstants.MaxLogBufferSize))
                {
                    lastTimeStamp = batch.BatchTimeStamp;
                    ++batchCount;
                    foreach (var item in batch.Items)
                    {
                        var(metaData, content) = SplitDocument(item, _initialized.LogFile.PartitionParts);

                        await buffer.WriteAsync(metaData, content);
                    }
                }
                await buffer.FlushAsync();

                _initialized.IndexFile.UpdateLastDocumentTimeStamp(lastTimeStamp);

                _logger.Display($"Indexed {buffer.ItemCount} documents in {batchCount} batches");
                _logger
                .AddContext("documentCount", buffer.ItemCount)
                .AddContext("batchCount", batchCount)
                .WriteEvent("Index-Collection-Documents-End");
            }
        }
コード例 #2
0
        private async Task LoadStoredProceduresAsync()
        {
            if (_initialized == null)
            {
                throw new InvalidOperationException("InitializeAsync hasn't been called");
            }

            _logger.Display("Index Stored Procedures");
            _logger.WriteEvent("Index-Collection-Sprocs-Start");

            var bufferSizes = GetStoredProcedureBufferSizes();
            var sprocs      = await LoadUntilNowSprocsIndexAsync();

            var  batchCount    = 0;
            long lastTimeStamp = 0;

            using (var buffer = new IndexingBuffer(
                       bufferSizes.indexSize,
                       bufferSizes.contentSize,
                       _initialized.IndexFile.PushDocumentsAsync))
            {
                await foreach (var batch in _initialized.LogFile.ReadStoredProceduresAsync(
                                   _initialized.IndexFile.LastStoredProcedureTimeStamp,
                                   _indexConstants.MaxLogBufferSize))
                {
                    var batchIds = ImmutableHashSet <string> .Empty;

                    lastTimeStamp = batch.BatchTimeStamp;
                    ++batchCount;
                    foreach (var item in batch.Items)
                    {
                        batchIds = batchIds.Add(item.Id.Id);
                        if (!sprocs.ContainsKey(item.Id.Id) ||
                            sprocs[item.Id.Id].TimeStamp < item.Id.TimeStamp)
                        {
                            var(metaData, content) = item.Split();

                            sprocs = sprocs.SetItem(item.Id.Id, metaData.Id);
                            await buffer.WriteAsync(metaData, content);
                        }
                    }
                    var deletedIds = sprocs.Keys.ToImmutableHashSet().Except(batchIds);

                    foreach (var id in deletedIds)
                    {
                        sprocs = sprocs.Remove(id);
                        await buffer.WriteAsync(new ScriptMetaData(
                                                    new ScriptIdentifier(id, lastTimeStamp), 0),
                                                EMPTY_CONTENT);
                    }
                }
                await buffer.FlushAsync();

                _initialized.IndexFile.UpdateLastSprocTimeStamp(lastTimeStamp);

                _logger.Display($"Indexed {buffer.ItemCount} stored procedures in {batchCount} batches");
                _logger
                .AddContext("documentCount", buffer.ItemCount)
                .AddContext("batchCount", batchCount)
                .WriteEvent("Index-Collection-Sprocs-End");
            }
            _logger.WriteEvent("Index-Collection-Sprocs-End");
        }