public override async Task TestFixtureSetUp() { await base.TestFixtureSetUp(); _logFormat = LogFormatHelper <TLogFormat, TStreamId> .LogFormatFactory.Create(new() { IndexDirectory = GetFilePathFor("index"), }); _provider = _logFormat.StreamNamesProvider; _publisher = new InMemoryBus("publisher"); _tfReader = new FakeInMemoryTfReader(RecordOffset); _tableIndex = new FakeInMemoryTableIndex <TStreamId>(); _provider.SetTableIndex(_tableIndex); _readerPool = new ObjectPool <ITransactionFileReader>( "ReadIndex readers pool", 5, 100, () => _tfReader); _indexBackend = new IndexBackend <TStreamId>(_readerPool, 100000, 100000); _streamIds = _logFormat.StreamIds; _validator = _logFormat.StreamIdValidator; var emptyStreamId = _logFormat.EmptyStreamId; _sizer = _logFormat.StreamIdSizer; _indexReader = new IndexReader <TStreamId>(_indexBackend, _tableIndex, _provider, _validator, _logFormat.StreamExistenceFilterReader, new StreamMetadata(maxCount: 100000), 100, false); _streamNames = _logFormat.StreamNames; _systemStreams = _logFormat.SystemStreams; _indexWriter = new IndexWriter <TStreamId>(_indexBackend, _indexReader, _streamIds, _streamNames, _systemStreams, emptyStreamId, _sizer); _indexCommitter = new IndexCommitter <TStreamId>(_publisher, _indexBackend, _indexReader, _tableIndex, _logFormat.StreamNameIndexConfirmer, _streamNames, _logFormat.EventTypeIndexConfirmer, _logFormat.EventTypes, _systemStreams, _logFormat.StreamExistenceFilter, _logFormat.StreamExistenceFilterInitializer, new InMemoryCheckpoint(-1), false); WriteEvents(); }
public virtual void TestFixtureSetUp() { _logFormat = LogFormatHelper <TLogFormat, TStreamId> .LogFormat; _publisher = new InMemoryBus("publisher"); _tfReader = new FakeInMemoryTfReader(RecordOffset); _tableIndex = new FakeInMemoryTableIndex <TStreamId>(); _readerPool = new ObjectPool <ITransactionFileReader>( "ReadIndex readers pool", 5, 100, () => _tfReader); _indexBackend = new IndexBackend <TStreamId>(_readerPool, 100000, 100000); var logFormat = LogFormatHelper <TLogFormat, TStreamId> .LogFormat; _streamIds = logFormat.StreamIds; _streamNames = logFormat.StreamNames; _systemStreams = logFormat.SystemStreams; _factory = logFormat.StreamNamesProvider; _validator = logFormat.StreamIdValidator; var emptyStreamId = logFormat.EmptyStreamId; _sizer = logFormat.StreamIdSizer; _indexReader = new IndexReader <TStreamId>(_indexBackend, _tableIndex, _factory, _validator, new StreamMetadata(maxCount: 100000), 100, false); _indexWriter = new IndexWriter <TStreamId>(_indexBackend, _indexReader, _streamIds, _streamNames, _systemStreams, emptyStreamId, _sizer); _indexCommitter = new Core.Services.Storage.ReaderIndex.IndexCommitter <TStreamId>(_publisher, _indexBackend, _indexReader, _tableIndex, _streamNames, _systemStreams, new InMemoryCheckpoint(-1), false); WriteEvents(); }
public ReadIndex(IPublisher bus, ObjectPool <ITransactionFileReader> readerPool, ITableIndex tableIndex, IHasher hasher, int streamInfoCacheCapacity, bool additionalCommitChecks, int metastreamMaxCount) { Ensure.NotNull(bus, "bus"); Ensure.NotNull(readerPool, "readerPool"); Ensure.NotNull(tableIndex, "tableIndex"); Ensure.NotNull(hasher, "hasher"); Ensure.Nonnegative(streamInfoCacheCapacity, "streamInfoCacheCapacity"); Ensure.Positive(metastreamMaxCount, "metastreamMaxCount"); var metastreamMetadata = new StreamMetadata(maxCount: metastreamMaxCount); _indexCache = new IndexCache(readerPool, streamInfoCacheCapacity, streamInfoCacheCapacity); _indexReader = new IndexReader(_indexCache, hasher, tableIndex, metastreamMetadata); var writer = new IndexWriter(bus, tableIndex, hasher, _indexCache, _indexReader, additionalCommitChecks); _indexWriter = writer; _indexCommitter = writer; _allReader = new AllReader(_indexCache); }
public ReadIndex(IPublisher bus, ObjectPool <ITransactionFileReader> readerPool, ITableIndex tableIndex, int streamInfoCacheCapacity, bool additionalCommitChecks, long metastreamMaxCount, int hashCollisionReadLimit, bool skipIndexScanOnReads, ICheckpoint replicationCheckpoint) { Ensure.NotNull(bus, "bus"); Ensure.NotNull(readerPool, "readerPool"); Ensure.NotNull(tableIndex, "tableIndex"); Ensure.Nonnegative(streamInfoCacheCapacity, "streamInfoCacheCapacity"); Ensure.Positive(metastreamMaxCount, "metastreamMaxCount"); Ensure.NotNull(replicationCheckpoint, "replicationCheckpoint"); var metastreamMetadata = new StreamMetadata(maxCount: metastreamMaxCount); _indexBackend = new IndexBackend(readerPool, streamInfoCacheCapacity, streamInfoCacheCapacity); _indexReader = new IndexReader(_indexBackend, tableIndex, metastreamMetadata, hashCollisionReadLimit, skipIndexScanOnReads); _indexWriter = new IndexWriter(_indexBackend, _indexReader); _indexCommitter = new IndexCommitter(bus, _indexBackend, _indexReader, tableIndex, additionalCommitChecks); _allReader = new AllReader(_indexBackend, _indexCommitter, replicationCheckpoint); _replicationCheckpoint = replicationCheckpoint; }
public LuceneSession(IDocumentMapper <T> mapper, IIndexWriter writer, Context context, IQueryable <T> queryable) { this.mapper = mapper; this.writer = writer; this.context = context; this.queryable = queryable; documentTracker = new SessionDocumentTracker(mapper); }
public Context(Directory directory, Analyzer analyzer, Version version, IIndexWriter indexWriter, object transactionLock) { this.directory = directory; this.analyzer = analyzer; this.version = version; this.indexWriter = indexWriter; this.transactionLock = transactionLock; }
public IndexService(IIndexWriter indexWriter) { if (indexWriter == null) { throw new ArgumentNullException("indexWriter"); } this.indexWriter = indexWriter; }
public ReadIndex(IPublisher bus, ObjectPool <ITransactionFileReader> readerPool, ITableIndex <TStreamId> tableIndex, INameIndexConfirmer <TStreamId> streamNameIndex, IValueLookup <TStreamId> streamIds, IStreamNamesProvider <TStreamId> streamNamesProvider, TStreamId emptyStreamName, IValidator <TStreamId> streamIdValidator, ISizer <TStreamId> sizer, INameExistenceFilter streamExistenceFilter, IExistenceFilterReader <TStreamId> streamExistenceFilterReader, INameIndexConfirmer <TStreamId> eventTypeIndex, int streamInfoCacheCapacity, bool additionalCommitChecks, long metastreamMaxCount, int hashCollisionReadLimit, bool skipIndexScanOnReads, IReadOnlyCheckpoint replicationCheckpoint, ICheckpoint indexCheckpoint) { Ensure.NotNull(bus, "bus"); Ensure.NotNull(readerPool, "readerPool"); Ensure.NotNull(tableIndex, "tableIndex"); Ensure.NotNull(streamIds, nameof(streamIds)); Ensure.NotNull(streamNamesProvider, nameof(streamNamesProvider)); Ensure.NotNull(streamIdValidator, nameof(streamIdValidator)); Ensure.NotNull(sizer, nameof(sizer)); Ensure.NotNull(streamExistenceFilter, nameof(streamExistenceFilter)); Ensure.NotNull(streamExistenceFilterReader, nameof(streamExistenceFilterReader)); Ensure.Nonnegative(streamInfoCacheCapacity, "streamInfoCacheCapacity"); Ensure.Positive(metastreamMaxCount, "metastreamMaxCount"); Ensure.NotNull(replicationCheckpoint, "replicationCheckpoint"); Ensure.NotNull(indexCheckpoint, "indexCheckpoint"); var metastreamMetadata = new StreamMetadata(maxCount: metastreamMaxCount); var indexBackend = new IndexBackend <TStreamId>(readerPool, streamInfoCacheCapacity, streamInfoCacheCapacity); _indexReader = new IndexReader <TStreamId>(indexBackend, tableIndex, streamNamesProvider, streamIdValidator, streamExistenceFilterReader, metastreamMetadata, hashCollisionReadLimit, skipIndexScanOnReads); _streamIds = streamIds; _streamNames = streamNamesProvider.StreamNames; var systemStreams = streamNamesProvider.SystemStreams; var eventTypeNames = streamNamesProvider.EventTypes; var streamExistenceFilterInitializer = streamNamesProvider.StreamExistenceFilterInitializer; _indexWriter = new IndexWriter <TStreamId>(indexBackend, _indexReader, _streamIds, _streamNames, systemStreams, emptyStreamName, sizer); _indexCommitter = new IndexCommitter <TStreamId>(bus, indexBackend, _indexReader, tableIndex, streamNameIndex, _streamNames, eventTypeIndex, eventTypeNames, systemStreams, streamExistenceFilter, streamExistenceFilterInitializer, indexCheckpoint, additionalCommitChecks); _allReader = new AllReader <TStreamId>(indexBackend, _indexCommitter, _streamNames, eventTypeNames); }
/// <summary> /// Constructs a new instance. /// If the supplied IndexWriter will be written to outside of this instance of LuceneDataProvider, /// the <paramref name="transactionLock"/> will be used to coordinate writes. /// </summary> public LuceneDataProvider(Directory directory, Analyzer externalAnalyzer, Version version, IIndexWriter externalWriter, object transactionLock) { this.directory = directory; this.externalAnalyzer = externalAnalyzer; this.perFieldAnalyzer = new PerFieldAnalyzer(new KeywordAnalyzer()); this.version = version; this.writerIsExternal = externalWriter != null; this.writer = externalWriter ?? IndexWriter; queryParser = RelinqQueryParserFactory.CreateQueryParser(); context = new Context(this.directory, transactionLock); }
public void SetUp() { mapper = MockRepository.GenerateStrictMock<IDocumentMapper<Record>>(); writer = MockRepository.GenerateStrictMock<IIndexWriter>(); context = MockRepository.GenerateStub<Context>(null, new object()); session = new LuceneSession<Record>(mapper, writer, context, null); mapper.Expect(m => m.ToKey(Arg<Record>.Is.NotNull)) .WhenCalled(mi => mi.ReturnValue = new DocumentKey(new Dictionary<IFieldMappingInfo, object> { { new FakeFieldMappingInfo { FieldName = "Id"}, ((Record)mi.Arguments[0]).Id } })) .Repeat.Any(); }
/// <summary> /// Gets the index builder. /// </summary> /// <param name="logger">The optional logger to use.</param> /// <returns>The index builder.</returns> public IndexBuilder GetBuilder(IMetadataSupplier supplier, ILogger logger) { EmbixProfile profile = new EmbixProfile(_profileCode, GetFactory()); IIndexWriter writer = GetIndexWriter(); writer.Logger = logger; writer.MetadataSupplier = supplier; return(new IndexBuilder(profile, _connFactory, writer) { Logger = logger }); }
public virtual void TestFixtureSetUp() { _publisher = new InMemoryBus("publisher"); _tfReader = new FakeInMemoryTfReader(RecordOffset); _tableIndex = new FakeInMemoryTableIndex(); _readerPool = new ObjectPool <ITransactionFileReader>( "ReadIndex readers pool", 5, 100, () => _tfReader); _indexBackend = new IndexBackend(_readerPool, 100000, 100000); _indexReader = new IndexReader(_indexBackend, _tableIndex, new StreamMetadata(maxCount: 100000), 100, false); _indexWriter = new IndexWriter(_indexBackend, _indexReader); _indexCommitter = new IndexCommitter(_publisher, _indexBackend, _indexReader, _tableIndex, false); WriteEvents(); }
public StorageWriterService(IPublisher bus, ISubscriber subscribeToBus, TimeSpan minFlushDelay, TFChunkDb db, TFChunkWriter writer, IIndexWriter indexWriter, IEpochManager epochManager, QueueStatsManager queueStatsManager) { Ensure.NotNull(bus, "bus"); Ensure.NotNull(subscribeToBus, "subscribeToBus"); Ensure.NotNull(db, "db"); Ensure.NotNull(writer, "writer"); Ensure.NotNull(indexWriter, "indexWriter"); Ensure.NotNull(epochManager, "epochManager"); Bus = bus; _subscribeToBus = subscribeToBus; Db = db; _indexWriter = indexWriter; EpochManager = epochManager; _minFlushDelay = minFlushDelay.TotalMilliseconds * TicksPerMs; _lastFlushDelay = 0; _lastFlushTimestamp = _watch.ElapsedTicks; Writer = writer; Writer.Open(); _writerBus = new InMemoryBus("StorageWriterBus", watchSlowMsg: false); StorageWriterQueue = QueuedHandler.CreateQueuedHandler(new AdHocHandler <Message>(CommonHandle), "StorageWriterQueue", queueStatsManager, true, TimeSpan.FromMilliseconds(500)); _tasks.Add(StorageWriterQueue.Start()); SubscribeToMessage <SystemMessage.SystemInit>(); SubscribeToMessage <SystemMessage.StateChangeMessage>(); SubscribeToMessage <SystemMessage.WriteEpoch>(); SubscribeToMessage <SystemMessage.WaitForChaserToCatchUp>(); SubscribeToMessage <StorageMessage.WritePrepares>(); SubscribeToMessage <StorageMessage.WriteDelete>(); SubscribeToMessage <StorageMessage.WriteTransactionStart>(); SubscribeToMessage <StorageMessage.WriteTransactionData>(); SubscribeToMessage <StorageMessage.WriteTransactionEnd>(); SubscribeToMessage <StorageMessage.WriteCommit>(); }
public void SetUp() { mapper = MockRepository.GenerateStrictMock <IDocumentMapper <Record> >(); writer = MockRepository.GenerateStrictMock <IIndexWriter>(); context = MockRepository.GenerateStub <Context>(null, new object()); session = new LuceneSession <Record>(mapper, writer, context, null); mapper.Expect(m => m.ToKey(Arg <Record> .Is.NotNull)) .WhenCalled(mi => mi.ReturnValue = new DocumentKey(new Dictionary <IFieldMappingInfo, object> { { new FakeFieldMappingInfo { FieldName = "Id" }, ((Record)mi.Arguments[0]).Id } })) .Repeat.Any(); }
public void TestBaseSetUp() { packagePathResolver = new Mock<IPackagePathResolver>(); loader = new Mock<ILucenePackageRepository>(MockBehavior.Strict); fileSystem = new Mock<IFileSystem>(); packagePathResolver.Setup(p => p.GetPackageDirectory(It.IsAny<IPackage>())).Returns("package-dir"); packagePathResolver.Setup(p => p.GetPackageFileName(It.IsAny<IPackage>())).Returns((Func<IPackage, string>)(pkg => pkg.Id)); var dir = new RAMDirectory(); provider = new LuceneDataProvider(dir, Version.LUCENE_30); indexWriter = provider.IndexWriter; datasource = provider.AsQueryable(() => new LucenePackage(fileSystem.Object)); }
public void TestBaseSetUp() { packagePathResolver = new Mock <IPackagePathResolver>(); loader = new Mock <ILucenePackageRepository>(MockBehavior.Strict); fileSystem = new Mock <IFileSystem>(); packagePathResolver.Setup(p => p.GetPackageDirectory(It.IsAny <IPackage>())).Returns("package-dir"); packagePathResolver.Setup(p => p.GetPackageDirectory(It.IsAny <string>(), It.IsAny <SemanticVersion>())).Returns("package-dir"); packagePathResolver.Setup(p => p.GetPackageFileName(It.IsAny <IPackage>())).Returns((Func <IPackage, string>)(pkg => pkg.Id)); packagePathResolver.Setup(p => p.GetPackageFileName(It.IsAny <string>(), It.IsAny <SemanticVersion>())).Returns((Func <string, SemanticVersion, string>)((id, version) => id + "." + version)); var dir = new RAMDirectory(); provider = new LuceneDataProvider(dir, Version.LUCENE_30); indexWriter = provider.IndexWriter; datasource = provider.AsQueryable(() => new LucenePackage(fileSystem.Object)); }
public ClusterStorageWriterService(IPublisher bus, ISubscriber subscribeToBus, TimeSpan minFlushDelay, TFChunkDb db, TFChunkWriter writer, IIndexWriter indexWriter, IEpochManager epochManager, Func <long> getLastCommitPosition) : base(bus, subscribeToBus, minFlushDelay, db, writer, indexWriter, epochManager) { Ensure.NotNull(getLastCommitPosition, "getLastCommitPosition"); _getLastCommitPosition = getLastCommitPosition; _framer = new LengthPrefixSuffixFramer(OnLogRecordUnframed, TFConsts.MaxLogRecordSize); SubscribeToMessage <ReplicationMessage.ReplicaSubscribed>(); SubscribeToMessage <ReplicationMessage.CreateChunk>(); SubscribeToMessage <ReplicationMessage.RawChunkBulk>(); SubscribeToMessage <ReplicationMessage.DataChunkBulk>(); }
public ReadIndex(IPublisher bus, ObjectPool<ITransactionFileReader> readerPool, ITableIndex tableIndex, int streamInfoCacheCapacity, bool additionalCommitChecks, int metastreamMaxCount, int hashCollisionReadLimit) { Ensure.NotNull(bus, "bus"); Ensure.NotNull(readerPool, "readerPool"); Ensure.NotNull(tableIndex, "tableIndex"); Ensure.Nonnegative(streamInfoCacheCapacity, "streamInfoCacheCapacity"); Ensure.Positive(metastreamMaxCount, "metastreamMaxCount"); var metastreamMetadata = new StreamMetadata(maxCount: metastreamMaxCount); _indexBackend = new IndexBackend(readerPool, streamInfoCacheCapacity, streamInfoCacheCapacity); _indexReader = new IndexReader(_indexBackend, tableIndex, metastreamMetadata, hashCollisionReadLimit); _indexWriter = new IndexWriter(_indexBackend, _indexReader); _indexCommitter = new IndexCommitter(bus, _indexBackend, _indexReader, tableIndex, additionalCommitChecks); _allReader = new AllReader(_indexBackend, _indexCommitter); }
public ReadIndex(IPublisher bus, ObjectPool<ITransactionFileReader> readerPool, ITableIndex tableIndex, IHasher hasher, int streamInfoCacheCapacity, bool additionalCommitChecks, int metastreamMaxCount) { Ensure.NotNull(bus, "bus"); Ensure.NotNull(readerPool, "readerPool"); Ensure.NotNull(tableIndex, "tableIndex"); Ensure.NotNull(hasher, "hasher"); Ensure.Nonnegative(streamInfoCacheCapacity, "streamInfoCacheCapacity"); Ensure.Positive(metastreamMaxCount, "metastreamMaxCount"); var metastreamMetadata = new StreamMetadata(maxCount: metastreamMaxCount); _indexCache = new IndexCache(readerPool, streamInfoCacheCapacity, streamInfoCacheCapacity); _indexReader = new IndexReader(_indexCache, hasher, tableIndex, metastreamMetadata); var writer = new IndexWriter(bus, tableIndex, hasher, _indexCache, _indexReader, additionalCommitChecks); _indexWriter = writer; _indexCommitter = writer; _allReader = new AllReader(_indexCache); }
/// <summary> /// init the index flow controller /// </summary> /// <param name="processor"></param> public IndexController(IIndexWriter processor) { this.Processor = processor; }
/// <summary> /// If the supplied IndexWriter will be written to outside of this instance of LuceneDataProvider, /// the <paramref name="transactionLock"/> will be used to coordinate writes. /// </summary> /// <param name="directory"></param> /// <param name="analyzer"></param> /// <param name="version"></param> /// <param name="indexWriter"></param> /// <param name="transactionLock"></param> public LuceneDataProvider(Directory directory, Analyzer analyzer, Version version, IIndexWriter indexWriter, object transactionLock) { this.directory = directory; this.analyzer = analyzer; this.version = version; queryParser = RelinqQueryParserFactory.CreateQueryParser(); context = new Context(this.directory, this.analyzer, this.version, indexWriter, transactionLock); }
/// <summary> /// Constructs a new instance with a client provided <see cref="Analyzer"/>. /// If the supplied IndexWriter will be written to outside of this instance of LuceneDataProvider, /// the <paramref name="transactionLock"/> will be used to coordinate writes. /// </summary> public LuceneDataProvider(Directory directory, Version version, IIndexWriter externalWriter, object transactionLock) : this(directory, null, version, externalWriter, transactionLock) { }
public TestableContext(Directory directory, Analyzer analyzer, Version version, IIndexWriter indexWriter, object transactionLock) : base(directory, transactionLock) { }
public void Build(IIndexWriter writer, Dynamicweb.Diagnostics.Tracking.Tracker tracker) { #region Implementation writer.Open(openForAppend: false); if (!TaskManager.Context.ContainsKey("Database.ConnectionString")) { return; } using (var conn = new SqlConnection(TaskManager.Context["Database.ConnectionString"].ToString())) { #region Connection conn.Open(); // Get count using (var cmd = conn.CreateCommand()) { tracker.LogInformation("Getting count"); cmd.CommandText = "SELECT COUNT(AccessUserID) FROM AccessUser WHERE AccessUserType = 5"; tracker.Status.TotalCount = (int)cmd.ExecuteScalar(); } // Handle users using (var cmd = conn.CreateCommand()) { cmd.CommandText = "SELECT AccessUserID, AccessUserUserName, AccessUserName, AccessUserEmail, AccessUserNewsletterAllowed, AccessUserGroups FROM AccessUser WHERE AccessUserType = 5"; using (var reader = cmd.ExecuteReader()) { tracker.LogInformation("Handling users"); while (reader.Read()) { #region Build document var doc = new IndexDocument(); for (var i = 0; i < reader.FieldCount; i++) { var name = reader.GetName(i); var value = reader.GetValue(i); if (name == "AccessUserGroups") { #region Handle groups if (reader.IsDBNull(i)) { continue; } var groupIds = value.ToString().Split(new[] { '@' }, StringSplitOptions.RemoveEmptyEntries); var groupIdList = groupIds .Select(id => id.Trim('@')) .Where(id => !string.IsNullOrEmpty(id)) .Select(int.Parse) .ToList(); if (groupIdList.Count > 0) { doc.Add("Groups", groupIdList); } #endregion } else { doc.Add(name, value); } } writer.AddDocument(doc); tracker.IncrementCounter(); #endregion } } } #endregion } #endregion }
public LuceneProcessor(string indexPath) { this.indexWriter = new DirectoryIndexWriter(new DirectoryInfo(indexPath)); }
public LuceneProcessor(IIndexWriter indexWriter) { this.indexWriter = indexWriter; }
public IndexWriterFilterDecorator(IIndexWriter writer, ILogger <SingleThreadedIndexer> logger) { _writer = Condition.Requires(writer, nameof(writer)).IsNotNull().Value; _logger = Condition.Requires(logger, nameof(logger)).IsNotNull().Value; }
public CustomerIndexEngine(IIndexWriter <CustomerCacheModel> indexWriter) { _indexWriter = indexWriter ?? throw new ArgumentNullException(nameof(indexWriter)); }