public FileSystemBase()
{
string testFolder = Path.Join(AppDomain.CurrentDomain.BaseDirectory, this.GetType().Name);
if (Directory.Exists(testFolder))
{
Directory.Delete(testFolder, true);
}
Directory.CreateDirectory(testFolder);
Settings = new TetriSettings(new TestLogger <TetriSettings>())
{
RepositoryPath = Path.Join(testFolder, "repository"),
PackagePath = Path.Join(testFolder, "packages"),
TempPath = Path.Join(testFolder, "temp"),
ArchivePath = Path.Join(testFolder, "archives"),
TagsPath = Path.Join(testFolder, "tags")
};
Logger = new TestLogger <IIndexReader>();
TagService = new Core.TagsService(
Settings,
new TestLogger <ITagsService>(), new PackageListCache(MemoryCacheHelper.GetInstance()));
IndexReader = new Core.IndexReader(Settings, new Core.ThreadDefault(), TagService, Logger, new FileSystem(), HashServiceHelper.Instance());
Thread.Sleep(200);// fixes race condition when scaffolding up index between consecutive tests
IndexReader.Initialize();
}
public ReadIndex(IPublisher bus,
ObjectPool <ITransactionFileReader> readerPool,
ITableIndex tableIndex,
IHasher hasher,
int streamInfoCacheCapacity,
bool additionalCommitChecks,
int metastreamMaxCount)
{
Ensure.NotNull(bus, "bus");
Ensure.NotNull(readerPool, "readerPool");
Ensure.NotNull(tableIndex, "tableIndex");
Ensure.NotNull(hasher, "hasher");
Ensure.Nonnegative(streamInfoCacheCapacity, "streamInfoCacheCapacity");
Ensure.Positive(metastreamMaxCount, "metastreamMaxCount");
var metastreamMetadata = new StreamMetadata(maxCount: metastreamMaxCount);
_indexCache = new IndexCache(readerPool, streamInfoCacheCapacity, streamInfoCacheCapacity);
_indexReader = new IndexReader(_indexCache, hasher, tableIndex, metastreamMetadata);
var writer = new IndexWriter(bus, tableIndex, hasher, _indexCache, _indexReader, additionalCommitChecks);
_indexWriter = writer;
_indexCommitter = writer;
_allReader = new AllReader(_indexCache);
}
public virtual void TestFixtureSetUp()
{
_logFormat = LogFormatHelper <TLogFormat, TStreamId> .LogFormat;
_publisher = new InMemoryBus("publisher");
_tfReader = new FakeInMemoryTfReader(RecordOffset);
_tableIndex = new FakeInMemoryTableIndex <TStreamId>();
_readerPool = new ObjectPool <ITransactionFileReader>(
"ReadIndex readers pool", 5, 100,
() => _tfReader);
_indexBackend = new IndexBackend <TStreamId>(_readerPool, 100000, 100000);
var logFormat = LogFormatHelper <TLogFormat, TStreamId> .LogFormat;
_streamIds = logFormat.StreamIds;
_streamNames = logFormat.StreamNames;
_systemStreams = logFormat.SystemStreams;
_factory = logFormat.StreamNamesProvider;
_validator = logFormat.StreamIdValidator;
var emptyStreamId = logFormat.EmptyStreamId;
_sizer = logFormat.StreamIdSizer;
_indexReader = new IndexReader <TStreamId>(_indexBackend, _tableIndex, _factory, _validator, new StreamMetadata(maxCount: 100000), 100, false);
_indexWriter = new IndexWriter <TStreamId>(_indexBackend, _indexReader, _streamIds, _streamNames, _systemStreams, emptyStreamId, _sizer);
_indexCommitter = new Core.Services.Storage.ReaderIndex.IndexCommitter <TStreamId>(_publisher, _indexBackend, _indexReader, _tableIndex, _streamNames, _systemStreams, new InMemoryCheckpoint(-1), false);
WriteEvents();
}
public void Setup()
{
given();
_indexDir = PathName;
_fakeReader = new TFReaderLease(new FakeReader());
_indexBackend = new FakeIndexBackend <string>(_fakeReader);
_logFormat = LogFormatHelper <LogFormat.V2, string> .LogFormatFactory.Create(new() {
InMemory = true,
});
_lowHasher = _logFormat.LowHasher;
_highHasher = _logFormat.HighHasher;
_tableIndex = new TableIndex <string>(_indexDir, _lowHasher, _highHasher, _logFormat.EmptyStreamId,
() => new HashListMemTable(PTableVersions.IndexV1, maxSize: _maxMemTableSize),
() => _fakeReader,
PTableVersions.IndexV1,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: _maxMemTableSize,
maxTablesPerLevel: 2);
_logFormat.StreamNamesProvider.SetTableIndex(_tableIndex);
_tableIndex.Initialize(long.MaxValue);
_indexReader = new IndexReader <string>(_indexBackend, _tableIndex,
_logFormat.StreamNamesProvider,
_logFormat.StreamIdValidator,
_logFormat.StreamExistenceFilterReader,
new EventStore.Core.Data.StreamMetadata(),
_hashCollisionReadLimit, skipIndexScanOnRead: false);
when();
//wait for the mem table to be dumped
System.Threading.Thread.Sleep(500);
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
_logFormat = LogFormatHelper <TLogFormat, TStreamId> .LogFormatFactory.Create(new() {
IndexDirectory = GetFilePathFor("index"),
});
_provider = _logFormat.StreamNamesProvider;
_publisher = new InMemoryBus("publisher");
_tfReader = new FakeInMemoryTfReader(RecordOffset);
_tableIndex = new FakeInMemoryTableIndex <TStreamId>();
_provider.SetTableIndex(_tableIndex);
_readerPool = new ObjectPool <ITransactionFileReader>(
"ReadIndex readers pool", 5, 100,
() => _tfReader);
_indexBackend = new IndexBackend <TStreamId>(_readerPool, 100000, 100000);
_streamIds = _logFormat.StreamIds;
_validator = _logFormat.StreamIdValidator;
var emptyStreamId = _logFormat.EmptyStreamId;
_sizer = _logFormat.StreamIdSizer;
_indexReader = new IndexReader <TStreamId>(_indexBackend, _tableIndex, _provider, _validator,
_logFormat.StreamExistenceFilterReader, new StreamMetadata(maxCount: 100000), 100, false);
_streamNames = _logFormat.StreamNames;
_systemStreams = _logFormat.SystemStreams;
_indexWriter = new IndexWriter <TStreamId>(_indexBackend, _indexReader, _streamIds, _streamNames,
_systemStreams, emptyStreamId, _sizer);
_indexCommitter = new IndexCommitter <TStreamId>(_publisher, _indexBackend, _indexReader, _tableIndex,
_logFormat.StreamNameIndexConfirmer, _streamNames, _logFormat.EventTypeIndexConfirmer, _logFormat.EventTypes,
_systemStreams, _logFormat.StreamExistenceFilter, _logFormat.StreamExistenceFilterInitializer, new InMemoryCheckpoint(-1), false);
WriteEvents();
}
public ReadIndex(IPublisher bus,
ObjectPool <ITransactionFileReader> readerPool,
ITableIndex tableIndex,
int streamInfoCacheCapacity,
bool additionalCommitChecks,
long metastreamMaxCount,
int hashCollisionReadLimit,
bool skipIndexScanOnReads,
ICheckpoint replicationCheckpoint)
{
Ensure.NotNull(bus, "bus");
Ensure.NotNull(readerPool, "readerPool");
Ensure.NotNull(tableIndex, "tableIndex");
Ensure.Nonnegative(streamInfoCacheCapacity, "streamInfoCacheCapacity");
Ensure.Positive(metastreamMaxCount, "metastreamMaxCount");
Ensure.NotNull(replicationCheckpoint, "replicationCheckpoint");
var metastreamMetadata = new StreamMetadata(maxCount: metastreamMaxCount);
_indexBackend = new IndexBackend(readerPool, streamInfoCacheCapacity, streamInfoCacheCapacity);
_indexReader = new IndexReader(_indexBackend, tableIndex, metastreamMetadata, hashCollisionReadLimit, skipIndexScanOnReads);
_indexWriter = new IndexWriter(_indexBackend, _indexReader);
_indexCommitter = new IndexCommitter(bus, _indexBackend, _indexReader, tableIndex, additionalCommitChecks);
_allReader = new AllReader(_indexBackend, _indexCommitter, replicationCheckpoint);
_replicationCheckpoint = replicationCheckpoint;
}
public IndexWriter(IIndexBackend indexBackend, IIndexReader indexReader)
{
Ensure.NotNull(indexBackend, "indexBackend");
Ensure.NotNull(indexReader, "indexReader");
_indexBackend = indexBackend;
_indexReader = indexReader;
}
public IndexWriter(IIndexBackend indexBackend, IIndexReader indexReader)
{
Ensure.NotNull(indexBackend, "indexBackend");
Ensure.NotNull(indexReader, "indexReader");
_indexBackend = indexBackend;
_indexReader = indexReader;
}
public PackageCreate(IIndexReader indexReader, ITetriSettings settings, ILogger <IPackageCreate> log, IWorkspace workspace, IHashService hashService)
{
_indexReader = indexReader;
_log = log;
_workspace = workspace;
_settings = settings;
_hashService = hashService;
}
public IndexCommitter(IPublisher bus, IIndexBackend backend, IIndexReader indexReader,
ITableIndex tableIndex, bool additionalCommitChecks)
{
_bus = bus;
_backend = backend;
_indexReader = indexReader;
_tableIndex = tableIndex;
_additionalCommitChecks = additionalCommitChecks;
}
public IndexCommitter(IPublisher bus, IIndexBackend backend, IIndexReader indexReader,
ITableIndex tableIndex, bool additionalCommitChecks)
{
_bus = bus;
_backend = backend;
_indexReader = indexReader;
_tableIndex = tableIndex;
_additionalCommitChecks = additionalCommitChecks;
}
/// <summary>
///
/// </summary>
/// <param name="packageService"></param>
/// <param name="settings"></param>
/// <param name="indexService"></param>
/// <param name="settings"></param>
/// <param name="log"></param>
public PackagesController(IPackageCreate packageService, IPackageList packageList, IPackageListCache packageListCache, IIndexReader indexService, ITetriSettings settings, ILogger <PackagesController> log)
{
_packageList = packageList;
_packageService = packageService;
_packageListCache = packageListCache;
_indexService = indexService;
_settings = settings;
_log = log;
}
public ReadIndex(IPublisher bus,
ObjectPool <ITransactionFileReader> readerPool,
ITableIndex <TStreamId> tableIndex,
INameIndexConfirmer <TStreamId> streamNameIndex,
IValueLookup <TStreamId> streamIds,
IStreamNamesProvider <TStreamId> streamNamesProvider,
TStreamId emptyStreamName,
IValidator <TStreamId> streamIdValidator,
ISizer <TStreamId> sizer,
INameExistenceFilter streamExistenceFilter,
IExistenceFilterReader <TStreamId> streamExistenceFilterReader,
INameIndexConfirmer <TStreamId> eventTypeIndex,
int streamInfoCacheCapacity,
bool additionalCommitChecks,
long metastreamMaxCount,
int hashCollisionReadLimit,
bool skipIndexScanOnReads,
IReadOnlyCheckpoint replicationCheckpoint,
ICheckpoint indexCheckpoint)
{
Ensure.NotNull(bus, "bus");
Ensure.NotNull(readerPool, "readerPool");
Ensure.NotNull(tableIndex, "tableIndex");
Ensure.NotNull(streamIds, nameof(streamIds));
Ensure.NotNull(streamNamesProvider, nameof(streamNamesProvider));
Ensure.NotNull(streamIdValidator, nameof(streamIdValidator));
Ensure.NotNull(sizer, nameof(sizer));
Ensure.NotNull(streamExistenceFilter, nameof(streamExistenceFilter));
Ensure.NotNull(streamExistenceFilterReader, nameof(streamExistenceFilterReader));
Ensure.Nonnegative(streamInfoCacheCapacity, "streamInfoCacheCapacity");
Ensure.Positive(metastreamMaxCount, "metastreamMaxCount");
Ensure.NotNull(replicationCheckpoint, "replicationCheckpoint");
Ensure.NotNull(indexCheckpoint, "indexCheckpoint");
var metastreamMetadata = new StreamMetadata(maxCount: metastreamMaxCount);
var indexBackend = new IndexBackend <TStreamId>(readerPool, streamInfoCacheCapacity, streamInfoCacheCapacity);
_indexReader = new IndexReader <TStreamId>(indexBackend, tableIndex, streamNamesProvider, streamIdValidator,
streamExistenceFilterReader, metastreamMetadata, hashCollisionReadLimit, skipIndexScanOnReads);
_streamIds = streamIds;
_streamNames = streamNamesProvider.StreamNames;
var systemStreams = streamNamesProvider.SystemStreams;
var eventTypeNames = streamNamesProvider.EventTypes;
var streamExistenceFilterInitializer = streamNamesProvider.StreamExistenceFilterInitializer;
_indexWriter = new IndexWriter <TStreamId>(indexBackend, _indexReader, _streamIds, _streamNames, systemStreams, emptyStreamName, sizer);
_indexCommitter = new IndexCommitter <TStreamId>(bus, indexBackend, _indexReader, tableIndex, streamNameIndex,
_streamNames, eventTypeIndex, eventTypeNames, systemStreams, streamExistenceFilter,
streamExistenceFilterInitializer, indexCheckpoint, additionalCommitChecks);
_allReader = new AllReader <TStreamId>(indexBackend, _indexCommitter, _streamNames, eventTypeNames);
}
public IndexWriter(IPublisher bus, ITableIndex tableIndex, IHasher hasher, IIndexCache indexCache, IIndexReader indexReader, bool additionalCommitChecks)
{
Ensure.NotNull(indexCache, "indexBackend");
Ensure.NotNull(indexReader, "indexReader");
Ensure.NotNull(bus, "bus");
Ensure.NotNull(tableIndex, "tableIndex");
Ensure.NotNull(hasher, "hasher");
_bus = bus;
_tableIndex = tableIndex;
_hasher = hasher;
_indexCache = indexCache;
_indexReader = indexReader;
_additionalCommitChecks = additionalCommitChecks;
}
public virtual void TestFixtureSetUp()
{
_publisher = new InMemoryBus("publisher");
_tfReader = new FakeInMemoryTfReader(RecordOffset);
_tableIndex = new FakeInMemoryTableIndex();
_readerPool = new ObjectPool <ITransactionFileReader>(
"ReadIndex readers pool", 5, 100,
() => _tfReader);
_indexBackend = new IndexBackend(_readerPool, 100000, 100000);
_indexReader = new IndexReader(_indexBackend, _tableIndex, new StreamMetadata(maxCount: 100000), 100, false);
_indexWriter = new IndexWriter(_indexBackend, _indexReader);
_indexCommitter = new IndexCommitter(_publisher, _indexBackend, _indexReader, _tableIndex, false);
WriteEvents();
}
public void Setup(){
given();
_indexDir = PathName;
_fakeReader = new TFReaderLease(new FakeReader());
_indexBackend = new FakeIndexBackend(_fakeReader);
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.Index32Bit, maxSize: _maxMemTableSize),
() => _fakeReader,
PTableVersions.Index32Bit,
maxSizeForMemory: _maxMemTableSize,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_indexReader = new IndexReader(_indexBackend, _tableIndex, new EventStore.Core.Data.StreamMetadata(), _hashCollisionReadLimit);
when();
//wait for the mem table to be dumped
System.Threading.Thread.Sleep(500);
}
public void Setup()
{
given();
_indexDir = PathName;
_fakeReader = new TFReaderLease(new FakeReader());
_indexBackend = new FakeIndexBackend(_fakeReader);
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV1, maxSize: _maxMemTableSize),
() => _fakeReader,
PTableVersions.IndexV1,
maxSizeForMemory: _maxMemTableSize,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_indexReader = new IndexReader(_indexBackend, _tableIndex, new EventStore.Core.Data.StreamMetadata(), _hashCollisionReadLimit);
when();
//wait for the mem table to be dumped
System.Threading.Thread.Sleep(500);
}
public ReadIndex(IPublisher bus,
ObjectPool<ITransactionFileReader> readerPool,
ITableIndex tableIndex,
int streamInfoCacheCapacity,
bool additionalCommitChecks,
int metastreamMaxCount,
int hashCollisionReadLimit)
{
Ensure.NotNull(bus, "bus");
Ensure.NotNull(readerPool, "readerPool");
Ensure.NotNull(tableIndex, "tableIndex");
Ensure.Nonnegative(streamInfoCacheCapacity, "streamInfoCacheCapacity");
Ensure.Positive(metastreamMaxCount, "metastreamMaxCount");
var metastreamMetadata = new StreamMetadata(maxCount: metastreamMaxCount);
_indexBackend = new IndexBackend(readerPool, streamInfoCacheCapacity, streamInfoCacheCapacity);
_indexReader = new IndexReader(_indexBackend, tableIndex, metastreamMetadata, hashCollisionReadLimit);
_indexWriter = new IndexWriter(_indexBackend, _indexReader);
_indexCommitter = new IndexCommitter(bus, _indexBackend, _indexReader, tableIndex, additionalCommitChecks);
_allReader = new AllReader(_indexBackend, _indexCommitter);
}
public ReadIndex(IPublisher bus,
ObjectPool<ITransactionFileReader> readerPool,
ITableIndex tableIndex,
IHasher hasher,
int streamInfoCacheCapacity,
bool additionalCommitChecks,
int metastreamMaxCount)
{
Ensure.NotNull(bus, "bus");
Ensure.NotNull(readerPool, "readerPool");
Ensure.NotNull(tableIndex, "tableIndex");
Ensure.NotNull(hasher, "hasher");
Ensure.Nonnegative(streamInfoCacheCapacity, "streamInfoCacheCapacity");
Ensure.Positive(metastreamMaxCount, "metastreamMaxCount");
var metastreamMetadata = new StreamMetadata(maxCount: metastreamMaxCount);
_indexCache = new IndexCache(readerPool, streamInfoCacheCapacity, streamInfoCacheCapacity);
_indexReader = new IndexReader(_indexCache, hasher, tableIndex, metastreamMetadata);
var writer = new IndexWriter(bus, tableIndex, hasher, _indexCache, _indexReader, additionalCommitChecks);
_indexWriter = writer;
_indexCommitter = writer;
_allReader = new AllReader(_indexCache);
}
public RepositoryCleaner(IIndexReader indexReader, ITetriSettings settings, ILogger <IRepositoryCleaner> logger)
{
_settings = settings;
_logger = logger;
_indexReader = indexReader;
}
private static void DoQuery(IIndexReader<int, string, string> index, string queryText, IInverter<string, string> inverter)
{
Console.Out.WriteLine("=== graph for " + queryText + "===");
var results = index.ExecuteQuery(queryText, inverter);
Console.Out.WriteLine("=== results for: " + queryText + " ===");
foreach (var result in results)
{
Console.Out.WriteLine("doc: {0}, text: {1}", result.Id, result.Value);
}
}
public IndexWriter(IPublisher bus, ITableIndex tableIndex, IHasher hasher, IIndexCache indexCache, IIndexReader indexReader, bool additionalCommitChecks)
{
Ensure.NotNull(indexCache, "indexBackend");
Ensure.NotNull(indexReader, "indexReader");
Ensure.NotNull(bus, "bus");
Ensure.NotNull(tableIndex, "tableIndex");
Ensure.NotNull(hasher, "hasher");
_bus = bus;
_tableIndex = tableIndex;
_hasher = hasher;
_indexCache = indexCache;
_indexReader = indexReader;
_additionalCommitChecks = additionalCommitChecks;
}
/// <summary>
///
/// </summary>
/// <param name="packageService"></param>
/// <param name="settings"></param>
/// <param name="indexService"></param>
/// <param name="log"></param>
public CleanController(IRepositoryCleaner repositoryCleaner, IIndexReader indexService)
{
_indexService = indexService;
_repositoryCleaner = repositoryCleaner;
}
public static IndexReadStreamResult ReadStreamEventsBackward(this IIndexReader<string> index, string streamName, long fromEventNumber, int maxCount) => index.ReadStreamEventsBackward(streamName, streamName, fromEventNumber, maxCount);
public static IndexReadEventResult ReadEvent(this IIndexReader<string> index, string streamName, long eventNumber) => index.ReadEvent(streamName, streamName, eventNumber);
public void SetReader(IIndexReader <TStreamId> reader)
{
}
/// <summary>
///
/// </summary>
/// <param name="packageService"></param>
/// <param name="settings"></param>
/// <param name="indexService"></param>
/// <param name="log"></param>
public FilesController(IIndexReader indexService, ILogger <FilesController> log)
{
_indexService = indexService;
_log = log;
}
public StreamIdToNameFromStandardIndex(IIndexReader <StreamId> indexReader)
{
_indexReader = indexReader;
}
public HomeController(ITetriSettings settings, IIndexReader indexService, IPackageList packageList)
{
_settings = settings;
_indexService = indexService;
_packageList = packageList;
}
public EventTypeIdToNameFromStandardIndex(IIndexReader <uint> indexReader)
{
_indexReader = indexReader;
}
/// <summary>
///
/// </summary>
/// <param name="packageService"></param>
/// <param name="settings"></param>
/// <param name="indexService"></param>
/// <param name="log"></param>
public ArchivesController(IIndexReader indexService, ILogger <ArchivesController> log)
{
_indexService = indexService;
_log = log;
}
public Daemon(IRepositoryCleaner repositoryCleaner, IIndexReader indexService, ILogger <Daemon> log)
{
_indexService = indexService;
_repositoryCleaner = repositoryCleaner;
_log = log;
}