public NameExistenceFilterValueLookupDecorator(
IValueLookup <TValue> wrapped,
INameExistenceFilter existenceFilter)
{
_wrapped = wrapped;
_existenceFilter = existenceFilter;
}
private void InitializeFromLog(INameExistenceFilter filter)
{
// if we have a checkpoint, start from that position in the log. this will work
// whether the checkpoint is the pre or post position of the last processed record.
var startPosition = filter.CurrentCheckpoint == -1 ? 0 : filter.CurrentCheckpoint;
Log.Information("Initializing from log starting at {startPosition:N0}", startPosition);
using var reader = _tfReaderFactory();
reader.Reposition(startPosition);
while (TryReadNextLogRecord(reader, out var result))
{
switch (result.LogRecord.RecordType)
{
case LogRecordType.Prepare:
// add regardless of expectedVersion because event 0 may be scavenged
// add regardless of committed or not because waiting for the commit is expensive
var prepare = (IPrepareLogRecord <string>)result.LogRecord;
filter.Add(prepare.EventStreamId);
filter.CurrentCheckpoint = result.RecordPostPosition;
break;
// no need to handle commits here, see comments in the prepare handling.
}
}
}
public void Initialize(INameExistenceFilter filter, long truncateToPosition)
{
int checkpoint = 0;
foreach (var name in _names)
{
filter.Add(name);
filter.CurrentCheckpoint = checkpoint++;
}
}
public void Initialize(INameExistenceFilter filter, long truncateToPosition)
{
if (truncateToPosition < filter.CurrentCheckpoint)
{
filter.TruncateTo(checkpoint: truncateToPosition);
}
InitializeFromIndex(filter);
InitializeFromLog(filter);
}
public ReadIndex(IPublisher bus,
ObjectPool <ITransactionFileReader> readerPool,
ITableIndex <TStreamId> tableIndex,
INameIndexConfirmer <TStreamId> streamNameIndex,
IValueLookup <TStreamId> streamIds,
IStreamNamesProvider <TStreamId> streamNamesProvider,
TStreamId emptyStreamName,
IValidator <TStreamId> streamIdValidator,
ISizer <TStreamId> sizer,
INameExistenceFilter streamExistenceFilter,
IExistenceFilterReader <TStreamId> streamExistenceFilterReader,
INameIndexConfirmer <TStreamId> eventTypeIndex,
int streamInfoCacheCapacity,
bool additionalCommitChecks,
long metastreamMaxCount,
int hashCollisionReadLimit,
bool skipIndexScanOnReads,
IReadOnlyCheckpoint replicationCheckpoint,
ICheckpoint indexCheckpoint)
{
Ensure.NotNull(bus, "bus");
Ensure.NotNull(readerPool, "readerPool");
Ensure.NotNull(tableIndex, "tableIndex");
Ensure.NotNull(streamIds, nameof(streamIds));
Ensure.NotNull(streamNamesProvider, nameof(streamNamesProvider));
Ensure.NotNull(streamIdValidator, nameof(streamIdValidator));
Ensure.NotNull(sizer, nameof(sizer));
Ensure.NotNull(streamExistenceFilter, nameof(streamExistenceFilter));
Ensure.NotNull(streamExistenceFilterReader, nameof(streamExistenceFilterReader));
Ensure.Nonnegative(streamInfoCacheCapacity, "streamInfoCacheCapacity");
Ensure.Positive(metastreamMaxCount, "metastreamMaxCount");
Ensure.NotNull(replicationCheckpoint, "replicationCheckpoint");
Ensure.NotNull(indexCheckpoint, "indexCheckpoint");
var metastreamMetadata = new StreamMetadata(maxCount: metastreamMaxCount);
var indexBackend = new IndexBackend <TStreamId>(readerPool, streamInfoCacheCapacity, streamInfoCacheCapacity);
_indexReader = new IndexReader <TStreamId>(indexBackend, tableIndex, streamNamesProvider, streamIdValidator,
streamExistenceFilterReader, metastreamMetadata, hashCollisionReadLimit, skipIndexScanOnReads);
_streamIds = streamIds;
_streamNames = streamNamesProvider.StreamNames;
var systemStreams = streamNamesProvider.SystemStreams;
var eventTypeNames = streamNamesProvider.EventTypes;
var streamExistenceFilterInitializer = streamNamesProvider.StreamExistenceFilterInitializer;
_indexWriter = new IndexWriter <TStreamId>(indexBackend, _indexReader, _streamIds, _streamNames, systemStreams, emptyStreamName, sizer);
_indexCommitter = new IndexCommitter <TStreamId>(bus, indexBackend, _indexReader, tableIndex, streamNameIndex,
_streamNames, eventTypeIndex, eventTypeNames, systemStreams, streamExistenceFilter,
streamExistenceFilterInitializer, indexCheckpoint, additionalCommitChecks);
_allReader = new AllReader <TStreamId>(indexBackend, _indexCommitter, _streamNames, eventTypeNames);
}
private void InitializeFromIndex(INameExistenceFilter filter)
{
if (filter.CurrentCheckpoint != -1L)
{
// can only use the index to build from scratch. if we have a checkpoint
// we need to build from the log in order to make use of it.
return;
}
Log.Information("Initializing from index");
// we have no checkpoint, build from the index. unfortunately there isn't
// a simple way to checkpoint in the middle of the index.
// keep track of the max position we see and use that as the checkpoint
// but only if we complete.
var checkpoint = -1L;
var enumerators = GetEnumerators();
var recordsTotal = enumerators.Sum(pair => pair.Count);
var recordsProcessed = 0L;
ulong?previousHash = null;
for (var t = 0; t < enumerators.Count; t++)
{
var pair = enumerators[t];
var enumerator = pair.Enumerator;
do
{
// enumerators are already advanced to first item
var entry = enumerator.Current;
checkpoint = Math.Max(checkpoint, entry.Position);
if (entry.Stream == previousHash)
{
continue;
}
// add regardless of version because event 0 may be scavenged
filter.Add(entry.Stream);
previousHash = entry.Stream;
} while (enumerator.MoveNext());
enumerator.Dispose();
recordsProcessed += pair.Count;
var percent = (double)recordsProcessed / recordsTotal * 100;
Log.Information("Stream Existence Filter initialization processed {tablesProcessed}/{tablesTotal} tables. {recordsProcessed:N0}/{recordsTotal:N0} log records ({percent:N2}%)",
t + 1, enumerators.Count, recordsProcessed, recordsTotal, percent);
}
// checkpoint at the end of the index.
filter.CurrentCheckpoint = checkpoint;
}
private void InitializeFromIndex(INameExistenceFilter filter)
{
if (filter.CurrentCheckpoint != -1L)
{
// can only use the index to build from scratch. if we have a checkpoint
// we need to build from the log in order to make use of it.
return;
}
Log.Information("Initializing from index");
// we have no checkpoint, build from the index. unfortunately there isn't
// a simple way to checkpoint in the middle of the index.
var tables = _tableIndex.IterateAllInOrder().ToList();
foreach (var table in tables)
{
if (table.Version == PTableVersions.IndexV1)
{
throw new NotSupportedException("The Stream Existence Filter is not supported with V1 index files. Please disable the filter by setting StreamExistenceFilterSize to 0, or rebuild the indexes.");
}
}
ulong?previousHash = null;
foreach (var table in tables)
{
foreach (var entry in table.IterateAllInOrder())
{
if (entry.Stream == previousHash)
{
continue;
}
// add regardless of version because event 0 may be scavenged
filter.Add(entry.Stream);
previousHash = entry.Stream;
}
}
// checkpoint at the end of the index.
if (previousHash != null)
{
filter.CurrentCheckpoint = _tableIndex.CommitCheckpoint;
}
}
public void Initialize(INameExistenceFilter filter)
{
if (!_streamNames.TryGetLastValue(out var sourceLastStreamId))
{
return;
}
var startStreamId = (uint)Math.Max(LogV3SystemStreams.FirstRealStream, filter.CurrentCheckpoint);
for (var streamId = startStreamId; streamId <= sourceLastStreamId; streamId += LogV3SystemStreams.StreamInterval)
{
if (!_streamNames.TryGetName(streamId, out var name))
{
throw new Exception($"NameExistenceFilter: this should never happen. could not find {streamId} in source");
}
filter.Add(name);
filter.CurrentCheckpoint = streamId;
}
}
public void Initialize(INameExistenceFilter filter, long truncateToPosition)
{
// todo: truncate if necessary. implementation will likely depend on how the indexes come out
if (!_streamNames.TryGetLastValue(out var sourceLastStreamId))
{
return;
}
var startStreamId = (uint)Math.Max(LogV3SystemStreams.FirstRealStream, filter.CurrentCheckpoint);
for (var streamId = startStreamId; streamId <= sourceLastStreamId; streamId += LogV3SystemStreams.StreamInterval)
{
if (!_streamNames.TryGetName(streamId, out var name))
{
throw new Exception($"NameExistenceFilter: this should never happen. could not find {streamId} in source");
}
filter.Add(name);
filter.CurrentCheckpoint = streamId;
}
}
public void Initialize(INameExistenceFilter filter)
{
InitializeFromIndex(filter);
InitializeFromLog(filter);
}
public StreamExistenceFilterValidator(INameExistenceFilter wrapped)
{
_wrapped = wrapped;
}
public LogV2StreamNameIndex(INameExistenceFilter existenceFilter)
{
_existenceFilter = existenceFilter;
}
