private void InitializeFromLog(INameExistenceFilter filter)
{
// if we have a checkpoint, start from that position in the log. this will work
// whether the checkpoint is the pre or post position of the last processed record.
var startPosition = filter.CurrentCheckpoint == -1 ? 0 : filter.CurrentCheckpoint;
Log.Information("Initializing from log starting at {startPosition:N0}", startPosition);
using var reader = _tfReaderFactory();
reader.Reposition(startPosition);
while (TryReadNextLogRecord(reader, out var result))
{
switch (result.LogRecord.RecordType)
{
case LogRecordType.Prepare:
// add regardless of expectedVersion because event 0 may be scavenged
// add regardless of committed or not because waiting for the commit is expensive
var prepare = (IPrepareLogRecord <string>)result.LogRecord;
filter.Add(prepare.EventStreamId);
filter.CurrentCheckpoint = result.RecordPostPosition;
break;
// no need to handle commits here, see comments in the prepare handling.
}
}
}
public void Initialize(INameExistenceFilter filter, long truncateToPosition)
{
int checkpoint = 0;
foreach (var name in _names)
{
filter.Add(name);
filter.CurrentCheckpoint = checkpoint++;
}
}
public void Confirm(IList <IPrepareLogRecord <string> > prepares, bool catchingUp, IIndexBackend <string> backend)
{
if (catchingUp)
{
// after the main index is caught up we will initialize the stream existence filter
return;
}
if (prepares.Count == 0)
{
return;
}
var lastPrepare = prepares[prepares.Count - 1];
if (prepares[0].ExpectedVersion == ExpectedVersion.NoStream)
{
_existenceFilter.Add(lastPrepare.EventStreamId);
}
_existenceFilter.CurrentCheckpoint = lastPrepare.LogPosition;
}
private void InitializeFromIndex(INameExistenceFilter filter)
{
if (filter.CurrentCheckpoint != -1L)
{
// can only use the index to build from scratch. if we have a checkpoint
// we need to build from the log in order to make use of it.
return;
}
Log.Information("Initializing from index");
// we have no checkpoint, build from the index. unfortunately there isn't
// a simple way to checkpoint in the middle of the index.
// keep track of the max position we see and use that as the checkpoint
// but only if we complete.
var checkpoint = -1L;
var enumerators = GetEnumerators();
var recordsTotal = enumerators.Sum(pair => pair.Count);
var recordsProcessed = 0L;
ulong?previousHash = null;
for (var t = 0; t < enumerators.Count; t++)
{
var pair = enumerators[t];
var enumerator = pair.Enumerator;
do
{
// enumerators are already advanced to first item
var entry = enumerator.Current;
checkpoint = Math.Max(checkpoint, entry.Position);
if (entry.Stream == previousHash)
{
continue;
}
// add regardless of version because event 0 may be scavenged
filter.Add(entry.Stream);
previousHash = entry.Stream;
} while (enumerator.MoveNext());
enumerator.Dispose();
recordsProcessed += pair.Count;
var percent = (double)recordsProcessed / recordsTotal * 100;
Log.Information("Stream Existence Filter initialization processed {tablesProcessed}/{tablesTotal} tables. {recordsProcessed:N0}/{recordsTotal:N0} log records ({percent:N2}%)",
t + 1, enumerators.Count, recordsProcessed, recordsTotal, percent);
}
// checkpoint at the end of the index.
filter.CurrentCheckpoint = checkpoint;
}
private void InitializeFromIndex(INameExistenceFilter filter)
{
if (filter.CurrentCheckpoint != -1L)
{
// can only use the index to build from scratch. if we have a checkpoint
// we need to build from the log in order to make use of it.
return;
}
Log.Information("Initializing from index");
// we have no checkpoint, build from the index. unfortunately there isn't
// a simple way to checkpoint in the middle of the index.
var tables = _tableIndex.IterateAllInOrder().ToList();
foreach (var table in tables)
{
if (table.Version == PTableVersions.IndexV1)
{
throw new NotSupportedException("The Stream Existence Filter is not supported with V1 index files. Please disable the filter by setting StreamExistenceFilterSize to 0, or rebuild the indexes.");
}
}
ulong?previousHash = null;
foreach (var table in tables)
{
foreach (var entry in table.IterateAllInOrder())
{
if (entry.Stream == previousHash)
{
continue;
}
// add regardless of version because event 0 may be scavenged
filter.Add(entry.Stream);
previousHash = entry.Stream;
}
}
// checkpoint at the end of the index.
if (previousHash != null)
{
filter.CurrentCheckpoint = _tableIndex.CommitCheckpoint;
}
}
public void Initialize(INameExistenceFilter filter)
{
if (!_streamNames.TryGetLastValue(out var sourceLastStreamId))
{
return;
}
var startStreamId = (uint)Math.Max(LogV3SystemStreams.FirstRealStream, filter.CurrentCheckpoint);
for (var streamId = startStreamId; streamId <= sourceLastStreamId; streamId += LogV3SystemStreams.StreamInterval)
{
if (!_streamNames.TryGetName(streamId, out var name))
{
throw new Exception($"NameExistenceFilter: this should never happen. could not find {streamId} in source");
}
filter.Add(name);
filter.CurrentCheckpoint = streamId;
}
}
public void Initialize(INameExistenceFilter filter, long truncateToPosition)
{
// todo: truncate if necessary. implementation will likely depend on how the indexes come out
if (!_streamNames.TryGetLastValue(out var sourceLastStreamId))
{
return;
}
var startStreamId = (uint)Math.Max(LogV3SystemStreams.FirstRealStream, filter.CurrentCheckpoint);
for (var streamId = startStreamId; streamId <= sourceLastStreamId; streamId += LogV3SystemStreams.StreamInterval)
{
if (!_streamNames.TryGetName(streamId, out var name))
{
throw new Exception($"NameExistenceFilter: this should never happen. could not find {streamId} in source");
}
filter.Add(name);
filter.CurrentCheckpoint = streamId;
}
}
