private void Compact <T>(T functions, long untilAddress, VariableLengthStructSettings <Key, Value> variableLengthStructSettings) where T : IFunctions <Key, Value, Input, Output, Context> { var originalUntilAddress = untilAddress; var tempKv = new FasterKV <Key, Value, Input, Output, Context, T> (fht.IndexSize, functions, new LogSettings(), comparer: fht.Comparer, variableLengthStructSettings: variableLengthStructSettings); tempKv.StartSession(); int cnt = 0; using (var iter1 = fht.Log.Scan(fht.Log.BeginAddress, untilAddress)) { while (iter1.GetNext(out RecordInfo recordInfo)) { ref var key = ref iter1.GetKey(); ref var value = ref iter1.GetValue(); if (recordInfo.Tombstone) { tempKv.Delete(ref key, default(Context), 0); } else { tempKv.Upsert(ref key, ref value, default(Context), 0); } if (++cnt % 1000 == 0) { fht.Refresh(); tempKv.Refresh(); } }
private void LogScanForValidity(ref long untilAddress, ref long scanUntil, ref FasterKV <Key, Value, Input, Output, Context, LogCompactFunctions> tempKv) { while (scanUntil < fht.Log.SafeReadOnlyAddress) { untilAddress = scanUntil; scanUntil = fht.Log.SafeReadOnlyAddress; int cnt = 0; using (var iter2 = fht.Log.Scan(untilAddress, scanUntil)) { while (iter2.GetNext(out RecordInfo recordInfo, out Key key, out Value value)) { tempKv.Delete(ref key, default(Context), 0); if (++cnt % 1000 == 0) { fht.Refresh(); } } } fht.Refresh(); } }
/// <summary> /// Compact the log until specified address, moving active /// records to the tail of the log /// </summary> /// <param name="untilAddress"></param> public void Compact(long untilAddress) { long originalUntilAddress = untilAddress; var tempKv = new FasterKV <Key, Value, Input, Output, Context, LogCompactFunctions> (fht.IndexSize, new LogCompactFunctions(), new LogSettings(), comparer: fht.Comparer); tempKv.StartSession(); int cnt = 0; using (var iter1 = fht.Log.Scan(fht.Log.BeginAddress, untilAddress)) { while (iter1.GetNext(out RecordInfo recordInfo, out Key key, out Value value)) { if (recordInfo.Tombstone) { tempKv.Delete(ref key, default(Context), 0); } else { tempKv.Upsert(ref key, ref value, default(Context), 0); } if (++cnt % 1000 == 0) { fht.Refresh(); tempKv.Refresh(); } } } // TODO: Scan until SafeReadOnlyAddress long scanUntil = untilAddress; LogScanForValidity(ref untilAddress, ref scanUntil, ref tempKv); // Make sure key wasn't inserted between SafeReadOnlyAddress and TailAddress cnt = 0; using (var iter3 = tempKv.Log.Scan(tempKv.Log.BeginAddress, tempKv.Log.TailAddress)) { while (iter3.GetNext(out RecordInfo recordInfo, out Key key, out Value value)) { if (!recordInfo.Tombstone) { if (fht.ContainsKeyInMemory(ref key, scanUntil) == Status.NOTFOUND) { fht.Upsert(ref key, ref value, default(Context), 0); } } if (++cnt % 1000 == 0) { fht.Refresh(); tempKv.Refresh(); } if (scanUntil < fht.Log.SafeReadOnlyAddress) { LogScanForValidity(ref untilAddress, ref scanUntil, ref tempKv); } } } tempKv.StopSession(); tempKv.Dispose(); ShiftBeginAddress(originalUntilAddress); }