internal bool Contains(byte[] checksum, string chunkname, int posInChunk, int bufSize, uint storageNode, ref long dedupId)
{
#if DEBUG
BenchmarkStats.Instance().DedupLookups++;
#endif
for (int j = 0; j < index.Count; j++)
{
if (UnsafeCompare(index[currentPos].Checksum, checksum))
{
Interlocked.Increment(ref index[currentPos].RefCounts); // in case parallelism >1 and multiple accesses to the same checksum entry
#if DEBUG
if (j < (index.Count - currentPos) + 2)
{
BenchmarkStats.Instance().DedupHotFound++;
}
else
{
BenchmarkStats.Instance().DedupColdFound++;
}
#endif
dedupId = index[currentPos].ID;
j++; // move to next dedup entry, hoping that next request will match it
return(true);
}
if (currentPos == index.Count - 1)
{
currentPos = 0;
}
else
{
currentPos = j;
}
currentPos++;
}
// key not found, add it
FullDedupedBlock newDdb = new FullDedupedBlock();
newDdb.Checksum = new byte[20];
Array.Copy(checksum, newDdb.Checksum, checksum.Length);
newDdb.DataChunkName = chunkname;
newDdb.Length = bufSize;
newDdb.StartPos = posInChunk;
newDdb.StorageNodes[0] = storageNode;
newDdb.RefCounts = 1;
Interlocked.Increment(ref maxId);
newDdb.ID = maxId;
addPending.Add(newDdb);
Interlocked.Increment(ref pendingAddCount);
#if DEBUG
BenchmarkStats.Instance().DedupAdd++;
#endif
if (pendingAddCount > 2000)
{
MergePending();
}
dedupId = maxId;
return(false);
}
private IEnumerable <FullDedupedBlock> GetFullDbEnumerator()
{
using (FileStream fs = new FileStream(dedupDB, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)){
FullDedupedBlock curBlock = null;
while ((curBlock = Serializer.DeserializeWithLengthPrefix <FullDedupedBlock>(fs, PrefixStyle.Base128)) != null)
{
yield return(curBlock);
}
}
}
/// <summary>
/// Renames previous DB file and serializes current dedup blocks list.
/// To be called after a backup operation.
/// Also empties the index list to allow content to be GCed.
/// Counts & reports medium deduped block size and total deduped data
/// </summary>
private void Update()
{
//TODO : check if dedup index is used and exit
Logger.Append(Severity.DEBUG, "Saving deduplication indexes database...");
if (initializing)
{
Logger.Append(Severity.NOTICE, "Won't save deduplication DB, not initialized yet");
return;
}
try{ // delete if exists (should not happen)
File.Delete(dedupDB + ".new");
}
catch (FileNotFoundException) {}
using (FileStream newDbFs = new FileStream(dedupDB + ".new", FileMode.Create, FileAccess.Write)){
// dedup db metrics
long totalDataSize = 0;
long totalRefCount = 0;
using (IEnumerator <FullDedupedBlock> fullBlocksRef = GetFullDbEnumerator().GetEnumerator()){
lock (index){
foreach (LightDedupedBlock ddb in index)
{
if (fullBlocksRef.MoveNext())
{
FullDedupedBlock fdb = fullBlocksRef.Current;
fdb.RefCounts = ddb.RefCounts;
totalRefCount += fdb.RefCounts;
if (ddb.RefCounts == 0) // auto-clean db by pruning records not referenced anymore
{
continue;
}
totalDataSize += fdb.Length;
Serializer.SerializeWithLengthPrefix <FullDedupedBlock>(newDbFs, fdb, PrefixStyle.Base128);
}
}
Logger.Append(Severity.INFO, "Saved " + index.Count + " deduplication records (previously " + oldCount + ") into database. ");
if (index.Count > 0)
{
Console.WriteLine("Total data size represented by deduplication : " + totalDataSize / 1024 / 1024 + "MB, medium block size=" + totalDataSize / index.Count / 1024 + "KB, medium refcount=" + totalRefCount / index.Count);
}
index = null;
}
currentBackupSet = 0;
}
}
string ddbBackup = dedupDB.Replace(".idx", ".idx.bak");
try{ // remove previous backup
File.Delete(ddbBackup);
}
catch (FileNotFoundException) {}
try{
File.Move(dedupDB, ddbBackup);
}
catch (FileNotFoundException) { /*nothing wrong :first use of the dedup db*/ }
catch (Exception _e) {
Logger.Append(Severity.ERROR, "Could not move old dedup DB to backup location: " + _e.Message);
throw(_e);
}
try{
File.Move(dedupDB + ".new", dedupDB);
}
catch (Exception f) {
Logger.Append(Severity.ERROR, "Could not move new dedup DB to definitive location: " + f.Message);
throw(f);
}
}