public override void Write(byte[] fromBuffer, int offset, int count)
{
if (count - offset < minDedupBlockSize)
{
inputStream.Write(fromBuffer, offset, count);
length += count - offset;
return;
}
#if DEBUG
sw.Start();
#endif
if (ddb.Contains(this.ChecksumToMatch, this.CurrentChunkName, this.CurrentChunkPos, count - offset, storageNodeId, ref dedupedBlockId))
{
this.DedupedCount++;
#if DEBUG
sw.Stop();
BenchmarkStats.Instance().DedupTime += sw.ElapsedMilliseconds;
sw.Reset();
#endif
}
// not duplicated, let's write to underlying stream
else
{
inputStream.Write(fromBuffer, offset, count);
length += count - offset;
}
dedupedBlocks.Add(dedupedBlockId);
}
public override void Write(byte[] fromBuffer, int offset, int count)
{
//Console.WriteLine("nullsink : write "+(count-offset));
#if DEBUG
sw.Start();
#endif
/*
* if(count >0){
* //Console.WriteLine("Write() : begin "+DateTime.Now);
* if( (count - offset) > (512*1024 - bufferPos)){
* RealWrite();
* }
* Array.Copy(fromBuffer, offset, buffer, bufferPos, count - offset);
* bufferPos += (count - offset);
* //innerStream.Write(fromBuffer, offset, count);
*
* }
* length += count-offset;*/
innerStream.Write(fromBuffer, offset, count);
length += (count - offset);
#if DEBUG
sw.Stop();
BenchmarkStats.Instance().SendTime += sw.ElapsedMilliseconds;
sw.Reset();
#endif
}
internal bool Contains(byte[] checksum, string chunkname, int posInChunk, int bufSize, uint storageNode, ref long dedupId)
{
#if DEBUG
BenchmarkStats.Instance().DedupLookups++;
#endif
for (int j = 0; j < index.Count; j++)
{
if (UnsafeCompare(index[currentPos].Checksum, checksum))
{
Interlocked.Increment(ref index[currentPos].RefCounts); // in case parallelism >1 and multiple accesses to the same checksum entry
#if DEBUG
if (j < (index.Count - currentPos) + 2)
{
BenchmarkStats.Instance().DedupHotFound++;
}
else
{
BenchmarkStats.Instance().DedupColdFound++;
}
#endif
dedupId = index[currentPos].ID;
j++; // move to next dedup entry, hoping that next request will match it
return(true);
}
if (currentPos == index.Count - 1)
{
currentPos = 0;
}
else
{
currentPos = j;
}
currentPos++;
}
// key not found, add it
FullDedupedBlock newDdb = new FullDedupedBlock();
newDdb.Checksum = new byte[20];
Array.Copy(checksum, newDdb.Checksum, checksum.Length);
newDdb.DataChunkName = chunkname;
newDdb.Length = bufSize;
newDdb.StartPos = posInChunk;
newDdb.StorageNodes[0] = storageNode;
newDdb.RefCounts = 1;
Interlocked.Increment(ref maxId);
newDdb.ID = maxId;
addPending.Add(newDdb);
Interlocked.Increment(ref pendingAddCount);
#if DEBUG
BenchmarkStats.Instance().DedupAdd++;
#endif
if (pendingAddCount > 2000)
{
MergePending();
}
dedupId = maxId;
return(false);
}
public override void Write(byte[] fromBuffer, int offset, int count){
#if DEBUG
sw.Start();
#endif
lastChecksum = md5.ComputeHash(fromBuffer, offset, count);
#if DEBUG
sw.Stop();
BenchmarkStats.Instance().ChecksumTime += sw.ElapsedMilliseconds;
sw.Reset();
#endif
//Console.WriteLine("ChecksummerStream:md5="+Convert.ToBase64String(lastChecksum));
inputStream.ChecksumToMatch = lastChecksum;
inputStream.Write(fromBuffer, offset, count);
length += count;
}
public override void Write(byte[] fromBuffer, int offset, int count)
{
if (count == 0)
{
return;
}
#if DEBUG
sw.Start();
#endif
lz.Write(fromBuffer, offset, count);
#if DEBUG
sw.Stop();
BenchmarkStats.Instance().CompressTime += sw.ElapsedMilliseconds;
sw.Reset();
#endif
}
public override void Write(byte[] fromBuffer, int offset, int count)
{
if (count < 1)
{
return;
}
// innerStream.Write(fromBuffer, offset, count);
#if DEBUG
sw.Start();
#endif
gz.Write(fromBuffer, offset, count);
//length += count-offset;
#if DEBUG
sw.Stop();
BenchmarkStats.Instance().CompressTime += sw.ElapsedMilliseconds;
sw.Reset();
#endif
}
/*private string ByteArrayToString(byte [] toConvert){
* StringBuilder sb = new StringBuilder(toConvert.Length);
* for (int i = 0; i < toConvert.Length - 1; i++){
* sb.Append(toConvert[i].ToString("X"));
* }
* return sb.ToString();
* }*/
public override void Write(byte[] fromBuffer, int offset, int count)
{
if ((count - offset) >= minBlockSize)
{
#if DEBUG
sw.Start();
#endif
Array.Copy(hasher.ComputeHash(fromBuffer, offset, count), lastChecksum, 16);
Array.Copy(BitConverter.GetBytes(count - offset), 0, lastChecksum, 16, 4);
#if DEBUG
sw.Stop();
BenchmarkStats.Instance().ChecksumTime += sw.ElapsedMilliseconds;
sw.Reset();
#endif
outputStream.ChecksumToMatch = lastChecksum;
}
outputStream.Write(fromBuffer, offset, count);
length += count;
}
public override void Write(byte[] fromBuffer, int offset, int count)
{
#if DEBUG
sw.Start();
#endif
data[dataPos] = 0x00;
Array.Copy(fromBuffer, offset, data, dataPos + 1, count);
previousChecksum = lastChecksum;
lastChecksum = hashAlgo.ComputeHash(data, dataPos, count - offset + 1);
if (!havePair)
{
dataPos = count - offset + 1;
havePair = true;
}
else
{
LeafCollection.Add(
new HashHolder(
IH(
previousChecksum, lastChecksum
)
)
);
havePair = false;
dataPos = 0;
}
#if DEBUG
sw.Stop();
BenchmarkStats.Instance().ChecksumTime += sw.ElapsedMilliseconds;
sw.Reset();
#endif
//Console.WriteLine("ChecksummerStream:md5="+Convert.ToBase64String(lastChecksum));
inputStream.ChecksumToMatch = lastChecksum;
inputStream.Write(fromBuffer, offset, count);
length += count;
}
public unsafe override void Write(byte[] fromBuffer, int offset, int count)
{
//Console.WriteLine ("called Write(), offset="+offset+", count="+count+", gatherpos="+currentGatherPos);
#if DEBUG
sw.Start();
#endif
// we do calculations in order to pack data by minblocksize sized arrays
// this increases performance by calling compress once for potentially multiple small/deduped blocks
// additionallt we get better compression ratios when compressing longer blocks
int realDataToCopy = count - offset;
if (realDataToCopy + currentGatherPos == minBlockSize) // perfect case, let's directly compress
//Console.WriteLine("data : perfect");
{
gatherBuffer = fromBuffer;
DoCompressAndWrite();
currentGatherPos = 0;
return;
}
if (realDataToCopy <= minBlockSize)
{
//try{
if (realDataToCopy + currentGatherPos > minBlockSize)
{
/*
* Array sourceArray, int sourceIndex, Array destinationArray, int destinationIndex, int length
*/
//Console.WriteLine("data : split ("+realDataToCopy+"). gather buffer size="+currentGatherPos+", remzining space="+(minBlockSize - currentGatherPos));
Array.Copy(fromBuffer, offset, gatherBuffer, currentGatherPos, minBlockSize - currentGatherPos);
DoCompressAndWrite();
int remainingData = realDataToCopy - (minBlockSize - currentGatherPos);
//Console.WriteLine("remaining="+remainingData);
currentGatherPos = 0;
//Console.WriteLine("copying "+remainingData+" bytes to gatherbuffer
Array.Copy(fromBuffer, realDataToCopy - (remainingData) /*+1*/, gatherBuffer, 0, remainingData /*+1*/);
currentGatherPos = remainingData;
//Console.WriteLine("data : added ("+remainingData+"). gather buffer size="+currentGatherPos);
}
else
{
try{
Array.Copy(fromBuffer, offset, gatherBuffer, currentGatherPos, realDataToCopy);
currentGatherPos += realDataToCopy /*+1*/;
}
catch (Exception) {
Console.WriteLine(" @@@@@@@@@@ orig buffer=" + fromBuffer.Length + ", offset=" + offset + ", current buf size=" + currentGatherPos + ", realdatatocopy=" + realDataToCopy);
}
//Console.WriteLine("data : insufficient ("+realDataToCopy+"). gather buffer size="+currentGatherPos);
}
/*}
* catch(Exception e){
* Console.WriteLine("error "+e.Message+"-----"+e.StackTrace);
* throw;
* }*/
}
else //count > minBlockSize, don't handle this case as we are supposed to coordinate between provided data and ProcessorStreams blocksize
{
Console.WriteLine("data : too much (" + realDataToCopy + "/" + minBlockSize + ")");
throw new NotSupportedException("Cannot compress data block bigger than " + minBlockSize + ", but got a " + count + " length data block.");
}
#if DEBUG
sw.Stop();
BenchmarkStats.Instance().CompressTime += sw.ElapsedMilliseconds;
sw.Reset();
#endif
}