private List <DedupeChunk> ChunkStream(string key, long contentLength, Stream stream, Action <DedupeChunk, DedupeObjectMap> processChunk)
{
if (String.IsNullOrEmpty(key))
{
throw new ArgumentNullException(nameof(key));
}
if (contentLength < 1)
{
throw new ArgumentException("Content length must be greater than zero.");
}
if (stream == null)
{
throw new ArgumentNullException(nameof(stream));
}
if (!stream.CanRead)
{
throw new ArgumentException("Cannot read from supplied stream.");
}
if (processChunk == null)
{
throw new ArgumentNullException(nameof(processChunk));
}
#region Initialize
List <DedupeChunk> chunks = new List <DedupeChunk>();
DedupeObjectMap map = null;
DedupeChunk chunk = null;
long bytesRead = 0;
string chunkKey = null;
#endregion
if (contentLength <= _Settings.MinChunkSize)
{
#region Single-Chunk
byte[] chunkData = DedupeCommon.ReadBytesFromStream(stream, contentLength, out bytesRead);
chunkKey = DedupeCommon.BytesToBase64(DedupeCommon.Sha256(chunkData));
chunk = new DedupeChunk(chunkKey, chunkData.Length, 1, chunkData);
chunks.Add(chunk);
map = new DedupeObjectMap(key, chunkKey, chunk.Length, 0, 0);
processChunk(chunk, map);
return(chunks);
#endregion
}
else
{
#region Sliding-Window
Streams streamWindow = new Streams(stream, contentLength, _Settings.MinChunkSize, _Settings.ShiftCount);
byte[] chunkData = null;
long chunkAddress = 0; // should only be set at the beginning of a new chunk
while (true)
{
byte[] newData = null;
bool finalChunk = false;
long tempPosition = 0;
byte[] window = streamWindow.GetNextChunk(out tempPosition, out newData, out finalChunk);
if (window == null)
{
return(chunks);
}
if (chunkData == null)
{
chunkAddress = tempPosition;
}
if (chunkData == null)
{
// starting a new chunk
chunkData = new byte[window.Length];
Buffer.BlockCopy(window, 0, chunkData, 0, window.Length);
}
else
{
// append new data
chunkData = DedupeCommon.AppendBytes(chunkData, newData);
}
byte[] md5Hash = DedupeCommon.Md5(window);
if (DedupeCommon.IsZeroBytes(md5Hash, _Settings.BoundaryCheckBytes) ||
chunkData.Length >= _Settings.MaxChunkSize)
{
#region Chunk-Boundary
chunkKey = DedupeCommon.BytesToBase64(DedupeCommon.Sha256(chunkData));
chunk = new DedupeChunk(chunkKey, chunkData.Length, 1, chunkData);
map = new DedupeObjectMap(key, chunk.Key, chunkData.Length, chunks.Count, chunkAddress);
processChunk(chunk, map);
chunk.Data = null;
chunks.Add(chunk);
chunk = null;
chunkData = null;
streamWindow.AdvanceToNewChunk();
#endregion
}
else
{
// do nothing, continue;
}
if (finalChunk)
{
#region Final-Chunk
if (chunkData != null)
{
chunkKey = DedupeCommon.BytesToBase64(DedupeCommon.Sha256(chunkData));
chunk = new DedupeChunk(chunkKey, chunkData.Length, 1, chunkData);
map = new DedupeObjectMap(key, chunk.Key, chunk.Length, chunks.Count, chunkAddress);
processChunk(chunk, map);
chunk.Data = null;
chunks.Add(chunk);
break;
}
#endregion
}
}
#endregion
}
return(chunks);
}
private bool ChunkStream(long contentLength, Stream stream, Func <Chunk, bool> processChunk, out List <Chunk> chunks)
{
#region Initialize
chunks = new List <Chunk>();
Chunk chunk = null;
long bytesRead = 0;
string key = null;
if (stream == null || !stream.CanRead || contentLength < 1)
{
return(false);
}
#endregion
#region Single-Chunk
if (contentLength <= _MinChunkSize)
{
byte[] chunkData = DedupeCommon.ReadBytesFromStream(stream, contentLength, out bytesRead);
key = DedupeCommon.BytesToBase64(DedupeCommon.Sha256(chunkData));
chunk = new Chunk(
key,
contentLength,
0,
0,
chunkData);
chunks.Add(chunk);
return(processChunk(chunk));
}
#endregion
#region Process-Sliding-Window
Streams streamWindow = new Streams(stream, contentLength, _MinChunkSize, _ShiftCount);
byte[] currChunk = null;
long chunkPosition = 0; // should only be set at the beginning of a new chunk
while (true)
{
byte[] window = streamWindow.GetNextChunk(out long tempPosition, out byte[] newData, out bool finalChunk);
if (window == null)
{
return(true);
}
if (currChunk == null)
{
chunkPosition = tempPosition;
}
if (currChunk == null)
{
// starting a new chunk
currChunk = new byte[window.Length];
Buffer.BlockCopy(window, 0, currChunk, 0, window.Length);
}
else
{
// append new data
currChunk = DedupeCommon.AppendBytes(currChunk, newData);
}
byte[] md5Hash = DedupeCommon.Md5(window);
if (DedupeCommon.IsZeroBytes(md5Hash, _BoundaryCheckBytes)
||
(currChunk.Length >= _MaxChunkSize))
{
#region Chunk-Boundary
key = DedupeCommon.BytesToBase64(DedupeCommon.Sha256(currChunk));
chunk = new Chunk(
key,
currChunk.Length,
chunks.Count,
chunkPosition,
currChunk);
if (!processChunk(chunk))
{
return(false);
}
chunk.Value = null;
chunks.Add(chunk);
chunk = null;
currChunk = null;
streamWindow.AdvanceToNewChunk();
#endregion
}
else
{
// do nothing, continue;
}
if (finalChunk)
{
#region Final-Chunk
if (currChunk != null)
{
key = DedupeCommon.BytesToBase64(DedupeCommon.Sha256(currChunk));
chunk = new Chunk(
key,
currChunk.Length,
chunks.Count,
chunkPosition,
currChunk);
if (!processChunk(chunk))
{
return(false);
}
chunk.Value = null;
chunks.Add(chunk);
chunk = null;
currChunk = null;
break;
}
#endregion
}
}
#endregion
return(true);
}
