/// <summary>
/// Retrieve a read-only stream over an object that has been stored.
/// </summary>
/// <param name="objectName">The name of the object.</param>
/// <param name="callbacks">CallbackMethods object containing callback methods.</param>
/// <param name="stream">Read-only stream.</param>
/// <returns>True if successful.</returns>
public bool RetrieveObjectStream(string objectName, CallbackMethods callbacks, out DedupeStream stream)
{
stream = null;
if (String.IsNullOrEmpty(objectName))
{
throw new ArgumentNullException(nameof(objectName));
}
if (callbacks == null)
{
throw new ArgumentNullException(nameof(callbacks));
}
if (callbacks.ReadChunk == null)
{
throw new ArgumentException("ReadChunk callback must be specified.");
}
objectName = DedupeCommon.SanitizeString(objectName);
if (!RetrieveObjectMetadata(objectName, out ObjectMetadata md))
{
return(false);
}
stream = new DedupeStream(md, _Database, callbacks);
return(true);
}
/// <summary>
/// Instantiates the object.
/// </summary>
/// <param name="key">The key of the chunk.</param>
/// <param name="len">The length of the chunk data.</param>
/// <param name="pos">The order of the chunk within the object.</param>
/// <param name="address">The address of the chunk within the current object.</param>
/// <param name="value">The byte data of the chunk.</param>
public Chunk(string key, long len, long pos, long address, byte[] value)
{
if (String.IsNullOrEmpty(key))
{
throw new ArgumentNullException(key);
}
if (len < 1)
{
throw new ArgumentOutOfRangeException(nameof(len));
}
if (pos < 0)
{
throw new ArgumentOutOfRangeException(nameof(pos));
}
if (address < 0)
{
throw new ArgumentOutOfRangeException(nameof(Address));
}
if (value == null || value.Length < 1)
{
throw new ArgumentNullException(nameof(value));
}
Key = DedupeCommon.SanitizeString(key);
Length = len;
Position = pos;
Address = address;
Value = value;
}
/// <summary>
/// Retrieve a read-only stream over an object that has been stored.
/// </summary>
/// <param name="key">The object key.</param>
/// <param name="callbacks">CallbackMethods object containing callback methods.</param>
/// <param name="data">Read-only stream.</param>
/// <returns>True if successful.</returns>
public bool TryGetStream(string key, DedupeCallbacks callbacks, out DedupeStream data)
{
if (String.IsNullOrEmpty(key))
{
throw new ArgumentNullException(nameof(key));
}
if (callbacks == null)
{
throw new ArgumentNullException(nameof(callbacks));
}
if (callbacks.ReadChunk == null)
{
throw new ArgumentException("ReadChunk callback must be specified.");
}
key = DedupeCommon.SanitizeString(key);
data = null;
try
{
data = GetStream(key, callbacks);
return(true);
}
catch (Exception)
{
return(false);
}
}
/// <summary>
/// Delete an object stored in the deduplication index.
/// This method will use the callbacks supplied in the method signature.
/// </summary>
/// <param name="objectName">The name of the object.</param>
/// <param name="callbacks">CallbackMethods object containing callback methods.</param>
/// <returns>True if successful.</returns>
public bool DeleteObject(string objectName, CallbackMethods callbacks)
{
if (String.IsNullOrEmpty(objectName))
{
throw new ArgumentNullException(nameof(objectName));
}
if (callbacks == null)
{
throw new ArgumentNullException(nameof(callbacks));
}
if (callbacks.DeleteChunk == null)
{
throw new ArgumentException("DeleteChunk callback must be specified.");
}
objectName = DedupeCommon.SanitizeString(objectName);
List <string> garbageCollectChunks = null;
lock (_ChunkLock)
{
_Database.DeleteObjectChunks(objectName, out garbageCollectChunks);
if (garbageCollectChunks != null && garbageCollectChunks.Count > 0)
{
foreach (string key in garbageCollectChunks)
{
if (!callbacks.DeleteChunk(key))
{
Log("Unable to delete chunk: " + key);
}
}
}
}
return(true);
}
/// <summary>
/// Retrieve metadata about an object from the deduplication index.
/// </summary>
/// <param name="objectName">The name of the object.</param>
/// <param name="includeChunks">Set to true to include metadata about associated chunks.</param>
/// <param name="md">Object metadata.</param>
/// <returns>True if successful.</returns>
public bool RetrieveObjectMetadata(string objectName, bool includeChunks, out ObjectMetadata md)
{
md = null;
if (String.IsNullOrEmpty(objectName))
{
throw new ArgumentNullException(nameof(objectName));
}
objectName = DedupeCommon.SanitizeString(objectName);
lock (_ChunkLock)
{
if (!_Database.GetObjectMetadata(objectName, out md))
{
return(false);
}
if (includeChunks)
{
md.Chunks = new List <Chunk>();
List <Chunk> chunks = new List <Chunk>();
if (!_Database.GetObjectChunks(objectName, out chunks))
{
return(false);
}
md.Chunks = new List <Chunk>(chunks);
}
return(true);
}
}
/// <summary>
/// Retrieve a read-only stream over an object that has been stored.
/// </summary>
/// <param name="key">The object key.</param>
/// <param name="callbacks">CallbackMethods object containing callback methods.</param>
/// <returns>Read-only stream.</returns>
public DedupeStream GetStream(string key, DedupeCallbacks callbacks)
{
if (String.IsNullOrEmpty(key))
{
throw new ArgumentNullException(nameof(key));
}
if (callbacks == null)
{
throw new ArgumentNullException(nameof(callbacks));
}
if (callbacks.ReadChunk == null)
{
throw new ArgumentException("ReadChunk callback must be specified.");
}
key = DedupeCommon.SanitizeString(key);
DedupeObject md = GetMetadata(key);
if (md == null)
{
throw new KeyNotFoundException("Object key '" + key + "' not found.");
}
return(new DedupeStream(md, _Database, callbacks));
}
/// <summary>
/// Delete an object stored in the deduplication index.
/// This method will use the callbacks supplied in the method signature.
/// </summary>
/// <param name="key">The object key.</param>
/// <param name="callbacks">CallbackMethods object containing callback methods.</param>
public void Delete(string key, DedupeCallbacks callbacks)
{
if (String.IsNullOrEmpty(key))
{
throw new ArgumentNullException(nameof(key));
}
if (callbacks == null)
{
throw new ArgumentNullException(nameof(callbacks));
}
if (callbacks.DeleteChunk == null)
{
throw new ArgumentException("DeleteChunk callback must be specified.");
}
key = DedupeCommon.SanitizeString(key);
List <string> garbageCollectChunks = _Database.Delete(key);
if (garbageCollectChunks != null && garbageCollectChunks.Count > 0)
{
foreach (string gcKey in garbageCollectChunks)
{
callbacks.DeleteChunk(gcKey);
}
}
}
/// <summary>
/// Retrieve an object from the deduplication index.
/// This method will use the callbacks supplied in the method signature.
/// </summary>
/// <param name="objectName">The name of the object.</param>
/// <param name="callbacks">CallbackMethods object containing callback methods.</param>
/// <param name="data">The byte data from the object.</param>
/// <returns>True if successful.</returns>
public bool RetrieveObject(string objectName, CallbackMethods callbacks, out byte[] data)
{
bool success = RetrieveObject(objectName, callbacks, out long contentLength, out Stream stream);
data = DedupeCommon.StreamToBytes(stream);
return(success);
}
/// <summary>
/// Store an object within a container in the deduplication index if it doesn't already exist, or, replace the object if it does.
/// This method will use the callbacks supplied in the method signature.
/// </summary>
/// <param name="objectName">The name of the object. Must be unique in the index.</param>
/// <param name="callbacks">CallbackMethods object containing callback methods.</param>
/// <param name="data">The byte data for the object.</param>
/// <param name="chunks">The list of chunks identified during the deduplication operation.</param>
/// <returns>True if successful.</returns>
public bool StoreOrReplaceObject(string objectName, CallbackMethods callbacks, byte[] data, out List <Chunk> chunks)
{
if (data == null || data.Length < 1)
{
throw new ArgumentNullException(nameof(data));
}
return(StoreOrReplaceObject(objectName, callbacks, data.Length, DedupeCommon.BytesToStream(data), out chunks));
}
/// <summary>
/// Retrieve metadata about an object from the deduplication index.
/// </summary>
/// <param name="key">The object key.</param>
/// <returns>Object metadata.</returns>
public DedupeObject GetMetadata(string key)
{
if (String.IsNullOrEmpty(key))
{
throw new ArgumentNullException(nameof(key));
}
key = DedupeCommon.SanitizeString(key);
return(_Database.GetObjectMetadata(key));
}
/// <summary>
/// Retrieve an object from the deduplication index.
/// This method will use the callbacks supplied in the method signature.
/// </summary>
/// <param name="objectName">The name of the object.</param>
/// <param name="callbacks">CallbackMethods object containing callback methods.</param>
/// <param name="contentLength">The length of the data.</param>
/// <param name="stream">The stream containing the data.</param>
/// <returns>True if successful.</returns>
public bool RetrieveObject(string objectName, CallbackMethods callbacks, out long contentLength, out Stream stream)
{
stream = null;
contentLength = 0;
if (String.IsNullOrEmpty(objectName))
{
throw new ArgumentNullException(nameof(objectName));
}
if (callbacks == null)
{
throw new ArgumentNullException(nameof(callbacks));
}
if (callbacks.ReadChunk == null)
{
throw new ArgumentException("ReadChunk callback must be specified.");
}
objectName = DedupeCommon.SanitizeString(objectName);
ObjectMetadata md = null;
lock (_ChunkLock)
{
if (!_Database.GetObjectMetadata(objectName, out md))
{
Log("Unable to retrieve object metadata for object " + objectName);
return(false);
}
if (md.Chunks == null || md.Chunks.Count < 1)
{
Log("No chunks returned");
return(false);
}
stream = new MemoryStream();
foreach (Chunk curr in md.Chunks)
{
byte[] chunkData = callbacks.ReadChunk(curr.Key);
if (chunkData == null || chunkData.Length < 1)
{
Log("Unable to read chunk " + curr.Key);
return(false);
}
stream.Write(chunkData, 0, chunkData.Length);
contentLength += chunkData.Length;
}
if (contentLength > 0)
{
stream.Seek(0, SeekOrigin.Begin);
}
}
return(true);
}
/// <summary>
/// Retrieve an object from the deduplication index.
/// </summary>
/// <param name="objectName">The name of the object.</param>
/// <param name="data">The byte data from the object.</param>
/// <returns>True if successful.</returns>
public bool RetrieveObject(string objectName, out byte[] data)
{
long contentLength = 0;
Stream stream = null;
bool success = RetrieveObject(objectName, Callbacks, out contentLength, out stream);
data = DedupeCommon.StreamToBytes(stream);
return(success);
}
/// <summary>
/// Store an object in the deduplication index.
/// </summary>
/// <param name="objectName">The name of the object. Must be unique in the index.</param>
/// <param name="data">The byte data for the object.</param>
/// <param name="chunks">The list of chunks identified during the deduplication operation.</param>
/// <returns>True if successful.</returns>
public bool StoreObject(string objectName, byte[] data, out List <Chunk> chunks)
{
//if (data == null || data.Length < 1) throw new ArgumentNullException(nameof(data));
if (data == null || data.Length < 1)
{
chunks = new List <Chunk>();
return(false);
}
return(StoreObject(objectName, Callbacks, data.Length, DedupeCommon.BytesToStream(data), out chunks));
}
/// <summary>
/// Store an object within a container in the deduplication index if it doesn't already exist, or, replace the object if it does.
/// This method will use the callbacks supplied in the method signature.
/// </summary>
/// <param name="objectName">The name of the object. Must be unique in the index.</param>
/// <param name="callbacks">CallbackMethods object containing callback methods.</param>
/// <param name="contentLength">The length of the data.</param>
/// <param name="stream">The stream containing the data.</param>
/// <param name="chunks">The list of chunks identified during the deduplication operation.</param>
/// <returns>True if successful.</returns>
public bool StoreOrReplaceObject(string objectName, CallbackMethods callbacks, long contentLength, Stream stream, out List <Chunk> chunks)
{
#region Initialize
chunks = new List <Chunk>();
if (String.IsNullOrEmpty(objectName))
{
throw new ArgumentNullException(nameof(objectName));
}
if (callbacks == null)
{
throw new ArgumentNullException(nameof(callbacks));
}
if (callbacks.WriteChunk == null)
{
throw new ArgumentException("WriteChunk callback must be specified.");
}
if (callbacks.DeleteChunk == null)
{
throw new ArgumentException("DeleteChunk callback must be specified.");
}
if (stream == null)
{
throw new ArgumentNullException(nameof(stream));
}
if (!stream.CanRead)
{
throw new ArgumentException("Cannot read from supplied stream.");
}
objectName = DedupeCommon.SanitizeString(objectName);
#endregion
#region Delete-if-Exists
if (_Database.ObjectExists(objectName))
{
Log("Object " + objectName + " already exists, deleting");
if (!DeleteObject(objectName))
{
Log("Unable to delete existing object");
return(false);
}
else
{
Log("Successfully deleted object for replacement");
}
}
#endregion
return(StoreObject(objectName, callbacks, contentLength, stream, out chunks));
}
/// <summary>
/// Write an object to the deduplication index if it doesn't already exist, or, replace the object if it does.
/// This method will use the callbacks supplied in the method signature.
/// </summary>
/// <param name="key">The object key. Must be unique in the index.</param>
/// <param name="callbacks">CallbackMethods object containing callback methods.</param>
/// <param name="data">The byte data for the object.</param>
public void WriteOrReplace(string key, DedupeCallbacks callbacks, byte[] data)
{
if (String.IsNullOrEmpty(key))
{
throw new ArgumentNullException(nameof(key));
}
if (data == null || data.Length < 1)
{
throw new ArgumentNullException(nameof(data));
}
WriteOrReplace(key, callbacks, data.Length, DedupeCommon.BytesToStream(data));
}
/// <summary>
/// Write an object to the deduplication index.
/// This method will use the callbacks supplied in the method signature.
/// </summary>
/// <param name="key">The object key. Must be unique in the index.</param>
/// <param name="callbacks">CallbackMethods object containing callback methods.</param>
/// <param name="bytes">The byte data for the object.</param>
public void Write(string key, DedupeCallbacks callbacks, byte[] bytes)
{
if (String.IsNullOrEmpty(key))
{
throw new ArgumentNullException(nameof(key));
}
if (bytes == null || bytes.Length < 1)
{
throw new ArgumentNullException(nameof(bytes));
}
Write(key, callbacks, bytes.Length, DedupeCommon.BytesToStream(bytes));
}
/// <summary>
/// Write an object to the deduplication index.
/// This method will use the callbacks supplied in the method signature.
/// </summary>
/// <param name="key">The object key. Must be unique in the index.</param>
/// <param name="callbacks">CallbackMethods object containing callback methods.</param>
/// <param name="data">The string data for the object.</param>
public void Write(string key, DedupeCallbacks callbacks, string data)
{
if (String.IsNullOrEmpty(key))
{
throw new ArgumentNullException(nameof(key));
}
if (String.IsNullOrEmpty(data))
{
throw new ArgumentNullException(nameof(data));
}
byte[] bytes = Encoding.UTF8.GetBytes(data);
Write(key, callbacks, bytes.Length, DedupeCommon.BytesToStream(bytes));
}
/// <summary>
/// Retrieve an object from the deduplication index.
/// This method will use the callbacks supplied in the method signature.
/// </summary>
/// <param name="key">The object key.</param>
/// <param name="callbacks">CallbackMethods object containing callback methods.</param>
/// <returns>Object data.</returns>
public DedupeObject Get(string key, DedupeCallbacks callbacks)
{
if (String.IsNullOrEmpty(key))
{
throw new ArgumentNullException(nameof(key));
}
if (callbacks == null)
{
throw new ArgumentNullException(nameof(callbacks));
}
if (callbacks.ReadChunk == null)
{
throw new ArgumentException("ReadChunk callback must be specified.");
}
key = DedupeCommon.SanitizeString(key);
DedupeObject md = _Database.GetObjectMetadata(key);
if (md == null)
{
throw new KeyNotFoundException("Object key '" + key + "' not found.");
}
if (md.Chunks == null || md.Chunks.Count < 1)
{
throw new IOException("No chunks returned for object key '" + key + "'.");
}
MemoryStream stream = new MemoryStream();
long contentLength = 0;
foreach (DedupeObjectMap curr in md.ObjectMap)
{
byte[] chunkData = callbacks.ReadChunk(curr.ChunkKey);
if (chunkData == null || chunkData.Length < 1)
{
throw new IOException("Unable to read chunk '" + curr.ChunkKey + "'.");
}
stream.Write(chunkData, 0, chunkData.Length);
contentLength += chunkData.Length;
}
if (contentLength > 0)
{
stream.Seek(0, SeekOrigin.Begin);
}
md.DataStream = stream;
return(md);
}
/// <summary>
/// Retrieve metadata about an object from the deduplication index.
/// </summary>
/// <param name="objectName">The name of the object.</param>
/// <param name="md">Object metadata.</param>
/// <returns>True if successful.</returns>
public bool RetrieveObjectMetadata(string objectName, out ObjectMetadata md)
{
md = null;
if (String.IsNullOrEmpty(objectName))
{
throw new ArgumentNullException(nameof(objectName));
}
objectName = DedupeCommon.SanitizeString(objectName);
lock (_ChunkLock)
{
return(_Database.GetObjectMetadata(objectName, out md));
}
}
/// <summary>
/// Retrieve an object from the deduplication index.
/// </summary>
/// <param name="objectName">The name of the object.</param>
/// <param name="data">The byte data from the object.</param>
/// <returns>True if successful.</returns>
public bool RetrieveObject(string objectName, out byte[] data)
{
bool success = RetrieveObject(objectName, Callbacks, out long contentLength, out Stream stream);
if (stream.Length == 0)
{
data = new byte[0];
return(false);
}
else
{
data = DedupeCommon.StreamToBytes(stream);
return(success);
}
}
/// <summary>
/// Retrieve a read-only stream over an object that has been stored.
/// </summary>
/// <param name="key">The object key.</param>
/// <param name="data">Read-only stream.</param>
/// <returns>True if successful.</returns>
public bool TryGetStream(string key, out DedupeStream data)
{
if (String.IsNullOrEmpty(key))
{
throw new ArgumentNullException(nameof(key));
}
key = DedupeCommon.SanitizeString(key);
data = null;
try
{
data = GetStream(key, Callbacks);
return(true);
}
catch (Exception)
{
return(false);
}
}
/// <summary>
/// Initialize a new or existing index using an internal Sqlite database.
/// </summary>
/// <param name="indexFile">Path and filename.</param>
/// <param name="settings">Deduplication settings.</param>
/// <param name="callbacks">Object containing callback functions for writing, reading, and deleting chunks.</param>
public DedupeLibrary(string indexFile, DedupeSettings settings, DedupeCallbacks callbacks)
{
if (String.IsNullOrEmpty(indexFile))
{
throw new ArgumentNullException(nameof(indexFile));
}
if (settings == null)
{
throw new ArgumentNullException(nameof(settings));
}
if (callbacks == null)
{
throw new ArgumentNullException(nameof(callbacks));
}
_Settings = settings;
_Callbacks = callbacks;
_IndexFile = DedupeCommon.SanitizeString(indexFile);
_Database = new SqliteProvider(_IndexFile);
InitializeIndex();
}
/// <summary>
/// Initialize an existing index using an internal Sqlite database.
/// </summary>
/// <param name="indexFile">Path and filename.</param>
/// <param name="writeChunkMethod">Method to call to write a chunk to storage.</param>
/// <param name="readChunkMethod">Method to call to read a chunk from storage.</param>
/// <param name="deleteChunkMethod">Method to call to delete a chunk from storage.</param>
/// <param name="debugDedupe">Enable console logging for deduplication operations.</param>
/// <param name="debugSql">Enable console logging for SQL operations.</param>
public DedupeLibrary(string indexFile, Func <Chunk, bool> writeChunkMethod, Func <string, byte[]> readChunkMethod, Func <string, bool> deleteChunkMethod, bool debugDedupe, bool debugSql)
{
if (String.IsNullOrEmpty(indexFile))
{
throw new ArgumentNullException(nameof(indexFile));
}
if (!File.Exists(indexFile))
{
throw new FileNotFoundException("Index file not found.");
}
if (writeChunkMethod == null)
{
throw new ArgumentNullException(nameof(writeChunkMethod));
}
if (readChunkMethod == null)
{
throw new ArgumentNullException(nameof(readChunkMethod));
}
if (deleteChunkMethod == null)
{
throw new ArgumentNullException(nameof(deleteChunkMethod));
}
_IndexFile = DedupeCommon.SanitizeString(indexFile);
Callbacks = new CallbackMethods
{
WriteChunk = writeChunkMethod,
ReadChunk = readChunkMethod,
DeleteChunk = deleteChunkMethod
};
DebugDedupe = debugDedupe;
DebugSql = debugSql;
_ChunkLock = new object();
_Database = new SqliteProvider(_IndexFile, DebugSql);
InitFromExistingIndex();
}
/// <summary>
/// Write an object to the deduplication index if it doesn't already exist, or, replace the object if it does.
/// This method will use the callbacks supplied in the method signature.
/// </summary>
/// <param name="key">The object key. Must be unique in the index.</param>
/// <param name="callbacks">CallbackMethods object containing callback methods.</param>
/// <param name="contentLength">The length of the data.</param>
/// <param name="stream">The stream containing the data.</param>
public void WriteOrReplace(string key, DedupeCallbacks callbacks, long contentLength, Stream stream)
{
if (String.IsNullOrEmpty(key))
{
throw new ArgumentNullException(nameof(key));
}
if (callbacks == null)
{
throw new ArgumentNullException(nameof(callbacks));
}
if (callbacks.WriteChunk == null)
{
throw new ArgumentException("WriteChunk callback must be specified.");
}
if (callbacks.DeleteChunk == null)
{
throw new ArgumentException("DeleteChunk callback must be specified.");
}
if (stream == null)
{
throw new ArgumentNullException(nameof(stream));
}
if (!stream.CanRead)
{
throw new ArgumentException("Cannot read from the supplied stream.");
}
key = DedupeCommon.SanitizeString(key);
if (_Database.Exists(key))
{
Logger?.Invoke(_Header + "Object " + key + " already exists, deleting");
Delete(key);
}
Write(key, callbacks, contentLength, stream);
}
private bool ChunkStream(long contentLength, Stream stream, Func <Chunk, bool> processChunk, out List <Chunk> chunks)
{
#region Initialize
chunks = new List <Chunk>();
Chunk chunk = null;
long bytesRead = 0;
string key = null;
if (stream == null || !stream.CanRead || contentLength < 1)
{
return(false);
}
#endregion
#region Single-Chunk
if (contentLength <= _MinChunkSize)
{
byte[] chunkData = DedupeCommon.ReadBytesFromStream(stream, contentLength, out bytesRead);
key = DedupeCommon.BytesToBase64(DedupeCommon.Sha256(chunkData));
chunk = new Chunk(
key,
contentLength,
0,
0,
chunkData);
chunks.Add(chunk);
return(processChunk(chunk));
}
#endregion
#region Process-Sliding-Window
Streams streamWindow = new Streams(stream, contentLength, _MinChunkSize, _ShiftCount);
byte[] currChunk = null;
long chunkPosition = 0; // should only be set at the beginning of a new chunk
while (true)
{
byte[] window = streamWindow.GetNextChunk(out long tempPosition, out byte[] newData, out bool finalChunk);
if (window == null)
{
return(true);
}
if (currChunk == null)
{
chunkPosition = tempPosition;
}
if (currChunk == null)
{
// starting a new chunk
currChunk = new byte[window.Length];
Buffer.BlockCopy(window, 0, currChunk, 0, window.Length);
}
else
{
// append new data
currChunk = DedupeCommon.AppendBytes(currChunk, newData);
}
byte[] md5Hash = DedupeCommon.Md5(window);
if (DedupeCommon.IsZeroBytes(md5Hash, _BoundaryCheckBytes)
||
(currChunk.Length >= _MaxChunkSize))
{
#region Chunk-Boundary
key = DedupeCommon.BytesToBase64(DedupeCommon.Sha256(currChunk));
chunk = new Chunk(
key,
currChunk.Length,
chunks.Count,
chunkPosition,
currChunk);
if (!processChunk(chunk))
{
return(false);
}
chunk.Value = null;
chunks.Add(chunk);
chunk = null;
currChunk = null;
streamWindow.AdvanceToNewChunk();
#endregion
}
else
{
// do nothing, continue;
}
if (finalChunk)
{
#region Final-Chunk
if (currChunk != null)
{
key = DedupeCommon.BytesToBase64(DedupeCommon.Sha256(currChunk));
chunk = new Chunk(
key,
currChunk.Length,
chunks.Count,
chunkPosition,
currChunk);
if (!processChunk(chunk))
{
return(false);
}
chunk.Value = null;
chunks.Add(chunk);
chunk = null;
currChunk = null;
break;
}
#endregion
}
}
#endregion
return(true);
}
private List <DedupeChunk> ChunkStream(string key, long contentLength, Stream stream, Action <DedupeChunk, DedupeObjectMap> processChunk)
{
if (String.IsNullOrEmpty(key))
{
throw new ArgumentNullException(nameof(key));
}
if (contentLength < 1)
{
throw new ArgumentException("Content length must be greater than zero.");
}
if (stream == null)
{
throw new ArgumentNullException(nameof(stream));
}
if (!stream.CanRead)
{
throw new ArgumentException("Cannot read from supplied stream.");
}
if (processChunk == null)
{
throw new ArgumentNullException(nameof(processChunk));
}
#region Initialize
List <DedupeChunk> chunks = new List <DedupeChunk>();
DedupeObjectMap map = null;
DedupeChunk chunk = null;
long bytesRead = 0;
string chunkKey = null;
#endregion
if (contentLength <= _Settings.MinChunkSize)
{
#region Single-Chunk
byte[] chunkData = DedupeCommon.ReadBytesFromStream(stream, contentLength, out bytesRead);
chunkKey = DedupeCommon.BytesToBase64(DedupeCommon.Sha256(chunkData));
chunk = new DedupeChunk(chunkKey, chunkData.Length, 1, chunkData);
chunks.Add(chunk);
map = new DedupeObjectMap(key, chunkKey, chunk.Length, 0, 0);
processChunk(chunk, map);
return(chunks);
#endregion
}
else
{
#region Sliding-Window
Streams streamWindow = new Streams(stream, contentLength, _Settings.MinChunkSize, _Settings.ShiftCount);
byte[] chunkData = null;
long chunkAddress = 0; // should only be set at the beginning of a new chunk
while (true)
{
byte[] newData = null;
bool finalChunk = false;
long tempPosition = 0;
byte[] window = streamWindow.GetNextChunk(out tempPosition, out newData, out finalChunk);
if (window == null)
{
return(chunks);
}
if (chunkData == null)
{
chunkAddress = tempPosition;
}
if (chunkData == null)
{
// starting a new chunk
chunkData = new byte[window.Length];
Buffer.BlockCopy(window, 0, chunkData, 0, window.Length);
}
else
{
// append new data
chunkData = DedupeCommon.AppendBytes(chunkData, newData);
}
byte[] md5Hash = DedupeCommon.Md5(window);
if (DedupeCommon.IsZeroBytes(md5Hash, _Settings.BoundaryCheckBytes) ||
chunkData.Length >= _Settings.MaxChunkSize)
{
#region Chunk-Boundary
chunkKey = DedupeCommon.BytesToBase64(DedupeCommon.Sha256(chunkData));
chunk = new DedupeChunk(chunkKey, chunkData.Length, 1, chunkData);
map = new DedupeObjectMap(key, chunk.Key, chunkData.Length, chunks.Count, chunkAddress);
processChunk(chunk, map);
chunk.Data = null;
chunks.Add(chunk);
chunk = null;
chunkData = null;
streamWindow.AdvanceToNewChunk();
#endregion
}
else
{
// do nothing, continue;
}
if (finalChunk)
{
#region Final-Chunk
if (chunkData != null)
{
chunkKey = DedupeCommon.BytesToBase64(DedupeCommon.Sha256(chunkData));
chunk = new DedupeChunk(chunkKey, chunkData.Length, 1, chunkData);
map = new DedupeObjectMap(key, chunk.Key, chunk.Length, chunks.Count, chunkAddress);
processChunk(chunk, map);
chunk.Data = null;
chunks.Add(chunk);
break;
}
#endregion
}
}
#endregion
}
return(chunks);
}
/// <summary>
/// Store an object in the deduplication index.
/// This method will use the callbacks supplied in the method signature.
/// </summary>
/// <param name="objectName">The name of the object. Must be unique in the index.</param>
/// <param name="callbacks">CallbackMethods object containing callback methods.</param>
/// <param name="contentLength">The length of the data.</param>
/// <param name="stream">The stream containing the data.</param>
/// <param name="chunks">The list of chunks identified during the deduplication operation.</param>
/// <returns>True if successful.</returns>
public bool StoreObject(string objectName, CallbackMethods callbacks, long contentLength, Stream stream, out List <Chunk> chunks)
{
#region Initialize
chunks = new List <Chunk>();
if (String.IsNullOrEmpty(objectName))
{
throw new ArgumentNullException(nameof(objectName));
}
if (callbacks == null)
{
throw new ArgumentNullException(nameof(callbacks));
}
if (callbacks.WriteChunk == null)
{
throw new ArgumentException("WriteChunk callback must be specified.");
}
if (callbacks.DeleteChunk == null)
{
throw new ArgumentException("DeleteChunk callback must be specified.");
}
if (contentLength < 1)
{
throw new ArgumentException("Content length must be at least one byte.");
}
if (stream == null)
{
throw new ArgumentNullException(nameof(stream));
}
if (!stream.CanRead)
{
throw new ArgumentException("Cannot read from supplied stream.");
}
objectName = DedupeCommon.SanitizeString(objectName);
if (_Database.ObjectExists(objectName))
{
Log("Object " + objectName + " already exists");
return(false);
}
bool garbageCollectionRequired = false;
#endregion
#region Chunk-Data
try
{
Func <Chunk, bool> processChunk = delegate(Chunk chunk)
{
if (chunk == null)
{
return(false);
}
lock (_ChunkLock)
{
if (!_Database.AddObjectChunk(objectName, contentLength, chunk))
{
Log("Unable to add chunk key " + chunk.Key);
garbageCollectionRequired = true;
return(false);
}
if (!callbacks.WriteChunk(chunk))
{
Log("Unable to write chunk key " + chunk.Key);
garbageCollectionRequired = true;
return(false);
}
}
return(true);
};
if (!ChunkStream(contentLength, stream, processChunk, out chunks))
{
Log("Unable to chunk object " + objectName);
garbageCollectionRequired = true;
return(false);
}
}
finally
{
if (garbageCollectionRequired)
{
List <string> garbageCollectKeys = new List <string>();
_Database.DeleteObjectChunks(objectName, out garbageCollectKeys);
if (garbageCollectKeys != null && garbageCollectKeys.Count > 0)
{
foreach (string key in garbageCollectKeys)
{
if (!callbacks.DeleteChunk(key))
{
Log("Unable to garbage collect chunk " + key);
}
}
}
}
}
#endregion
return(true);
}
/// <summary>
/// Create a new index using an internal Sqlite database.
/// </summary>
/// <param name="indexFile">Path and filename.</param>
/// <param name="minChunkSize">Minimum chunk size, must be divisible by 8, divisible by 64, and 128 or greater.</param>
/// <param name="maxChunkSize">Maximum chunk size, must be divisible by 8, divisible by 64, and at least 8 times larger than minimum chunk size.</param>
/// <param name="shiftCount">Number of bytes to shift while identifying chunk boundaries, must be less than or equal to minimum chunk size.</param>
/// <param name="boundaryCheckBytes">Number of bytes to examine while checking for a chunk boundary, must be 8 or fewer.</param>
/// <param name="writeChunkMethod">Method to call to write a chunk to storage.</param>
/// <param name="readChunkMethod">Method to call to read a chunk from storage.</param>
/// <param name="deleteChunkMethod">Method to call to delete a chunk from storage.</param>
/// <param name="debugDedupe">Enable console logging for deduplication operations.</param>
/// <param name="debugSql">Enable console logging for SQL operations.</param>
public DedupeLibrary(
string indexFile,
int minChunkSize,
int maxChunkSize,
int shiftCount,
int boundaryCheckBytes,
Func <Chunk, bool> writeChunkMethod,
Func <string, byte[]> readChunkMethod,
Func <string, bool> deleteChunkMethod,
bool debugDedupe,
bool debugSql)
{
if (String.IsNullOrEmpty(indexFile))
{
throw new ArgumentNullException(nameof(indexFile));
}
if (minChunkSize % 8 != 0)
{
throw new ArgumentException("Value for minChunkSize must be evenly divisible by 8.");
}
if (maxChunkSize % 8 != 0)
{
throw new ArgumentException("Value for maxChunkSize must be evenly divisible by 8.");
}
if (minChunkSize % 64 != 0)
{
throw new ArgumentException("Value for minChunkSize must be evenly divisible by 64.");
}
if (maxChunkSize % 64 != 0)
{
throw new ArgumentException("Value for maxChunkSize must be evenly divisible by 64.");
}
if (minChunkSize < 64)
{
throw new ArgumentOutOfRangeException("Value for minChunkSize must be 256 or greater.");
}
if (maxChunkSize <= minChunkSize)
{
throw new ArgumentOutOfRangeException("Value for maxChunkSize must be greater than minChunkSize and " + (8 * minChunkSize) + " or less.");
}
if (maxChunkSize < (8 * minChunkSize))
{
throw new ArgumentOutOfRangeException("Value for maxChunkSize must be " + (8 * minChunkSize) + " or greater.");
}
if (shiftCount > minChunkSize)
{
throw new ArgumentOutOfRangeException("Value for shiftCount must be less than or equal to minChunkSize.");
}
if (writeChunkMethod == null)
{
throw new ArgumentNullException(nameof(writeChunkMethod));
}
if (readChunkMethod == null)
{
throw new ArgumentNullException(nameof(readChunkMethod));
}
if (deleteChunkMethod == null)
{
throw new ArgumentNullException(nameof(deleteChunkMethod));
}
if (boundaryCheckBytes < 1 || boundaryCheckBytes > 8)
{
throw new ArgumentNullException(nameof(boundaryCheckBytes));
}
if (File.Exists(indexFile))
{
throw new IOException("Index file already exists.");
}
_IndexFile = DedupeCommon.SanitizeString(indexFile);
_MinChunkSize = minChunkSize;
_MaxChunkSize = maxChunkSize;
_ShiftCount = shiftCount;
_BoundaryCheckBytes = boundaryCheckBytes;
Callbacks = new CallbackMethods
{
WriteChunk = writeChunkMethod,
ReadChunk = readChunkMethod,
DeleteChunk = deleteChunkMethod
};
DebugDedupe = debugDedupe;
DebugSql = debugSql;
_ChunkLock = new object();
_Database = new SqliteProvider(_IndexFile, DebugSql);
InitNewIndex();
}
/// <summary>
/// Write an object to the deduplication index.
/// This method will use the callbacks supplied in the method signature.
/// </summary>
/// <param name="key">The object key. Must be unique in the index.</param>
/// <param name="callbacks">CallbackMethods object containing callback methods.</param>
/// <param name="contentLength">The length of the data.</param>
/// <param name="stream">The stream containing the data.</param>
public void Write(string key, DedupeCallbacks callbacks, long contentLength, Stream stream)
{
#region Initialize
if (String.IsNullOrEmpty(key))
{
throw new ArgumentNullException(nameof(key));
}
if (_Database.Exists(key))
{
throw new ArgumentException("An object with key '" + key + "' already exists.");
}
if (callbacks == null)
{
throw new ArgumentNullException(nameof(callbacks));
}
if (callbacks.WriteChunk == null)
{
throw new ArgumentException("WriteChunk callback must be specified.");
}
if (callbacks.DeleteChunk == null)
{
throw new ArgumentException("DeleteChunk callback must be specified.");
}
if (contentLength < 1)
{
throw new ArgumentException("Content length must be at least one byte.");
}
if (stream == null)
{
throw new ArgumentNullException(nameof(stream));
}
if (!stream.CanRead)
{
throw new InvalidOperationException("Cannot read from the supplied stream.");
}
key = DedupeCommon.SanitizeString(key);
bool garbageCollectionRequired = false;
#endregion
#region Chunk-Data
List <DedupeChunk> chunks = new List <DedupeChunk>();
try
{
Action <DedupeChunk, DedupeObjectMap> processChunk = delegate(DedupeChunk chunk, DedupeObjectMap map)
{
if (chunk == null || map == null)
{
return;
}
_Database.IncrementChunkRefcount(chunk.Key, chunk.Length);
_Database.AddObjectMap(key, chunk.Key, chunk.Length, map.ChunkPosition, map.ChunkAddress);
callbacks.WriteChunk(chunk);
};
chunks = ChunkStream(key, contentLength, stream, processChunk);
_Database.AddObject(key, contentLength);
}
finally
{
if (garbageCollectionRequired)
{
List <string> garbageCollectKeys = _Database.Delete(key);
if (garbageCollectKeys != null && garbageCollectKeys.Count > 0)
{
foreach (string gcKey in garbageCollectKeys)
{
callbacks.DeleteChunk(gcKey);
}
}
}
}
#endregion
}
