/// <summary>
/// Retrieve a read-only stream over an object that has been stored.
/// </summary>
/// <param name="key">The object key.</param>
/// <param name="callbacks">CallbackMethods object containing callback methods.</param>
/// <param name="data">Read-only stream.</param>
/// <returns>True if successful.</returns>
public bool TryGetStream(string key, DedupeCallbacks callbacks, out DedupeStream data)
{
if (String.IsNullOrEmpty(key))
{
throw new ArgumentNullException(nameof(key));
}
if (callbacks == null)
{
throw new ArgumentNullException(nameof(callbacks));
}
if (callbacks.ReadChunk == null)
{
throw new ArgumentException("ReadChunk callback must be specified.");
}
key = DedupeCommon.SanitizeString(key);
data = null;
try
{
data = GetStream(key, callbacks);
return(true);
}
catch (Exception)
{
return(false);
}
}
/// <summary>
/// Delete an object stored in the deduplication index.
/// This method will use the callbacks supplied in the method signature.
/// </summary>
/// <param name="key">The object key.</param>
/// <param name="callbacks">CallbackMethods object containing callback methods.</param>
public void Delete(string key, DedupeCallbacks callbacks)
{
if (String.IsNullOrEmpty(key))
{
throw new ArgumentNullException(nameof(key));
}
if (callbacks == null)
{
throw new ArgumentNullException(nameof(callbacks));
}
if (callbacks.DeleteChunk == null)
{
throw new ArgumentException("DeleteChunk callback must be specified.");
}
key = DedupeCommon.SanitizeString(key);
List <string> garbageCollectChunks = _Database.Delete(key);
if (garbageCollectChunks != null && garbageCollectChunks.Count > 0)
{
foreach (string gcKey in garbageCollectChunks)
{
callbacks.DeleteChunk(gcKey);
}
}
}
/// <summary>
/// Retrieve a read-only stream over an object that has been stored.
/// </summary>
/// <param name="key">The object key.</param>
/// <param name="callbacks">CallbackMethods object containing callback methods.</param>
/// <returns>Read-only stream.</returns>
public DedupeStream GetStream(string key, DedupeCallbacks callbacks)
{
if (String.IsNullOrEmpty(key))
{
throw new ArgumentNullException(nameof(key));
}
if (callbacks == null)
{
throw new ArgumentNullException(nameof(callbacks));
}
if (callbacks.ReadChunk == null)
{
throw new ArgumentException("ReadChunk callback must be specified.");
}
key = DedupeCommon.SanitizeString(key);
DedupeObject md = GetMetadata(key);
if (md == null)
{
throw new KeyNotFoundException("Object key '" + key + "' not found.");
}
return(new DedupeStream(md, _Database, callbacks));
}
/// <summary>
/// Write an object to the deduplication index if it doesn't already exist, or, replace the object if it does.
/// This method will use the callbacks supplied in the method signature.
/// </summary>
/// <param name="key">The object key. Must be unique in the index.</param>
/// <param name="callbacks">CallbackMethods object containing callback methods.</param>
/// <param name="data">The byte data for the object.</param>
public void WriteOrReplace(string key, DedupeCallbacks callbacks, byte[] data)
{
if (String.IsNullOrEmpty(key))
{
throw new ArgumentNullException(nameof(key));
}
if (data == null || data.Length < 1)
{
throw new ArgumentNullException(nameof(data));
}
WriteOrReplace(key, callbacks, data.Length, DedupeCommon.BytesToStream(data));
}
/// <summary>
/// Write an object to the deduplication index.
/// This method will use the callbacks supplied in the method signature.
/// </summary>
/// <param name="key">The object key. Must be unique in the index.</param>
/// <param name="callbacks">CallbackMethods object containing callback methods.</param>
/// <param name="bytes">The byte data for the object.</param>
public void Write(string key, DedupeCallbacks callbacks, byte[] bytes)
{
if (String.IsNullOrEmpty(key))
{
throw new ArgumentNullException(nameof(key));
}
if (bytes == null || bytes.Length < 1)
{
throw new ArgumentNullException(nameof(bytes));
}
Write(key, callbacks, bytes.Length, DedupeCommon.BytesToStream(bytes));
}
/// <summary>
/// Write an object to the deduplication index.
/// This method will use the callbacks supplied in the method signature.
/// </summary>
/// <param name="key">The object key. Must be unique in the index.</param>
/// <param name="callbacks">CallbackMethods object containing callback methods.</param>
/// <param name="data">The string data for the object.</param>
public void Write(string key, DedupeCallbacks callbacks, string data)
{
if (String.IsNullOrEmpty(key))
{
throw new ArgumentNullException(nameof(key));
}
if (String.IsNullOrEmpty(data))
{
throw new ArgumentNullException(nameof(data));
}
byte[] bytes = Encoding.UTF8.GetBytes(data);
Write(key, callbacks, bytes.Length, DedupeCommon.BytesToStream(bytes));
}
/// <summary>
/// Retrieve an object from the deduplication index.
/// This method will use the callbacks supplied in the method signature.
/// </summary>
/// <param name="key">The object key.</param>
/// <param name="callbacks">CallbackMethods object containing callback methods.</param>
/// <returns>Object data.</returns>
public DedupeObject Get(string key, DedupeCallbacks callbacks)
{
if (String.IsNullOrEmpty(key))
{
throw new ArgumentNullException(nameof(key));
}
if (callbacks == null)
{
throw new ArgumentNullException(nameof(callbacks));
}
if (callbacks.ReadChunk == null)
{
throw new ArgumentException("ReadChunk callback must be specified.");
}
key = DedupeCommon.SanitizeString(key);
DedupeObject md = _Database.GetObjectMetadata(key);
if (md == null)
{
throw new KeyNotFoundException("Object key '" + key + "' not found.");
}
if (md.Chunks == null || md.Chunks.Count < 1)
{
throw new IOException("No chunks returned for object key '" + key + "'.");
}
MemoryStream stream = new MemoryStream();
long contentLength = 0;
foreach (DedupeObjectMap curr in md.ObjectMap)
{
byte[] chunkData = callbacks.ReadChunk(curr.ChunkKey);
if (chunkData == null || chunkData.Length < 1)
{
throw new IOException("Unable to read chunk '" + curr.ChunkKey + "'.");
}
stream.Write(chunkData, 0, chunkData.Length);
contentLength += chunkData.Length;
}
if (contentLength > 0)
{
stream.Seek(0, SeekOrigin.Begin);
}
md.DataStream = stream;
return(md);
}
internal DedupeStream(DedupeObject md, DbProvider db, DedupeCallbacks callbacks)
{
if (md == null)
{
throw new ArgumentNullException(nameof(md));
}
if (db == null)
{
throw new ArgumentNullException(nameof(db));
}
if (callbacks == null)
{
throw new ArgumentNullException(nameof(callbacks));
}
_Metadata = md;
_Database = db;
_Callbacks = callbacks;
}
/// <summary>
/// Initialize a new or existing index using an internal Sqlite database.
/// </summary>
/// <param name="indexFile">Path and filename.</param>
/// <param name="settings">Deduplication settings.</param>
/// <param name="callbacks">Object containing callback functions for writing, reading, and deleting chunks.</param>
public DedupeLibrary(string indexFile, DedupeSettings settings, DedupeCallbacks callbacks)
{
if (String.IsNullOrEmpty(indexFile))
{
throw new ArgumentNullException(nameof(indexFile));
}
if (settings == null)
{
throw new ArgumentNullException(nameof(settings));
}
if (callbacks == null)
{
throw new ArgumentNullException(nameof(callbacks));
}
_Settings = settings;
_Callbacks = callbacks;
_IndexFile = DedupeCommon.SanitizeString(indexFile);
_Database = new SqliteProvider(_IndexFile);
InitializeIndex();
}
/// <summary>
/// Initialize an existing index using an external database. Tables must be created ahead of time.
/// </summary>
/// <param name="database">Database provider implemented using the Database.DbProvider class.</param>
/// <param name="settings">Deduplication settings.</param>
/// <param name="callbacks">Object containing callback functions for writing, reading, and deleting chunks.</param>
public DedupeLibrary(DbProvider database, DedupeSettings settings, DedupeCallbacks callbacks)
{
if (database == null)
{
throw new ArgumentNullException(nameof(database));
}
if (settings == null)
{
throw new ArgumentNullException(nameof(settings));
}
if (callbacks == null)
{
throw new ArgumentNullException(nameof(callbacks));
}
_Database = database;
_Settings = settings;
_Callbacks = callbacks;
InitializeIndex();
}
/// <summary>
/// Write an object to the deduplication index if it doesn't already exist, or, replace the object if it does.
/// This method will use the callbacks supplied in the method signature.
/// </summary>
/// <param name="key">The object key. Must be unique in the index.</param>
/// <param name="callbacks">CallbackMethods object containing callback methods.</param>
/// <param name="contentLength">The length of the data.</param>
/// <param name="stream">The stream containing the data.</param>
public void WriteOrReplace(string key, DedupeCallbacks callbacks, long contentLength, Stream stream)
{
if (String.IsNullOrEmpty(key))
{
throw new ArgumentNullException(nameof(key));
}
if (callbacks == null)
{
throw new ArgumentNullException(nameof(callbacks));
}
if (callbacks.WriteChunk == null)
{
throw new ArgumentException("WriteChunk callback must be specified.");
}
if (callbacks.DeleteChunk == null)
{
throw new ArgumentException("DeleteChunk callback must be specified.");
}
if (stream == null)
{
throw new ArgumentNullException(nameof(stream));
}
if (!stream.CanRead)
{
throw new ArgumentException("Cannot read from the supplied stream.");
}
key = DedupeCommon.SanitizeString(key);
if (_Database.Exists(key))
{
Logger?.Invoke(_Header + "Object " + key + " already exists, deleting");
Delete(key);
}
Write(key, callbacks, contentLength, stream);
}
/// <summary>
/// Write an object to the deduplication index.
/// This method will use the callbacks supplied in the method signature.
/// </summary>
/// <param name="key">The object key. Must be unique in the index.</param>
/// <param name="callbacks">CallbackMethods object containing callback methods.</param>
/// <param name="contentLength">The length of the data.</param>
/// <param name="stream">The stream containing the data.</param>
public void Write(string key, DedupeCallbacks callbacks, long contentLength, Stream stream)
{
#region Initialize
if (String.IsNullOrEmpty(key))
{
throw new ArgumentNullException(nameof(key));
}
if (_Database.Exists(key))
{
throw new ArgumentException("An object with key '" + key + "' already exists.");
}
if (callbacks == null)
{
throw new ArgumentNullException(nameof(callbacks));
}
if (callbacks.WriteChunk == null)
{
throw new ArgumentException("WriteChunk callback must be specified.");
}
if (callbacks.DeleteChunk == null)
{
throw new ArgumentException("DeleteChunk callback must be specified.");
}
if (contentLength < 1)
{
throw new ArgumentException("Content length must be at least one byte.");
}
if (stream == null)
{
throw new ArgumentNullException(nameof(stream));
}
if (!stream.CanRead)
{
throw new InvalidOperationException("Cannot read from the supplied stream.");
}
key = DedupeCommon.SanitizeString(key);
bool garbageCollectionRequired = false;
#endregion
#region Chunk-Data
List <DedupeChunk> chunks = new List <DedupeChunk>();
try
{
Action <DedupeChunk, DedupeObjectMap> processChunk = delegate(DedupeChunk chunk, DedupeObjectMap map)
{
if (chunk == null || map == null)
{
return;
}
_Database.IncrementChunkRefcount(chunk.Key, chunk.Length);
_Database.AddObjectMap(key, chunk.Key, chunk.Length, map.ChunkPosition, map.ChunkAddress);
callbacks.WriteChunk(chunk);
};
chunks = ChunkStream(key, contentLength, stream, processChunk);
_Database.AddObject(key, contentLength);
}
finally
{
if (garbageCollectionRequired)
{
List <string> garbageCollectKeys = _Database.Delete(key);
if (garbageCollectKeys != null && garbageCollectKeys.Count > 0)
{
foreach (string gcKey in garbageCollectKeys)
{
callbacks.DeleteChunk(gcKey);
}
}
}
}
#endregion
}