/// <summary> /// Retrieve a read-only stream over an object that has been stored. /// </summary> /// <param name="key">The object key.</param> /// <param name="callbacks">CallbackMethods object containing callback methods.</param> /// <param name="data">Read-only stream.</param> /// <returns>True if successful.</returns> public bool TryGetStream(string key, DedupeCallbacks callbacks, out DedupeStream data) { if (String.IsNullOrEmpty(key)) { throw new ArgumentNullException(nameof(key)); } if (callbacks == null) { throw new ArgumentNullException(nameof(callbacks)); } if (callbacks.ReadChunk == null) { throw new ArgumentException("ReadChunk callback must be specified."); } key = DedupeCommon.SanitizeString(key); data = null; try { data = GetStream(key, callbacks); return(true); } catch (Exception) { return(false); } }
/// <summary> /// Delete an object stored in the deduplication index. /// This method will use the callbacks supplied in the method signature. /// </summary> /// <param name="key">The object key.</param> /// <param name="callbacks">CallbackMethods object containing callback methods.</param> public void Delete(string key, DedupeCallbacks callbacks) { if (String.IsNullOrEmpty(key)) { throw new ArgumentNullException(nameof(key)); } if (callbacks == null) { throw new ArgumentNullException(nameof(callbacks)); } if (callbacks.DeleteChunk == null) { throw new ArgumentException("DeleteChunk callback must be specified."); } key = DedupeCommon.SanitizeString(key); List <string> garbageCollectChunks = _Database.Delete(key); if (garbageCollectChunks != null && garbageCollectChunks.Count > 0) { foreach (string gcKey in garbageCollectChunks) { callbacks.DeleteChunk(gcKey); } } }
/// <summary> /// Retrieve a read-only stream over an object that has been stored. /// </summary> /// <param name="key">The object key.</param> /// <param name="callbacks">CallbackMethods object containing callback methods.</param> /// <returns>Read-only stream.</returns> public DedupeStream GetStream(string key, DedupeCallbacks callbacks) { if (String.IsNullOrEmpty(key)) { throw new ArgumentNullException(nameof(key)); } if (callbacks == null) { throw new ArgumentNullException(nameof(callbacks)); } if (callbacks.ReadChunk == null) { throw new ArgumentException("ReadChunk callback must be specified."); } key = DedupeCommon.SanitizeString(key); DedupeObject md = GetMetadata(key); if (md == null) { throw new KeyNotFoundException("Object key '" + key + "' not found."); } return(new DedupeStream(md, _Database, callbacks)); }
/// <summary> /// Write an object to the deduplication index if it doesn't already exist, or, replace the object if it does. /// This method will use the callbacks supplied in the method signature. /// </summary> /// <param name="key">The object key. Must be unique in the index.</param> /// <param name="callbacks">CallbackMethods object containing callback methods.</param> /// <param name="data">The byte data for the object.</param> public void WriteOrReplace(string key, DedupeCallbacks callbacks, byte[] data) { if (String.IsNullOrEmpty(key)) { throw new ArgumentNullException(nameof(key)); } if (data == null || data.Length < 1) { throw new ArgumentNullException(nameof(data)); } WriteOrReplace(key, callbacks, data.Length, DedupeCommon.BytesToStream(data)); }
/// <summary> /// Write an object to the deduplication index. /// This method will use the callbacks supplied in the method signature. /// </summary> /// <param name="key">The object key. Must be unique in the index.</param> /// <param name="callbacks">CallbackMethods object containing callback methods.</param> /// <param name="bytes">The byte data for the object.</param> public void Write(string key, DedupeCallbacks callbacks, byte[] bytes) { if (String.IsNullOrEmpty(key)) { throw new ArgumentNullException(nameof(key)); } if (bytes == null || bytes.Length < 1) { throw new ArgumentNullException(nameof(bytes)); } Write(key, callbacks, bytes.Length, DedupeCommon.BytesToStream(bytes)); }
/// <summary> /// Write an object to the deduplication index. /// This method will use the callbacks supplied in the method signature. /// </summary> /// <param name="key">The object key. Must be unique in the index.</param> /// <param name="callbacks">CallbackMethods object containing callback methods.</param> /// <param name="data">The string data for the object.</param> public void Write(string key, DedupeCallbacks callbacks, string data) { if (String.IsNullOrEmpty(key)) { throw new ArgumentNullException(nameof(key)); } if (String.IsNullOrEmpty(data)) { throw new ArgumentNullException(nameof(data)); } byte[] bytes = Encoding.UTF8.GetBytes(data); Write(key, callbacks, bytes.Length, DedupeCommon.BytesToStream(bytes)); }
/// <summary> /// Retrieve an object from the deduplication index. /// This method will use the callbacks supplied in the method signature. /// </summary> /// <param name="key">The object key.</param> /// <param name="callbacks">CallbackMethods object containing callback methods.</param> /// <returns>Object data.</returns> public DedupeObject Get(string key, DedupeCallbacks callbacks) { if (String.IsNullOrEmpty(key)) { throw new ArgumentNullException(nameof(key)); } if (callbacks == null) { throw new ArgumentNullException(nameof(callbacks)); } if (callbacks.ReadChunk == null) { throw new ArgumentException("ReadChunk callback must be specified."); } key = DedupeCommon.SanitizeString(key); DedupeObject md = _Database.GetObjectMetadata(key); if (md == null) { throw new KeyNotFoundException("Object key '" + key + "' not found."); } if (md.Chunks == null || md.Chunks.Count < 1) { throw new IOException("No chunks returned for object key '" + key + "'."); } MemoryStream stream = new MemoryStream(); long contentLength = 0; foreach (DedupeObjectMap curr in md.ObjectMap) { byte[] chunkData = callbacks.ReadChunk(curr.ChunkKey); if (chunkData == null || chunkData.Length < 1) { throw new IOException("Unable to read chunk '" + curr.ChunkKey + "'."); } stream.Write(chunkData, 0, chunkData.Length); contentLength += chunkData.Length; } if (contentLength > 0) { stream.Seek(0, SeekOrigin.Begin); } md.DataStream = stream; return(md); }
internal DedupeStream(DedupeObject md, DbProvider db, DedupeCallbacks callbacks) { if (md == null) { throw new ArgumentNullException(nameof(md)); } if (db == null) { throw new ArgumentNullException(nameof(db)); } if (callbacks == null) { throw new ArgumentNullException(nameof(callbacks)); } _Metadata = md; _Database = db; _Callbacks = callbacks; }
/// <summary> /// Initialize a new or existing index using an internal Sqlite database. /// </summary> /// <param name="indexFile">Path and filename.</param> /// <param name="settings">Deduplication settings.</param> /// <param name="callbacks">Object containing callback functions for writing, reading, and deleting chunks.</param> public DedupeLibrary(string indexFile, DedupeSettings settings, DedupeCallbacks callbacks) { if (String.IsNullOrEmpty(indexFile)) { throw new ArgumentNullException(nameof(indexFile)); } if (settings == null) { throw new ArgumentNullException(nameof(settings)); } if (callbacks == null) { throw new ArgumentNullException(nameof(callbacks)); } _Settings = settings; _Callbacks = callbacks; _IndexFile = DedupeCommon.SanitizeString(indexFile); _Database = new SqliteProvider(_IndexFile); InitializeIndex(); }
/// <summary> /// Initialize an existing index using an external database. Tables must be created ahead of time. /// </summary> /// <param name="database">Database provider implemented using the Database.DbProvider class.</param> /// <param name="settings">Deduplication settings.</param> /// <param name="callbacks">Object containing callback functions for writing, reading, and deleting chunks.</param> public DedupeLibrary(DbProvider database, DedupeSettings settings, DedupeCallbacks callbacks) { if (database == null) { throw new ArgumentNullException(nameof(database)); } if (settings == null) { throw new ArgumentNullException(nameof(settings)); } if (callbacks == null) { throw new ArgumentNullException(nameof(callbacks)); } _Database = database; _Settings = settings; _Callbacks = callbacks; InitializeIndex(); }
/// <summary> /// Write an object to the deduplication index if it doesn't already exist, or, replace the object if it does. /// This method will use the callbacks supplied in the method signature. /// </summary> /// <param name="key">The object key. Must be unique in the index.</param> /// <param name="callbacks">CallbackMethods object containing callback methods.</param> /// <param name="contentLength">The length of the data.</param> /// <param name="stream">The stream containing the data.</param> public void WriteOrReplace(string key, DedupeCallbacks callbacks, long contentLength, Stream stream) { if (String.IsNullOrEmpty(key)) { throw new ArgumentNullException(nameof(key)); } if (callbacks == null) { throw new ArgumentNullException(nameof(callbacks)); } if (callbacks.WriteChunk == null) { throw new ArgumentException("WriteChunk callback must be specified."); } if (callbacks.DeleteChunk == null) { throw new ArgumentException("DeleteChunk callback must be specified."); } if (stream == null) { throw new ArgumentNullException(nameof(stream)); } if (!stream.CanRead) { throw new ArgumentException("Cannot read from the supplied stream."); } key = DedupeCommon.SanitizeString(key); if (_Database.Exists(key)) { Logger?.Invoke(_Header + "Object " + key + " already exists, deleting"); Delete(key); } Write(key, callbacks, contentLength, stream); }
/// <summary> /// Write an object to the deduplication index. /// This method will use the callbacks supplied in the method signature. /// </summary> /// <param name="key">The object key. Must be unique in the index.</param> /// <param name="callbacks">CallbackMethods object containing callback methods.</param> /// <param name="contentLength">The length of the data.</param> /// <param name="stream">The stream containing the data.</param> public void Write(string key, DedupeCallbacks callbacks, long contentLength, Stream stream) { #region Initialize if (String.IsNullOrEmpty(key)) { throw new ArgumentNullException(nameof(key)); } if (_Database.Exists(key)) { throw new ArgumentException("An object with key '" + key + "' already exists."); } if (callbacks == null) { throw new ArgumentNullException(nameof(callbacks)); } if (callbacks.WriteChunk == null) { throw new ArgumentException("WriteChunk callback must be specified."); } if (callbacks.DeleteChunk == null) { throw new ArgumentException("DeleteChunk callback must be specified."); } if (contentLength < 1) { throw new ArgumentException("Content length must be at least one byte."); } if (stream == null) { throw new ArgumentNullException(nameof(stream)); } if (!stream.CanRead) { throw new InvalidOperationException("Cannot read from the supplied stream."); } key = DedupeCommon.SanitizeString(key); bool garbageCollectionRequired = false; #endregion #region Chunk-Data List <DedupeChunk> chunks = new List <DedupeChunk>(); try { Action <DedupeChunk, DedupeObjectMap> processChunk = delegate(DedupeChunk chunk, DedupeObjectMap map) { if (chunk == null || map == null) { return; } _Database.IncrementChunkRefcount(chunk.Key, chunk.Length); _Database.AddObjectMap(key, chunk.Key, chunk.Length, map.ChunkPosition, map.ChunkAddress); callbacks.WriteChunk(chunk); }; chunks = ChunkStream(key, contentLength, stream, processChunk); _Database.AddObject(key, contentLength); } finally { if (garbageCollectionRequired) { List <string> garbageCollectKeys = _Database.Delete(key); if (garbageCollectKeys != null && garbageCollectKeys.Count > 0) { foreach (string gcKey in garbageCollectKeys) { callbacks.DeleteChunk(gcKey); } } } } #endregion }