static void Main(string[] args) { // Create chunk directory if (!Directory.Exists("chunks")) { Directory.CreateDirectory("chunks"); } // Define settings, callbacks, and initialize DedupeSettings settings = new DedupeSettings(32768, 262144, 2048, 2); DedupeCallbacks callbacks = new DedupeCallbacks(WriteChunk, ReadChunk, DeleteChunk); DedupeLibrary dedupe = new DedupeLibrary("test.db", settings, callbacks); // Store objects in the index dedupe.Write("kjv1", File.ReadAllBytes("samplefiles/kjv.txt")); dedupe.Write("kjv2", File.ReadAllBytes("samplefiles/kjv.txt")); dedupe.Write("kjv3", File.ReadAllBytes("samplefiles/kjv.txt")); // Check existence and retrieve an object from the index if (dedupe.Exists("kjv2")) { Console.WriteLine("Exists"); } DedupeObject obj = dedupe.Get("kjv1"); // List all objects Console.WriteLine(dedupe.ListObjects().ToTabularString()); // Display index statistics Console.WriteLine(dedupe.IndexStats().ToString()); // Delete an object from the index dedupe.Delete("kjv1"); }
/// <summary> /// Retrieve metadata for a given object. /// DedupeObjectMap objects returned should be ordered in ascending order based on the chunk's position or address. /// </summary> /// <param name="key">Object key.</param> /// <returns>Object metadata.</returns> public override DedupeObject GetObjectMetadata(string key) { if (String.IsNullOrEmpty(key)) { throw new ArgumentNullException(nameof(key)); } key = DedupeCommon.SanitizeString(key); DbExpression e = new DbExpression( _ORM.GetColumnName <DedupeObject>(nameof(DedupeObject.Key)), DbOperators.Equals, key); DedupeObject ret = _ORM.SelectFirst <DedupeObject>(e); if (ret != null) { ret.Chunks = GetChunks(key); ret.ObjectMap = GetObjectMap(key); if (ret.ObjectMap != null && ret.ObjectMap.Count > 0) { ret.ObjectMap = ret.ObjectMap.OrderBy(o => o.ChunkAddress).ToList(); } } return(ret); }
/// <summary> /// Add a new object to the index. /// </summary> /// <param name="key">Object key.</param> /// <param name="length">The total length of the object.</param> public override void AddObject(string key, long length) { if (String.IsNullOrEmpty(key)) { throw new ArgumentNullException(nameof(key)); } if (length < 1) { throw new ArgumentException("Length must be greater than zero."); } key = DedupeCommon.SanitizeString(key); if (Exists(key)) { throw new ArgumentException("An object with key '" + key + "' already exists."); } DedupeObject obj = _ORM.Insert <DedupeObject>(new DedupeObject(key, length)); }
/// <summary> /// Retrieve the object map containing the metadata for a given address within the original object. /// </summary> /// <param name="key">Object key.</param> /// <param name="position">Starting byte position.</param> /// <returns>Dedupe object map.</returns> public override DedupeObjectMap GetObjectMapForPosition(string key, long position) { if (String.IsNullOrEmpty(key)) { throw new ArgumentNullException(nameof(key)); } if (position < 0) { throw new ArgumentOutOfRangeException("Start of range must be zero or greater."); } key = DedupeCommon.SanitizeString(key); DedupeObject obj = GetObjectMetadata(key); if (obj == null) { return(null); } string objMapTable = _ORM.GetTableName(typeof(DedupeObjectMap)); string objKeyCol = _ORM.GetColumnName <DedupeObjectMap>(nameof(DedupeObjectMap.ObjectKey)); string chunkAddrCol = _ORM.GetColumnName <DedupeObjectMap>(nameof(DedupeObjectMap.ChunkAddress)); string chunkLenCol = _ORM.GetColumnName <DedupeObjectMap>(nameof(DedupeObjectMap.ChunkLength)); string query = "SELECT * FROM " + objMapTable + " " + "WHERE " + " " + objKeyCol + " = '" + obj.Key + "' " + " AND " + chunkAddrCol + " <= " + position + " AND " + chunkAddrCol + " + " + chunkLenCol + " > " + position + " "; DedupeObjectMap map = null; DataTable result = _ORM.Query(query); if (result != null && result.Rows.Count > 0) { map = _ORM.DataRowToObject <DedupeObjectMap>(result.Rows[0]); } return(map); }
/// <summary> /// Delete an object and dereference the associated chunks. /// </summary> /// <param name="key">Object key.</param> /// <returns>List of chunk keys that should be garbage collected.</returns> public override List <string> Delete(string key) { if (String.IsNullOrEmpty(key)) { throw new ArgumentNullException(nameof(key)); } List <string> ret = new List <string>(); key = DedupeCommon.SanitizeString(key); DedupeObject obj = GetObjectMetadata(key); if (obj == null) { throw new KeyNotFoundException("Key '" + key + "' not found."); } List <DedupeObjectMap> maps = GetObjectMap(key); if (maps != null && maps.Count > 0) { foreach (DedupeObjectMap map in maps) { if (DecrementChunkRefcount(map.ChunkKey)) { ret.Add(map.ChunkKey); } } } DbExpression e = new DbExpression( _ORM.GetColumnName <DedupeObjectMap>(nameof(DedupeObjectMap.ObjectKey)), DbOperators.Equals, key); _ORM.DeleteMany <DedupeObjectMap>(e); // chunks to GC return(ret); }
static void Main(string[] args) { bool runForever = true; string filename = ""; string key = ""; List <Chunk> chunks = new List <Chunk>(); DedupeObject obj = null; Initialize(); while (runForever) { Console.Write("Command [? for help] > "); string userInput = Console.ReadLine(); if (String.IsNullOrEmpty(userInput)) { continue; } switch (userInput) { case "?": Console.WriteLine("Available commands:"); Console.WriteLine(" q quit"); Console.WriteLine(" cls clear the screen"); Console.WriteLine(" write store an object"); Console.WriteLine(" get retrieve an object"); Console.WriteLine(" del delete an object"); Console.WriteLine(" md retrieve object metadata"); Console.WriteLine(" list list 100 objects in the index"); Console.WriteLine(" listp paginated list objects"); Console.WriteLine(" exists check if object exists in the index"); Console.WriteLine(" stats list index stats"); Console.WriteLine(""); break; case "q": case "Q": runForever = false; break; case "cls": Console.Clear(); break; case "write": filename = InputString("Input filename:", null, false); key = InputString("Object key:", null, false); long contentLength = GetContentLength(filename); using (FileStream fs = new FileStream(filename, FileMode.Open)) { _Dedupe.Write(key, contentLength, fs); } break; case "get": key = InputString("Object key:", null, false); filename = InputString("Output filename:", null, false); obj = _Dedupe.Get(key); if (obj != null) { if (obj.Length > 0) { using (FileStream fs = new FileStream(filename, FileMode.OpenOrCreate)) { int bytesRead = 0; long bytesRemaining = obj.Length; byte[] readBuffer = new byte[65536]; while (bytesRemaining > 0) { bytesRead = obj.DataStream.Read(readBuffer, 0, readBuffer.Length); if (bytesRead > 0) { fs.Write(readBuffer, 0, bytesRead); bytesRemaining -= bytesRead; } } } Console.WriteLine("Success"); } else { Console.WriteLine("Success, (no data)"); } } else { Console.WriteLine("Failed"); } break; case "del": key = InputString("Object key:", null, false); _Dedupe.Delete(key); break; case "md": key = InputString("Object key:", null, false); obj = _Dedupe.GetMetadata(key); if (obj != null) { Console.WriteLine("Success"); Console.WriteLine(obj.ToString()); } else { Console.WriteLine("Failed"); } break; case "list": _EnumResult = _Dedupe.ListObjects(); if (_EnumResult == null) { Console.WriteLine("No objects"); } else { Console.WriteLine(_EnumResult.ToTabularString()); } break; case "exists": key = InputString("Object key:", null, false); if (_Dedupe.Exists(key)) { Console.WriteLine("Object exists"); } else { Console.WriteLine("Object does not exist"); } break; case "stats": _Stats = _Dedupe.IndexStats(); if (_Stats != null) { Console.WriteLine("Statistics:"); Console.WriteLine(" Number of objects : " + _Stats.Objects); Console.WriteLine(" Number of chunks : " + _Stats.Chunks); Console.WriteLine(" Logical bytes : " + _Stats.LogicalBytes + " bytes"); Console.WriteLine(" Physical bytes : " + _Stats.PhysicalBytes + " bytes"); Console.WriteLine(" Dedupe ratio : " + DecimalToString(_Stats.RatioX) + "X, " + DecimalToString(_Stats.RatioPercent) + "%"); Console.WriteLine(""); } else { Console.WriteLine("Failed"); } break; default: break; } } }
static void Main(string[] args) { try { #region Parse-Arguments if (args == null || args.Length < 2) { Usage("No arguments specified"); return; } _IndexFile = args[0]; _Command = args[1]; for (int i = 2; i < args.Length; i++) { if (String.IsNullOrEmpty(args[i])) { continue; } if (args[i].StartsWith("--chunks=") && args[i].Length > 9) { _ChunkDir = args[i].Substring(9); if (!_ChunkDir.EndsWith("\\")) { _ChunkDir += "\\"; } if (!Directory.Exists(_ChunkDir)) { Directory.CreateDirectory(_ChunkDir); } } else if (args[i].StartsWith("--key=") && args[i].Length > 6) { _Key = args[i].Substring(6); } else if (args[i].StartsWith("--idxstart=") && args[i].Length > 11) { if (!Int32.TryParse(args[i].Substring(11), out _IndexStart)) { Usage("Index start must be an integer value."); return; } else { if (_IndexStart < 0) { Usage("Index start must be greater than zero."); return; } } } else if (args[i].StartsWith("--results=") && args[i].Length > 10) { if (!Int32.TryParse(args[i].Substring(10), out _MaxResults)) { Usage("Max results must be an integer value."); return; } else { if (_MaxResults < 1 || _MaxResults > 100) { Usage("Max results must be greater than zero and less than or equal to 100."); return; } } } else if (args[i].StartsWith("--params=") && args[i].Length > 9) { _CreateParams = args[i].Substring(9); if (new Regex(@"^\d+,\d+,\d+,\d+$").IsMatch(_CreateParams)) { string[] currParams = _CreateParams.Split(','); if (currParams.Length != 4) { Usage("Value for 'params' is invalid"); return; } if (!Int32.TryParse(currParams[0], out _MinChunkSize) || !Int32.TryParse(currParams[1], out _MaxChunkSize) || !Int32.TryParse(currParams[2], out _ShiftCount) || !Int32.TryParse(currParams[3], out _BoundaryCheckBytes) ) { Usage("Value for 'params' is not of the form int,int,int,int"); return; } } else { Usage("Value for 'params' is not of the form int,int,int,int"); return; } } else { Usage("Unknown argument: " + args[i]); return; } } #endregion #region Verify-Values List <string> validCommands = new List <string>() { "create", "stats", "write", "get", "del", "list", "exists", "md" }; if (!validCommands.Contains(_Command)) { Usage("Invalid command: " + _Command); return; } #endregion #region Create if (String.Compare(_Command, "create") == 0) { _Settings = new DedupeSettings(_MinChunkSize, _MaxChunkSize, _ShiftCount, _BoundaryCheckBytes); _Callbacks = new DedupeCallbacks(WriteChunk, ReadChunk, DeleteChunk); _Dedupe = new DedupeLibrary(_IndexFile, _Settings, _Callbacks); return; } #endregion #region Initialize-Index if (!File.Exists(_IndexFile)) { Console.WriteLine("*** Index file " + _IndexFile + " not found"); } _Settings = new DedupeSettings(); _Callbacks = new DedupeCallbacks(WriteChunk, ReadChunk, DeleteChunk); _Dedupe = new DedupeLibrary(_IndexFile, _Settings, _Callbacks); #endregion #region Process-by-Command switch (_Command) { case "stats": _Stats = _Dedupe.IndexStats(); if (_Stats != null) { Console.WriteLine("Statistics:"); Console.WriteLine(" Number of objects : " + _Stats.Objects); Console.WriteLine(" Number of chunks : " + _Stats.Chunks); Console.WriteLine(" Logical bytes : " + _Stats.LogicalBytes + " bytes"); Console.WriteLine(" Physical bytes : " + _Stats.PhysicalBytes + " bytes"); Console.WriteLine(" Dedupe ratio : " + DecimalToString(_Stats.RatioX) + "X, " + DecimalToString(_Stats.RatioPercent) + "%"); return; } else { Console.WriteLine("Failed"); } return; case "get": if (String.IsNullOrEmpty(_Key)) { Usage("Object key must be supplied"); } else { _Object = _Dedupe.Get(_Key); if (_Object == null) { Console.WriteLine("Failed"); } else { WriteConsoleData(_Object.Data); } } return; case "write": if (String.IsNullOrEmpty(_Key)) { Usage("Object key must be supplied"); } else { if (_Dedupe.Exists(_Key)) { Console.WriteLine("Already exists"); } else { ReadConsoleData(); _Dedupe.Write(_Key, _Request); } } return; case "del": if (String.IsNullOrEmpty(_Key)) { Usage("Object key must be supplied"); } else { _Dedupe.Delete(_Key); } return; case "md": if (String.IsNullOrEmpty(_Key)) { Usage("Object key must be supplied"); } else { _Object = _Dedupe.GetMetadata(_Key); if (_Object == null) { Console.WriteLine("Failed"); } else { Console.WriteLine(_Object.ToString()); } } return; case "list": _EnumResult = _Dedupe.ListObjects(_Key, _IndexStart, _MaxResults); if (_EnumResult == null) { Console.WriteLine("No objects"); } else { Console.WriteLine(_EnumResult.ToTabularString()); } return; case "exists": if (String.IsNullOrEmpty(_Key)) { Usage("Object key must be supplied"); return; } else { Console.WriteLine(_Dedupe.Exists(_Key)); } return; default: Usage("Unknown command: " + _Command); return; } #endregion } catch (Exception e) { ExceptionConsole("Dedupe", "Outer exception", e); } }