static void Main(string[] args)
{
// Create chunk directory
if (!Directory.Exists("chunks"))
{
Directory.CreateDirectory("chunks");
}
// Define settings, callbacks, and initialize
DedupeSettings settings = new DedupeSettings(32768, 262144, 2048, 2);
DedupeCallbacks callbacks = new DedupeCallbacks(WriteChunk, ReadChunk, DeleteChunk);
DedupeLibrary dedupe = new DedupeLibrary("test.db", settings, callbacks);
// Store objects in the index
dedupe.Write("kjv1", File.ReadAllBytes("samplefiles/kjv.txt"));
dedupe.Write("kjv2", File.ReadAllBytes("samplefiles/kjv.txt"));
dedupe.Write("kjv3", File.ReadAllBytes("samplefiles/kjv.txt"));
// Check existence and retrieve an object from the index
if (dedupe.Exists("kjv2"))
{
Console.WriteLine("Exists");
}
DedupeObject obj = dedupe.Get("kjv1");
// List all objects
Console.WriteLine(dedupe.ListObjects().ToTabularString());
// Display index statistics
Console.WriteLine(dedupe.IndexStats().ToString());
// Delete an object from the index
dedupe.Delete("kjv1");
}
static void Initialize()
{
if (!Directory.Exists("Chunks"))
{
Directory.CreateDirectory("Chunks");
}
_Settings = new DedupeSettings(32768, 262144, 2048, 2);
_Callbacks = new DedupeCallbacks(WriteChunk, ReadChunk, DeleteChunk);
_Dedupe = new DedupeLibrary("test.db", _Settings, _Callbacks);
}
static void Initialize()
{
if (!Directory.Exists("Chunks"))
{
Directory.CreateDirectory("Chunks");
}
_Settings = new DedupeSettings(32768, 262144, 2048, 2);
_Callbacks = new DedupeCallbacks(WriteChunk, ReadChunk, DeleteChunk);
_DbSettings = new DatabaseSettings(DbTypes.Mysql, "127.0.0.1", 3306, "root", "password", "dedupe");
_ORM = new WatsonORM(_DbSettings);
_DbProvider = new Database(_ORM);
_Dedupe = new DedupeLibrary(_DbProvider, _Settings, _Callbacks);
}
static void Initialize()
{
if (!Directory.Exists("Chunks"))
{
Directory.CreateDirectory("Chunks");
}
if (File.Exists("Test.db"))
{
_Dedupe = new DedupeLibrary("Test.db", WriteChunk, ReadChunk, DeleteChunk, _DebugDedupe, _DebugSql);
}
else
{
// Dedupe = new DedupeLibrary("Test.db", 2048, 16384, 64, 2, WriteChunk, ReadChunk, DeleteChunk, DebugDedupe, DebugSql);
_Dedupe = new DedupeLibrary("Test.db", 32768, 262144, 2048, 2, WriteChunk, ReadChunk, DeleteChunk, _DebugDedupe, _DebugSql);
}
}
static void Main(string[] args)
{
bool readIndex = false;
DedupeLibrary l;
bool r;
int numObjects;
int numChunks;
long logicalBytes;
long physicalBytes;
decimal dedupeRatioX;
decimal dedupeRatioPercent;
if (readIndex)
{
l = new DedupeLibrary("ff.idx", WriteChunk, ReadChunk, DeleteChunk, false, false);
}
else
//{
// l = new DedupeLibrary("ff.idx", 2048, 2048, 1, 4, WriteChunk, ReadChunk, DeleteChunk, false, false);
// byte[] d = File.ReadAllBytes(@"c:\Users\bsv79\source\repos\ConsoleApp1\ConsoleApp1\test_data\001.bin");
// List<Chunk> c = new List<Chunk>();
// r = l.StoreObject("test", d, out c);
// r = l.StoreObject("test1", d, out c);
//}
{
l = new DedupeLibrary("ff.idx", 2048, 2048, 0, 4, WriteChunk, ReadChunk, DeleteChunk, false, false);
l.Database.StartTransaction();
List <Chunk> c = new List <Chunk>();
FileStream f = File.OpenRead(@"c:\Users\bsv79\source\repos\ConsoleApp1\ConsoleApp1\test_data\001.bin");
byte[] d = new byte[2048];
int h = f.Read(d, 0, 2048);
while (h == 2048)
{
r = l.StoreObject("test_" + f.Position, d, out c);
h = f.Read(d, 0, 2048);
}
l.Database.EndTransaction();
}
r = l.IndexStats(out numObjects, out numChunks, out logicalBytes, out physicalBytes, out dedupeRatioX, out dedupeRatioPercent);
Console.WriteLine("numObjects: {0}, numChunks: {1}, logicalBytes: {2}, physicalBytes: {3}, dedupeRatioX: {4}, dedupeRatioPercent: {5}", numObjects, numChunks, logicalBytes, physicalBytes, dedupeRatioX, dedupeRatioPercent);
Console.WriteLine("res: {0}", r);
}
static void Initialize()
{
if (!Directory.Exists("Chunks"))
{
Directory.CreateDirectory("Chunks");
}
_Database = new DatabaseClient(DbTypes.MySql, "127.0.0.1", 3306, "root", "password", null, "dedupe");
_DbProvider = new Database(_Database, _DebugSql);
if (_DbProvider.IsInitialized())
{
_Dedupe = new DedupeLibrary(_DbProvider, WriteChunk, ReadChunk, DeleteChunk, _DebugDedupe, _DebugSql);
}
else
{
_Dedupe = new DedupeLibrary(_DbProvider, 32768, 262144, 2048, 2, WriteChunk, ReadChunk, DeleteChunk, _DebugDedupe, _DebugSql);
}
}
public override bool OnStart()
{
//Watson DeDupe
if (!Directory.Exists("Chunks"))
{
Directory.CreateDirectory("Chunks");
}
if (File.Exists("Test.db"))
{
Dedupe = new DedupeLibrary("Test.db", WriteChunk, ReadChunk, DeleteChunk, DebugDedupe, DebugSql);
}
else
{
Dedupe = new DedupeLibrary("Test.db", 64, 131072, 64, 2, WriteChunk, ReadChunk, DeleteChunk, DebugDedupe, DebugSql);
}
///END Watson DeDupe
///
// Buffer to maximise search size
//stripeBuffer = new ReadWriteBuffer(stripeSize);
Console.Write(Environment.NewLine + "-------------------------" + Environment.NewLine + "DeDupe Engine Initialized" + Environment.NewLine + "-------------------------" + Environment.NewLine);
int maxSizeMB = 50;
maxObjectSizeHTTP = maxSizeMB * 1024 * 1024;
Console.Write("<DeDupe> Maximum system supported HTTP object size:" + (Int32.MaxValue / 1024 / 1024) + "Mb / Maximum HTTP object size:" + (maxObjectSizeHTTP / 1024 / 1024) + "Mb" + Environment.NewLine);
// Gather index and dedupe stats
if (Dedupe.IndexStats(out NumObjects, out NumChunks, out LogicalBytes, out PhysicalBytes, out DedupeRatioX, out DedupeRatioPercent))
{
Console.WriteLine(" Number of objects: " + NumObjects + "/ Number of chunks: " + NumChunks);
Console.WriteLine(" Logical bytes : " + LogicalBytes + " bytes" + "/ Physical bytes : " + PhysicalBytes + " bytes");
Console.WriteLine(" Dedupe ratio : " + DedupeRatioX + "X, " + DedupeRatioPercent + "%");
Console.WriteLine("-------------------------");
}
return(true);
}
static void Main(string[] args)
{
try
{
#region Parse-Arguments
if (args == null || args.Length < 2)
{
Usage("No arguments specified");
return;
}
_IndexFile = args[0];
_Command = args[1];
for (int i = 2; i < args.Length; i++)
{
if (String.IsNullOrEmpty(args[i]))
{
continue;
}
if (args[i].StartsWith("--chunks=") && args[i].Length > 9)
{
_ChunkDir = args[i].Substring(9);
if (!_ChunkDir.EndsWith("\\"))
{
_ChunkDir += "\\";
}
if (!Directory.Exists(_ChunkDir))
{
Directory.CreateDirectory(_ChunkDir);
}
}
else if (args[i].StartsWith("--key=") && args[i].Length > 6)
{
_Key = args[i].Substring(6);
}
else if (args[i].StartsWith("--idxstart=") && args[i].Length > 11)
{
if (!Int32.TryParse(args[i].Substring(11), out _IndexStart))
{
Usage("Index start must be an integer value.");
return;
}
else
{
if (_IndexStart < 0)
{
Usage("Index start must be greater than zero.");
return;
}
}
}
else if (args[i].StartsWith("--results=") && args[i].Length > 10)
{
if (!Int32.TryParse(args[i].Substring(10), out _MaxResults))
{
Usage("Max results must be an integer value.");
return;
}
else
{
if (_MaxResults < 1 || _MaxResults > 100)
{
Usage("Max results must be greater than zero and less than or equal to 100.");
return;
}
}
}
else if (args[i].StartsWith("--params=") && args[i].Length > 9)
{
_CreateParams = args[i].Substring(9);
if (new Regex(@"^\d+,\d+,\d+,\d+$").IsMatch(_CreateParams))
{
string[] currParams = _CreateParams.Split(',');
if (currParams.Length != 4)
{
Usage("Value for 'params' is invalid");
return;
}
if (!Int32.TryParse(currParams[0], out _MinChunkSize) ||
!Int32.TryParse(currParams[1], out _MaxChunkSize) ||
!Int32.TryParse(currParams[2], out _ShiftCount) ||
!Int32.TryParse(currParams[3], out _BoundaryCheckBytes)
)
{
Usage("Value for 'params' is not of the form int,int,int,int");
return;
}
}
else
{
Usage("Value for 'params' is not of the form int,int,int,int");
return;
}
}
else
{
Usage("Unknown argument: " + args[i]);
return;
}
}
#endregion
#region Verify-Values
List <string> validCommands = new List <string>()
{
"create", "stats", "write", "get", "del", "list", "exists", "md"
};
if (!validCommands.Contains(_Command))
{
Usage("Invalid command: " + _Command);
return;
}
#endregion
#region Create
if (String.Compare(_Command, "create") == 0)
{
_Settings = new DedupeSettings(_MinChunkSize, _MaxChunkSize, _ShiftCount, _BoundaryCheckBytes);
_Callbacks = new DedupeCallbacks(WriteChunk, ReadChunk, DeleteChunk);
_Dedupe = new DedupeLibrary(_IndexFile, _Settings, _Callbacks);
return;
}
#endregion
#region Initialize-Index
if (!File.Exists(_IndexFile))
{
Console.WriteLine("*** Index file " + _IndexFile + " not found");
}
_Settings = new DedupeSettings();
_Callbacks = new DedupeCallbacks(WriteChunk, ReadChunk, DeleteChunk);
_Dedupe = new DedupeLibrary(_IndexFile, _Settings, _Callbacks);
#endregion
#region Process-by-Command
switch (_Command)
{
case "stats":
_Stats = _Dedupe.IndexStats();
if (_Stats != null)
{
Console.WriteLine("Statistics:");
Console.WriteLine(" Number of objects : " + _Stats.Objects);
Console.WriteLine(" Number of chunks : " + _Stats.Chunks);
Console.WriteLine(" Logical bytes : " + _Stats.LogicalBytes + " bytes");
Console.WriteLine(" Physical bytes : " + _Stats.PhysicalBytes + " bytes");
Console.WriteLine(" Dedupe ratio : " + DecimalToString(_Stats.RatioX) + "X, " + DecimalToString(_Stats.RatioPercent) + "%");
return;
}
else
{
Console.WriteLine("Failed");
}
return;
case "get":
if (String.IsNullOrEmpty(_Key))
{
Usage("Object key must be supplied");
}
else
{
_Object = _Dedupe.Get(_Key);
if (_Object == null)
{
Console.WriteLine("Failed");
}
else
{
WriteConsoleData(_Object.Data);
}
}
return;
case "write":
if (String.IsNullOrEmpty(_Key))
{
Usage("Object key must be supplied");
}
else
{
if (_Dedupe.Exists(_Key))
{
Console.WriteLine("Already exists");
}
else
{
ReadConsoleData();
_Dedupe.Write(_Key, _Request);
}
}
return;
case "del":
if (String.IsNullOrEmpty(_Key))
{
Usage("Object key must be supplied");
}
else
{
_Dedupe.Delete(_Key);
}
return;
case "md":
if (String.IsNullOrEmpty(_Key))
{
Usage("Object key must be supplied");
}
else
{
_Object = _Dedupe.GetMetadata(_Key);
if (_Object == null)
{
Console.WriteLine("Failed");
}
else
{
Console.WriteLine(_Object.ToString());
}
}
return;
case "list":
_EnumResult = _Dedupe.ListObjects(_Key, _IndexStart, _MaxResults);
if (_EnumResult == null)
{
Console.WriteLine("No objects");
}
else
{
Console.WriteLine(_EnumResult.ToTabularString());
}
return;
case "exists":
if (String.IsNullOrEmpty(_Key))
{
Usage("Object key must be supplied");
return;
}
else
{
Console.WriteLine(_Dedupe.Exists(_Key));
}
return;
default:
Usage("Unknown command: " + _Command);
return;
}
#endregion
}
catch (Exception e)
{
ExceptionConsole("Dedupe", "Outer exception", e);
}
}
static void Main(string[] args)
{
try
{
#region Parse-Arguments
if (args == null || args.Length < 2)
{
Usage("No arguments specified");
return;
}
IndexFile = args[0];
Command = args[1];
for (int i = 2; i < args.Length; i++)
{
if (String.IsNullOrEmpty(args[i]))
{
continue;
}
if (args[i].StartsWith("--chunks=") && args[i].Length > 9)
{
ChunkDirectory = args[i].Substring(9);
if (!ChunkDirectory.EndsWith("\\"))
{
ChunkDirectory += "\\";
}
if (!Directory.Exists(ChunkDirectory))
{
Directory.CreateDirectory(ChunkDirectory);
}
}
else if (args[i].StartsWith("--key=") && args[i].Length > 6)
{
ObjectKey = args[i].Substring(6);
}
else if (String.Compare(args[i], "--debug") == 0)
{
DebugDedupe = true;
}
else if (String.Compare(args[i], "--debugsql") == 0)
{
DebugSql = true;
}
else if (args[i].StartsWith("--params=") && args[i].Length > 9)
{
CreateParams = args[i].Substring(9);
if (new Regex(@"^\d+,\d+,\d+,\d+$").IsMatch(CreateParams))
{
string[] currParams = CreateParams.Split(',');
if (currParams.Length != 4)
{
Usage("Value for 'params' is invalid");
return;
}
if (!Int32.TryParse(currParams[0], out MinChunkSize) ||
!Int32.TryParse(currParams[1], out MaxChunkSize) ||
!Int32.TryParse(currParams[2], out ShiftCount) ||
!Int32.TryParse(currParams[3], out BoundaryCheckBytes)
)
{
Usage("Value for 'params' is not of the form int,int,int,int");
return;
}
}
else
{
Usage("Value for 'params' is not of the form int,int,int,int");
return;
}
}
else
{
Usage("Unknown argument: " + args[i]);
return;
}
}
#endregion
#region Verify-Values
List <string> validCommands = new List <string>()
{
"create", "stats", "store", "retrieve", "delete", "list", "exists", "metadata"
};
if (!validCommands.Contains(Command))
{
Usage("Invalid command: " + Command);
return;
}
#endregion
#region Enumerate
if (DebugDedupe)
{
Console.WriteLine("Command : " + Command);
Console.WriteLine("Index File : " + IndexFile);
Console.WriteLine("Chunk Directory : " + ChunkDirectory);
Console.WriteLine("Object Key : " + ObjectKey);
if (MinChunkSize > 0)
{
Console.WriteLine("Min Chunk Size : " + MinChunkSize);
}
if (MaxChunkSize > 0)
{
Console.WriteLine("Max Chunk Size : " + MaxChunkSize);
}
if (ShiftCount > 0)
{
Console.WriteLine("Shift Count : " + ShiftCount);
}
if (BoundaryCheckBytes > 0)
{
Console.WriteLine("Boundary Bytes : " + BoundaryCheckBytes);
}
}
#endregion
#region Create
if (String.Compare(Command, "create") == 0)
{
Dedupe = new DedupeLibrary(IndexFile, MinChunkSize, MaxChunkSize, ShiftCount, BoundaryCheckBytes, WriteChunk, ReadChunk, DeleteChunk, DebugDedupe, DebugSql);
if (DebugDedupe)
{
Console.WriteLine("Successfully wrote new index: " + IndexFile);
}
return;
}
#endregion
#region Initialize-Index
if (!File.Exists(IndexFile))
{
Console.WriteLine("*** Index file " + IndexFile + " not found");
}
Dedupe = new DedupeLibrary(IndexFile, WriteChunk, ReadChunk, DeleteChunk, DebugDedupe, DebugSql);
#endregion
#region Process-by-Command
switch (Command)
{
case "stats":
if (Dedupe.IndexStats(out NumObjects, out NumChunks, out LogicalBytes, out PhysicalBytes, out DedupeRatioX, out DedupeRatioPercent))
{
Console.WriteLine("Statistics:");
Console.WriteLine(" Number of objects : " + NumObjects);
Console.WriteLine(" Number of chunks : " + NumChunks);
Console.WriteLine(" Logical bytes : " + LogicalBytes + " bytes");
Console.WriteLine(" Physical bytes : " + PhysicalBytes + " bytes");
Console.WriteLine(" Dedupe ratio : " + DecimalToString(DedupeRatioX) + "X, " + DecimalToString(DedupeRatioPercent) + "%");
return;
}
else
{
Console.WriteLine("Failed");
}
return;
case "retrieve":
if (String.IsNullOrEmpty(ObjectKey))
{
Usage("Object key must be supplied");
}
else
{
if (!Dedupe.RetrieveObject(ObjectKey, out ResponseData))
{
Console.WriteLine("Failed");
}
else
{
WriteConsoleData(ResponseData);
}
}
return;
case "store":
if (String.IsNullOrEmpty(ObjectKey))
{
Usage("Object key must be supplied");
}
else
{
if (Dedupe.ObjectExists(ObjectKey))
{
Console.WriteLine("Already exists");
}
else
{
ReadConsoleData();
if (!Dedupe.StoreObject(ObjectKey, RequestData, out Chunks))
{
Console.WriteLine("Failed");
}
else
{
Console.WriteLine("Success");
}
}
}
return;
case "delete":
if (String.IsNullOrEmpty(ObjectKey))
{
Usage("Object key must be supplied");
}
else
{
if (!Dedupe.DeleteObject(ObjectKey))
{
Console.WriteLine("Failed");
}
else
{
Console.WriteLine("Success");
}
}
return;
case "metadata":
if (String.IsNullOrEmpty(ObjectKey))
{
Usage("Object key must be supplied");
}
else
{
if (!Dedupe.RetrieveObjectMetadata(ObjectKey, out Metadata))
{
Console.WriteLine("Failed");
}
else
{
Console.WriteLine(Metadata.ToString());
}
}
return;
case "list":
Dedupe.ListObjects(out Keys);
if (Keys == null || Keys.Count < 1)
{
Console.WriteLine("No objects");
}
else
{
Console.WriteLine("Objects:");
foreach (string curr in Keys)
{
Console.WriteLine(" " + curr);
}
Console.WriteLine(Keys.Count + " objects in index");
}
return;
case "exists":
if (String.IsNullOrEmpty(ObjectKey))
{
Usage("Object key must be supplied");
return;
}
else
{
Console.WriteLine(Dedupe.ObjectExists(ObjectKey));
}
return;
default:
Usage("Unknown command: " + Command);
return;
}
#endregion
}
catch (Exception e)
{
ExceptionConsole("Dedupe", "Outer exception", e);
}
}
