static void Main(string[] args)
{
bool readIndex = false;
DedupeLibrary l;
bool r;
int numObjects;
int numChunks;
long logicalBytes;
long physicalBytes;
decimal dedupeRatioX;
decimal dedupeRatioPercent;
if (readIndex)
{
l = new DedupeLibrary("ff.idx", WriteChunk, ReadChunk, DeleteChunk, false, false);
}
else
//{
// l = new DedupeLibrary("ff.idx", 2048, 2048, 1, 4, WriteChunk, ReadChunk, DeleteChunk, false, false);
// byte[] d = File.ReadAllBytes(@"c:\Users\bsv79\source\repos\ConsoleApp1\ConsoleApp1\test_data\001.bin");
// List<Chunk> c = new List<Chunk>();
// r = l.StoreObject("test", d, out c);
// r = l.StoreObject("test1", d, out c);
//}
{
l = new DedupeLibrary("ff.idx", 2048, 2048, 0, 4, WriteChunk, ReadChunk, DeleteChunk, false, false);
l.Database.StartTransaction();
List <Chunk> c = new List <Chunk>();
FileStream f = File.OpenRead(@"c:\Users\bsv79\source\repos\ConsoleApp1\ConsoleApp1\test_data\001.bin");
byte[] d = new byte[2048];
int h = f.Read(d, 0, 2048);
while (h == 2048)
{
r = l.StoreObject("test_" + f.Position, d, out c);
h = f.Read(d, 0, 2048);
}
l.Database.EndTransaction();
}
r = l.IndexStats(out numObjects, out numChunks, out logicalBytes, out physicalBytes, out dedupeRatioX, out dedupeRatioPercent);
Console.WriteLine("numObjects: {0}, numChunks: {1}, logicalBytes: {2}, physicalBytes: {3}, dedupeRatioX: {4}, dedupeRatioPercent: {5}", numObjects, numChunks, logicalBytes, physicalBytes, dedupeRatioX, dedupeRatioPercent);
Console.WriteLine("res: {0}", r);
}
static void Main(string[] args)
{
bool runForever = true;
string userInput = "";
string filename = "";
string key = "";
long contentLength = 0;
Stream stream = null;
List <Chunk> chunks;
List <string> keys;
ObjectMetadata md;
int numObjects;
int numChunks;
long logicalBytes;
long physicalBytes;
decimal dedupeRatioX;
decimal dedupeRatioPercent;
Initialize();
while (runForever)
{
Console.Write("Command [? for help] > ");
userInput = Console.ReadLine();
if (String.IsNullOrEmpty(userInput))
{
continue;
}
switch (userInput)
{
case "?":
Console.WriteLine("Available commands:");
Console.WriteLine(" q quit");
Console.WriteLine(" cls clear the screen");
Console.WriteLine(" store store an object");
Console.WriteLine(" retrieve retrieve an object");
Console.WriteLine(" delete delete an object");
Console.WriteLine(" metadata retrieve object metadata");
Console.WriteLine(" list list objects in the index");
Console.WriteLine(" exists check if object exists in the index");
Console.WriteLine(" stats list index stats");
Console.WriteLine("");
break;
case "q":
case "Q":
runForever = false;
break;
case "cls":
Console.Clear();
break;
case "store":
filename = InputString("Input filename:", null, false);
key = InputString("Object key:", null, false);
contentLength = GetContentLength(filename);
using (FileStream fs = new FileStream(filename, FileMode.Open))
{
if (_Dedupe.StoreObject(key, contentLength, fs, out chunks))
{
if (chunks != null && chunks.Count > 0)
{
Console.WriteLine("Success: " + chunks.Count + " chunks");
}
else
{
Console.WriteLine("Success (no chunks)");
}
}
else
{
Console.WriteLine("Failed");
}
}
break;
case "retrieve":
key = InputString("Object key:", null, false);
filename = InputString("Output filename:", null, false);
if (_Dedupe.RetrieveObject(key, out contentLength, out stream))
{
if (contentLength > 0)
{
using (FileStream fs = new FileStream(filename, FileMode.OpenOrCreate))
{
int bytesRead = 0;
long bytesRemaining = contentLength;
byte[] readBuffer = new byte[65536];
while (bytesRemaining > 0)
{
bytesRead = stream.Read(readBuffer, 0, readBuffer.Length);
if (bytesRead > 0)
{
fs.Write(readBuffer, 0, bytesRead);
bytesRemaining -= bytesRead;
}
}
}
Console.WriteLine("Success");
}
else
{
Console.WriteLine("Success, (no data)");
}
}
else
{
Console.WriteLine("Failed");
}
break;
case "delete":
key = InputString("Object key:", null, false);
if (_Dedupe.DeleteObject(key))
{
Console.WriteLine("Success");
}
else
{
Console.WriteLine("Failed");
}
break;
case "metadata":
key = InputString("Object key:", null, false);
if (_Dedupe.RetrieveObjectMetadata(key, true, out md))
{
Console.WriteLine("Success");
Console.WriteLine(md.ToString());
}
else
{
Console.WriteLine("Failed");
}
break;
case "list":
_Dedupe.ListObjects(out keys);
if (keys != null && keys.Count > 0)
{
Console.WriteLine("Objects: ");
foreach (string curr in keys)
{
Console.WriteLine(" " + curr);
}
Console.WriteLine(keys.Count + " objects listed");
}
break;
case "exists":
key = InputString("Object name:", null, false);
if (_Dedupe.ObjectExists(key))
{
Console.WriteLine("Object exists");
}
else
{
Console.WriteLine("Object does not exist");
}
break;
case "stats":
if (_Dedupe.IndexStats(out numObjects, out numChunks, out logicalBytes, out physicalBytes, out dedupeRatioX, out dedupeRatioPercent))
{
Console.WriteLine("Statistics:");
Console.WriteLine(" Number of objects : " + numObjects);
Console.WriteLine(" Number of chunks : " + numChunks);
Console.WriteLine(" Logical bytes : " + logicalBytes + " bytes");
Console.WriteLine(" Physical bytes : " + physicalBytes + " bytes");
Console.WriteLine(" Dedupe ratio : " + DecimalToString(dedupeRatioX) + "X, " + DecimalToString(dedupeRatioPercent) + "%");
Console.WriteLine("");
}
else
{
Console.WriteLine("Failed");
}
break;
default:
break;
}
}
}
static void Main(string[] args)
{
try
{
#region Parse-Arguments
if (args == null || args.Length < 2)
{
Usage("No arguments specified");
return;
}
IndexFile = args[0];
Command = args[1];
for (int i = 2; i < args.Length; i++)
{
if (String.IsNullOrEmpty(args[i]))
{
continue;
}
if (args[i].StartsWith("--chunks=") && args[i].Length > 9)
{
ChunkDirectory = args[i].Substring(9);
if (!ChunkDirectory.EndsWith("\\"))
{
ChunkDirectory += "\\";
}
if (!Directory.Exists(ChunkDirectory))
{
Directory.CreateDirectory(ChunkDirectory);
}
}
else if (args[i].StartsWith("--key=") && args[i].Length > 6)
{
ObjectKey = args[i].Substring(6);
}
else if (String.Compare(args[i], "--debug") == 0)
{
DebugDedupe = true;
}
else if (String.Compare(args[i], "--debugsql") == 0)
{
DebugSql = true;
}
else if (args[i].StartsWith("--params=") && args[i].Length > 9)
{
CreateParams = args[i].Substring(9);
if (new Regex(@"^\d+,\d+,\d+,\d+$").IsMatch(CreateParams))
{
string[] currParams = CreateParams.Split(',');
if (currParams.Length != 4)
{
Usage("Value for 'params' is invalid");
return;
}
if (!Int32.TryParse(currParams[0], out MinChunkSize) ||
!Int32.TryParse(currParams[1], out MaxChunkSize) ||
!Int32.TryParse(currParams[2], out ShiftCount) ||
!Int32.TryParse(currParams[3], out BoundaryCheckBytes)
)
{
Usage("Value for 'params' is not of the form int,int,int,int");
return;
}
}
else
{
Usage("Value for 'params' is not of the form int,int,int,int");
return;
}
}
else
{
Usage("Unknown argument: " + args[i]);
return;
}
}
#endregion
#region Verify-Values
List <string> validCommands = new List <string>()
{
"create", "stats", "store", "retrieve", "delete", "list", "exists", "metadata"
};
if (!validCommands.Contains(Command))
{
Usage("Invalid command: " + Command);
return;
}
#endregion
#region Enumerate
if (DebugDedupe)
{
Console.WriteLine("Command : " + Command);
Console.WriteLine("Index File : " + IndexFile);
Console.WriteLine("Chunk Directory : " + ChunkDirectory);
Console.WriteLine("Object Key : " + ObjectKey);
if (MinChunkSize > 0)
{
Console.WriteLine("Min Chunk Size : " + MinChunkSize);
}
if (MaxChunkSize > 0)
{
Console.WriteLine("Max Chunk Size : " + MaxChunkSize);
}
if (ShiftCount > 0)
{
Console.WriteLine("Shift Count : " + ShiftCount);
}
if (BoundaryCheckBytes > 0)
{
Console.WriteLine("Boundary Bytes : " + BoundaryCheckBytes);
}
}
#endregion
#region Create
if (String.Compare(Command, "create") == 0)
{
Dedupe = new DedupeLibrary(IndexFile, MinChunkSize, MaxChunkSize, ShiftCount, BoundaryCheckBytes, WriteChunk, ReadChunk, DeleteChunk, DebugDedupe, DebugSql);
if (DebugDedupe)
{
Console.WriteLine("Successfully wrote new index: " + IndexFile);
}
return;
}
#endregion
#region Initialize-Index
if (!File.Exists(IndexFile))
{
Console.WriteLine("*** Index file " + IndexFile + " not found");
}
Dedupe = new DedupeLibrary(IndexFile, WriteChunk, ReadChunk, DeleteChunk, DebugDedupe, DebugSql);
#endregion
#region Process-by-Command
switch (Command)
{
case "stats":
if (Dedupe.IndexStats(out NumObjects, out NumChunks, out LogicalBytes, out PhysicalBytes, out DedupeRatioX, out DedupeRatioPercent))
{
Console.WriteLine("Statistics:");
Console.WriteLine(" Number of objects : " + NumObjects);
Console.WriteLine(" Number of chunks : " + NumChunks);
Console.WriteLine(" Logical bytes : " + LogicalBytes + " bytes");
Console.WriteLine(" Physical bytes : " + PhysicalBytes + " bytes");
Console.WriteLine(" Dedupe ratio : " + DecimalToString(DedupeRatioX) + "X, " + DecimalToString(DedupeRatioPercent) + "%");
return;
}
else
{
Console.WriteLine("Failed");
}
return;
case "retrieve":
if (String.IsNullOrEmpty(ObjectKey))
{
Usage("Object key must be supplied");
}
else
{
if (!Dedupe.RetrieveObject(ObjectKey, out ResponseData))
{
Console.WriteLine("Failed");
}
else
{
WriteConsoleData(ResponseData);
}
}
return;
case "store":
if (String.IsNullOrEmpty(ObjectKey))
{
Usage("Object key must be supplied");
}
else
{
if (Dedupe.ObjectExists(ObjectKey))
{
Console.WriteLine("Already exists");
}
else
{
ReadConsoleData();
if (!Dedupe.StoreObject(ObjectKey, RequestData, out Chunks))
{
Console.WriteLine("Failed");
}
else
{
Console.WriteLine("Success");
}
}
}
return;
case "delete":
if (String.IsNullOrEmpty(ObjectKey))
{
Usage("Object key must be supplied");
}
else
{
if (!Dedupe.DeleteObject(ObjectKey))
{
Console.WriteLine("Failed");
}
else
{
Console.WriteLine("Success");
}
}
return;
case "metadata":
if (String.IsNullOrEmpty(ObjectKey))
{
Usage("Object key must be supplied");
}
else
{
if (!Dedupe.RetrieveObjectMetadata(ObjectKey, out Metadata))
{
Console.WriteLine("Failed");
}
else
{
Console.WriteLine(Metadata.ToString());
}
}
return;
case "list":
Dedupe.ListObjects(out Keys);
if (Keys == null || Keys.Count < 1)
{
Console.WriteLine("No objects");
}
else
{
Console.WriteLine("Objects:");
foreach (string curr in Keys)
{
Console.WriteLine(" " + curr);
}
Console.WriteLine(Keys.Count + " objects in index");
}
return;
case "exists":
if (String.IsNullOrEmpty(ObjectKey))
{
Usage("Object key must be supplied");
return;
}
else
{
Console.WriteLine(Dedupe.ObjectExists(ObjectKey));
}
return;
default:
Usage("Unknown command: " + Command);
return;
}
#endregion
}
catch (Exception e)
{
ExceptionConsole("Dedupe", "Outer exception", e);
}
}
public override void OnServerDataReceived(object sender, TCP.DataEventArgs e)
{
if (e.Request.Error != Socks.SocksError.Granted || e.Buffer.Length == 0)
{
Console.Write("<DeDupe> Error:" + e.Request.Error + " Length:" + e.Buffer.Length.ToString() + Environment.NewLine);
return;
}
requestData.packetSize = e.Count;
// Special handling for HTTP
if (requestData.isHTTP)
{
// Handle HTTP header gracefully
if (DataHandlerDeDupe.IsHTTPResponse200(e.Buffer) || requestData.isHeader)
{
requestData.isHeader = true;
requestData.isChunked = DataHandlerDeDupe.IsChunked(e.Buffer);
requestData.contentLength = GetContentLength(e.Buffer);
requestData.totalPackets = Math.Ceiling((float)(requestData.contentLength) / (float)packetSize);
Console.Write(Environment.NewLine + "<DeDupe> [Server response] HTTP - RequestType:" + requestData.requestType.ToString() + " Address:" + requestData.host + " Port:" + requestData.port.ToString() + " URL:" + requestData.uRI + Environment.NewLine);
Console.Write("<DeDupe> [Server response] HTTP Header - " + "ContentLength:" + requestData.contentLength + " ExpectedPackets:" + requestData.totalPackets + " Chunked:" + DataHandlerDeDupe.IsChunked(e.Buffer).ToString() + Environment.NewLine);
requestData.startPayload = e.Buffer.FindString("\r\n\r\n");
if (requestData.startPayload != -1)
{
requestData.startPayload += 4;
requestData.currentposition = 0;
requestData.isHeader = false;
}
if (requestData.contentLength != 0)
{
stripeBuffer = new ReadWriteBuffer(requestData.contentLength + 1);
}
requestData.currentRound = 0;
}
//else requestData.startPayload = 0;
///END
if (requestData.startPayload > 0)
{
//int stripeCount = stripeBuffer.Count;
//int stripeSpaceRemaining = stripeSize - stripeCount - 1;
int payloadLength = Math.Min((requestData.contentLength - requestData.currentposition), requestData.packetSize - requestData.startPayload);
int contentRemaining = requestData.contentLength - requestData.currentposition;
if (requestData.isChunked)
{
//Console.Write("Chunk Size = " + Chunked.GetChunkSize(e.Buffer, 10).ToString() + Environment.NewLine);
//e.Buffer = Chunked.GetChunkData(e.Buffer,
//.GetChunkSize(e.Buffer, e.Count));
//Chunked chunked = new Chunked(f, e.Buffer, e.Buffer.Length)
}
else
{
/*xxHash hash0 = new xxHash();
* hash0.Init();
* hash0.Update(e.Buffer, e.Count);
* Key = hash0.Digest().ToString();*/
//if (Dedupe.StoreObject(Key, e.Buffer, out Chunks)) Console.WriteLine("<DeDupe> Stored: " + Key + Environment.NewLine);
Key = "HTTP/" + WebUtility.UrlEncode(requestData.host + requestData.port + requestData.uRI);
if (Dedupe.ObjectExists(Key))
{
Console.Write("x");
}
else
{
// Stream entire payload into buffer to maximise efficiency
int writeLength = Math.Min(e.Count - requestData.startPayload, (requestData.contentLength - requestData.currentposition));
if (contentRemaining > 0)
{
byte[] shortBuffer = new byte[writeLength];
try
{
shortBuffer = e.Buffer.GetInBetween(requestData.startPayload, requestData.startPayload + writeLength);
requestData.startPayload = 0;
try
{
Console.Write(Environment.NewLine + "---------------------------------------" + Key + Environment.NewLine + shortBuffer.GetBetween(0, Math.Max(shortBuffer.Length, 10)) + Environment.NewLine + "---------------------------------------" + Environment.NewLine);
stripeBuffer.Write(shortBuffer);
requestData.currentposition += shortBuffer.Length;
//Console.WriteLine("---!!----" + shortBuffer.Length + "/" + stripeBuffer.Count + "---------!!----");
}
catch
{
Console.Write("<DeDupe> bufferLength:" + stripeBuffer.Count + " shortBufferLength:" + shortBuffer.Length + " currentposition:" + requestData.currentposition + " contentRemaining:" + contentRemaining + Environment.NewLine);
throw new Exception("<DeDupe> Write ERROR to resourceBuffer");
}
}
catch
{
Console.Write("<DeDupe> writeLength:" + writeLength + " shortBufferLength:" + shortBuffer + " bufferCount:" + e.Count + " length:" + e.Buffer.Length + " startPayload:" + requestData.startPayload + Environment.NewLine);
throw new Exception("<DeDupe> Write ERROR to shortBuffer");
}
Console.Write("-");
}
// When buffer is full
if ((contentRemaining == writeLength) && (stripeBuffer.Count != 0))
{
// Open a stream if none already exists
if (requestData.cacheStream == null)
{
Key = "HTTP/" + WebUtility.UrlEncode(requestData.host + requestData.port + requestData.uRI);
//byte[] derp = new byte[stripeBuffer.Count];
//derp = stripeBuffer.Read(stripeBuffer.Count);
//Console.WriteLine("---!!----" + stripeBuffer.Count + "---------!!----");
if (Dedupe == null)
{
throw new Exception("<DeDupe> Library pointer == null");
}
try
{
int derpCount = stripeBuffer.Count;
byte[] derp = new byte[derpCount];
derp = stripeBuffer.Read(derpCount);
//Console.Write(Environment.NewLine + "---------------------------------------" + Environment.NewLine + derp.GetBetween(0, derpCount) + Environment.NewLine + "---------------------------------------" + Environment.NewLine);
Dedupe.StoreObject(Key, derp, out Chunks);
}
catch
{
Console.Write(Environment.NewLine + "<DeDupe> [ERROR] Failed to open output stream!" + Environment.NewLine + " Key Named:" + Key + Environment.NewLine);
Console.Write(" Key Found:" + Dedupe.ObjectExists(Key).ToString() + Environment.NewLine);
Console.Write("contentRemaining:" + contentRemaining + " writeLength:" + writeLength + " stripeBufferCount:" + stripeBuffer.Count + Environment.NewLine);
throw new Exception(Environment.NewLine + "<DeDupe> [FATAL ERROR] Unable to write to cache:" + Key + Environment.NewLine);
}
}
Console.Write(Environment.NewLine + "<DeDupe> (STORED) length:" + requestData.contentLength + " key:" + Key + Environment.NewLine);
}
}
}
//We maintain this per 4k socks packet.
// Update position
//requestData.currentposition += payloadLength;
}
}
else
{
Console.Write("<DeDupe> [Server response] DATA - RequestType:" + requestData.requestType.ToString() + " - Chunked:" + DataHandlerDeDupe.IsChunked(e.Buffer).ToString() + " Address:" + requestData.host + " Port:" + requestData.port.ToString() + Environment.NewLine);
}
//if data is HTTP, make sure it's not compressed so we can capture it in plaintext.
/*if (e.Buffer.FindString(" HTTP/1.1") != -1 && e.Buffer.FindString("Accept-Encoding") != -1)
* {
* int x = e.Buffer.FindString("Accept-Encoding:");
* int y = e.Buffer.FindString("\r\n", x + 1);
* e.Buffer = e.Buffer.ReplaceBetween(x, y, Encoding.ASCII.GetBytes(replaceWith));
* e.Buffer = e.Buffer.ReplaceString("HTTP/1.1", "HTTP/1.0");
* e.Count = e.Count - (y - x) + replaceWith.Length;
* }*/
}
