static void Enumerate()
{
// Search(string indexName, SearchQuery query, out SearchResult result)
string indexName = InputString("Index name:", null, true);
if (String.IsNullOrEmpty(indexName))
{
return;
}
EnumerationQuery eq = new EnumerationQuery();
string filename = InputString("Enumeration filename:", "", true);
if (!String.IsNullOrEmpty(filename))
{
eq = DeserializeJson <EnumerationQuery>(File.ReadAllBytes(filename));
}
EnumerationResult result = _Sdk.Enumerate(indexName, eq).Result;
if (result != null)
{
Console.WriteLine(SerializeJson(result, true));
}
}
static void EnumeratePrefix()
{
EnumerationResult result = _Blobs.Enumerate(
InputString("Prefix:", null, true),
InputString("Continuation token:", null, true)).Result;
if (result.Blobs != null && result.Blobs.Count > 0)
{
foreach (BlobMetadata curr in result.Blobs)
{
Console.WriteLine(
String.Format("{0,-27}", curr.Key) +
String.Format("{0,-18}", curr.ContentLength.ToString() + " bytes") +
String.Format("{0,-30}", curr.Created.ToString("yyyy-MM-dd HH:mm:ss")));
}
}
else
{
Console.WriteLine("(none)");
}
if (!String.IsNullOrEmpty(result.NextContinuationToken))
{
Console.WriteLine("Continuation token: " + result.NextContinuationToken);
}
}
/// <summary>
/// List the objects stored in the index.
/// </summary>
/// <param name="prefix">Prefix upon which to match object keys.</param>
/// <param name="indexStart">The index (DedupeObject.Id) from which to begin the enumeration.</param>
/// <param name="maxResults">Maximum number of results to retrieve.</param>
/// <return>Enumeration result.</return>
public override EnumerationResult ListObjects(string prefix, int indexStart, int maxResults)
{
if (indexStart < 0)
{
throw new ArgumentException("Starting index must be zero or greater.");
}
if (maxResults < 1 || maxResults > 100)
{
throw new ArgumentException("Max results must be greater than zero and less than or equal to 100.");
}
EnumerationResult ret = new EnumerationResult(prefix, indexStart, indexStart, maxResults, new List <DedupeObject>());
DbExpression e = new DbExpression(
_ORM.GetColumnName <DedupeObject>(nameof(DedupeObject.Id)),
DbOperators.GreaterThan,
indexStart);
if (!String.IsNullOrEmpty(prefix))
{
e.PrependAnd(
_ORM.GetColumnName <DedupeObject>(nameof(DedupeObject.Key)),
DbOperators.StartsWith,
prefix);
}
List <DedupeObject> objects = _ORM.SelectMany <DedupeObject>(null, maxResults, e);
if (objects != null && objects.Count > 0)
{
foreach (DedupeObject obj in objects)
{
obj.Chunks = GetChunks(obj.Key);
obj.ObjectMap = GetObjectMap(obj.Key);
if (obj.ObjectMap != null && obj.ObjectMap.Count > 0)
{
obj.ObjectMap = obj.ObjectMap.OrderBy(o => o.ChunkAddress).ToList();
}
ret.Objects.Add(obj);
}
}
if (objects != null && objects.Count == maxResults)
{
ret.NextIndexStart = objects[(objects.Count - 1)].Id;
}
return(ret);
}
private static async Task PutEnumerateIndex(RequestMetadata md)
{
string header = "[Komodo.Server] " + md.Http.Request.Source.IpAddress + ":" + md.Http.Request.Source.Port + " PutEnumerateIndex ";
if (md.Http.Request.Data == null || md.Http.Request.ContentLength < 1)
{
md.Http.Response.StatusCode = 400;
md.Http.Response.ContentType = "application/json";
await md.Http.Response.Send(new ErrorResponse(400, "No request body.", null, null).ToJson(true));
return;
}
string indexName = md.Http.Request.Url.Elements[0];
if (!_Daemon.IndexExists(indexName))
{
_Logging.Warn(header + "index " + indexName + " does not exist");
md.Http.Response.StatusCode = 404;
md.Http.Response.ContentType = "application/json";
await md.Http.Response.Send(new ErrorResponse(404, "Unknown index.", null, null).ToJson(true));
return;
}
EnumerationQuery query = Common.DeserializeJson <EnumerationQuery>(Common.StreamToBytes(md.Http.Request.Data));
if (query.Filters == null)
{
query.Filters = new List <SearchFilter>();
}
EnumerationResult result = _Daemon.Enumerate(indexName, query);
if (!result.Success)
{
_Logging.Warn(header + "failed to execute enumeration in index " + indexName);
md.Http.Response.StatusCode = 500;
md.Http.Response.ContentType = "application/json";
await md.Http.Response.Send(new ErrorResponse(500, "Unable to enumerate index '" + indexName + "'.", null, result).ToJson(true));
return;
}
md.Http.Response.StatusCode = 200;
md.Http.Response.ContentType = "application/json";
await md.Http.Response.Send(Common.SerializeJson(result, md.Params.Pretty));
return;
}
static void Enumerate()
{
string indexName = Common.InputString("Index name :", null, false);
EnumerationQuery query = BuildEnumerationQuery();
EnumerationResult result = _Komodo.Enumerate(indexName, query);
if (result == null)
{
Console.WriteLine("(none)");
}
else
{
Console.WriteLine(Common.SerializeJson(result, true));
}
}
static void Main(string[] args)
{
bool runForever = true;
string filename = "";
string key = "";
List <Chunk> chunks = new List <Chunk>();
DedupeObject obj = null;
Initialize();
while (runForever)
{
Console.Write("Command [? for help] > ");
string userInput = Console.ReadLine();
if (String.IsNullOrEmpty(userInput))
{
continue;
}
switch (userInput)
{
case "?":
Console.WriteLine("Available commands:");
Console.WriteLine(" q quit");
Console.WriteLine(" cls clear the screen");
Console.WriteLine(" write store an object");
Console.WriteLine(" get retrieve an object");
Console.WriteLine(" del delete an object");
Console.WriteLine(" md retrieve object metadata");
Console.WriteLine(" list list 100 objects in the index");
Console.WriteLine(" listp paginated list objects");
Console.WriteLine(" exists check if object exists in the index");
Console.WriteLine(" stats list index stats");
Console.WriteLine("");
break;
case "q":
case "Q":
runForever = false;
break;
case "cls":
Console.Clear();
break;
case "write":
filename = InputString("Input filename:", null, false);
key = InputString("Object key:", null, false);
long contentLength = GetContentLength(filename);
using (FileStream fs = new FileStream(filename, FileMode.Open))
{
_Dedupe.Write(key, contentLength, fs);
}
break;
case "get":
key = InputString("Object key:", null, false);
filename = InputString("Output filename:", null, false);
obj = _Dedupe.Get(key);
if (obj != null)
{
if (obj.Length > 0)
{
using (FileStream fs = new FileStream(filename, FileMode.OpenOrCreate))
{
int bytesRead = 0;
long bytesRemaining = obj.Length;
byte[] readBuffer = new byte[65536];
while (bytesRemaining > 0)
{
bytesRead = obj.DataStream.Read(readBuffer, 0, readBuffer.Length);
if (bytesRead > 0)
{
fs.Write(readBuffer, 0, bytesRead);
bytesRemaining -= bytesRead;
}
}
}
Console.WriteLine("Success");
}
else
{
Console.WriteLine("Success, (no data)");
}
}
else
{
Console.WriteLine("Failed");
}
break;
case "del":
key = InputString("Object key:", null, false);
_Dedupe.Delete(key);
break;
case "md":
key = InputString("Object key:", null, false);
obj = _Dedupe.GetMetadata(key);
if (obj != null)
{
Console.WriteLine("Success");
Console.WriteLine(obj.ToString());
}
else
{
Console.WriteLine("Failed");
}
break;
case "list":
_EnumResult = _Dedupe.ListObjects();
if (_EnumResult == null)
{
Console.WriteLine("No objects");
}
else
{
Console.WriteLine(_EnumResult.ToTabularString());
}
break;
case "exists":
key = InputString("Object key:", null, false);
if (_Dedupe.Exists(key))
{
Console.WriteLine("Object exists");
}
else
{
Console.WriteLine("Object does not exist");
}
break;
case "stats":
_Stats = _Dedupe.IndexStats();
if (_Stats != null)
{
Console.WriteLine("Statistics:");
Console.WriteLine(" Number of objects : " + _Stats.Objects);
Console.WriteLine(" Number of chunks : " + _Stats.Chunks);
Console.WriteLine(" Logical bytes : " + _Stats.LogicalBytes + " bytes");
Console.WriteLine(" Physical bytes : " + _Stats.PhysicalBytes + " bytes");
Console.WriteLine(" Dedupe ratio : " + DecimalToString(_Stats.RatioX) + "X, " + DecimalToString(_Stats.RatioPercent) + "%");
Console.WriteLine("");
}
else
{
Console.WriteLine("Failed");
}
break;
default:
break;
}
}
}
static void Main(string[] args)
{
try
{
#region Parse-Arguments
if (args == null || args.Length < 2)
{
Usage("No arguments specified");
return;
}
_IndexFile = args[0];
_Command = args[1];
for (int i = 2; i < args.Length; i++)
{
if (String.IsNullOrEmpty(args[i]))
{
continue;
}
if (args[i].StartsWith("--chunks=") && args[i].Length > 9)
{
_ChunkDir = args[i].Substring(9);
if (!_ChunkDir.EndsWith("\\"))
{
_ChunkDir += "\\";
}
if (!Directory.Exists(_ChunkDir))
{
Directory.CreateDirectory(_ChunkDir);
}
}
else if (args[i].StartsWith("--key=") && args[i].Length > 6)
{
_Key = args[i].Substring(6);
}
else if (args[i].StartsWith("--idxstart=") && args[i].Length > 11)
{
if (!Int32.TryParse(args[i].Substring(11), out _IndexStart))
{
Usage("Index start must be an integer value.");
return;
}
else
{
if (_IndexStart < 0)
{
Usage("Index start must be greater than zero.");
return;
}
}
}
else if (args[i].StartsWith("--results=") && args[i].Length > 10)
{
if (!Int32.TryParse(args[i].Substring(10), out _MaxResults))
{
Usage("Max results must be an integer value.");
return;
}
else
{
if (_MaxResults < 1 || _MaxResults > 100)
{
Usage("Max results must be greater than zero and less than or equal to 100.");
return;
}
}
}
else if (args[i].StartsWith("--params=") && args[i].Length > 9)
{
_CreateParams = args[i].Substring(9);
if (new Regex(@"^\d+,\d+,\d+,\d+$").IsMatch(_CreateParams))
{
string[] currParams = _CreateParams.Split(',');
if (currParams.Length != 4)
{
Usage("Value for 'params' is invalid");
return;
}
if (!Int32.TryParse(currParams[0], out _MinChunkSize) ||
!Int32.TryParse(currParams[1], out _MaxChunkSize) ||
!Int32.TryParse(currParams[2], out _ShiftCount) ||
!Int32.TryParse(currParams[3], out _BoundaryCheckBytes)
)
{
Usage("Value for 'params' is not of the form int,int,int,int");
return;
}
}
else
{
Usage("Value for 'params' is not of the form int,int,int,int");
return;
}
}
else
{
Usage("Unknown argument: " + args[i]);
return;
}
}
#endregion
#region Verify-Values
List <string> validCommands = new List <string>()
{
"create", "stats", "write", "get", "del", "list", "exists", "md"
};
if (!validCommands.Contains(_Command))
{
Usage("Invalid command: " + _Command);
return;
}
#endregion
#region Create
if (String.Compare(_Command, "create") == 0)
{
_Settings = new DedupeSettings(_MinChunkSize, _MaxChunkSize, _ShiftCount, _BoundaryCheckBytes);
_Callbacks = new DedupeCallbacks(WriteChunk, ReadChunk, DeleteChunk);
_Dedupe = new DedupeLibrary(_IndexFile, _Settings, _Callbacks);
return;
}
#endregion
#region Initialize-Index
if (!File.Exists(_IndexFile))
{
Console.WriteLine("*** Index file " + _IndexFile + " not found");
}
_Settings = new DedupeSettings();
_Callbacks = new DedupeCallbacks(WriteChunk, ReadChunk, DeleteChunk);
_Dedupe = new DedupeLibrary(_IndexFile, _Settings, _Callbacks);
#endregion
#region Process-by-Command
switch (_Command)
{
case "stats":
_Stats = _Dedupe.IndexStats();
if (_Stats != null)
{
Console.WriteLine("Statistics:");
Console.WriteLine(" Number of objects : " + _Stats.Objects);
Console.WriteLine(" Number of chunks : " + _Stats.Chunks);
Console.WriteLine(" Logical bytes : " + _Stats.LogicalBytes + " bytes");
Console.WriteLine(" Physical bytes : " + _Stats.PhysicalBytes + " bytes");
Console.WriteLine(" Dedupe ratio : " + DecimalToString(_Stats.RatioX) + "X, " + DecimalToString(_Stats.RatioPercent) + "%");
return;
}
else
{
Console.WriteLine("Failed");
}
return;
case "get":
if (String.IsNullOrEmpty(_Key))
{
Usage("Object key must be supplied");
}
else
{
_Object = _Dedupe.Get(_Key);
if (_Object == null)
{
Console.WriteLine("Failed");
}
else
{
WriteConsoleData(_Object.Data);
}
}
return;
case "write":
if (String.IsNullOrEmpty(_Key))
{
Usage("Object key must be supplied");
}
else
{
if (_Dedupe.Exists(_Key))
{
Console.WriteLine("Already exists");
}
else
{
ReadConsoleData();
_Dedupe.Write(_Key, _Request);
}
}
return;
case "del":
if (String.IsNullOrEmpty(_Key))
{
Usage("Object key must be supplied");
}
else
{
_Dedupe.Delete(_Key);
}
return;
case "md":
if (String.IsNullOrEmpty(_Key))
{
Usage("Object key must be supplied");
}
else
{
_Object = _Dedupe.GetMetadata(_Key);
if (_Object == null)
{
Console.WriteLine("Failed");
}
else
{
Console.WriteLine(_Object.ToString());
}
}
return;
case "list":
_EnumResult = _Dedupe.ListObjects(_Key, _IndexStart, _MaxResults);
if (_EnumResult == null)
{
Console.WriteLine("No objects");
}
else
{
Console.WriteLine(_EnumResult.ToTabularString());
}
return;
case "exists":
if (String.IsNullOrEmpty(_Key))
{
Usage("Object key must be supplied");
return;
}
else
{
Console.WriteLine(_Dedupe.Exists(_Key));
}
return;
default:
Usage("Unknown command: " + _Command);
return;
}
#endregion
}
catch (Exception e)
{
ExceptionConsole("Dedupe", "Outer exception", e);
}
}