예제 #1
0
        static void Main(string[] args)
        {
            // Create chunk directory
            if (!Directory.Exists("chunks"))
            {
                Directory.CreateDirectory("chunks");
            }

            // Define settings, callbacks, and initialize
            DedupeSettings  settings  = new DedupeSettings(32768, 262144, 2048, 2);
            DedupeCallbacks callbacks = new DedupeCallbacks(WriteChunk, ReadChunk, DeleteChunk);
            DedupeLibrary   dedupe    = new DedupeLibrary("test.db", settings, callbacks);

            // Store objects in the index
            dedupe.Write("kjv1", File.ReadAllBytes("samplefiles/kjv.txt"));
            dedupe.Write("kjv2", File.ReadAllBytes("samplefiles/kjv.txt"));
            dedupe.Write("kjv3", File.ReadAllBytes("samplefiles/kjv.txt"));

            // Check existence and retrieve an object from the index
            if (dedupe.Exists("kjv2"))
            {
                Console.WriteLine("Exists");
            }
            DedupeObject obj = dedupe.Get("kjv1");

            // List all objects
            Console.WriteLine(dedupe.ListObjects().ToTabularString());

            // Display index statistics
            Console.WriteLine(dedupe.IndexStats().ToString());

            // Delete an object from the index
            dedupe.Delete("kjv1");
        }
예제 #2
0
        static void Main(string[] args)
        {
            bool         runForever = true;
            string       filename   = "";
            string       key        = "";
            List <Chunk> chunks     = new List <Chunk>();
            DedupeObject obj        = null;

            Initialize();

            while (runForever)
            {
                Console.Write("Command [? for help] > ");
                string userInput = Console.ReadLine();
                if (String.IsNullOrEmpty(userInput))
                {
                    continue;
                }

                switch (userInput)
                {
                case "?":
                    Console.WriteLine("Available commands:");
                    Console.WriteLine("  q          quit");
                    Console.WriteLine("  cls        clear the screen");
                    Console.WriteLine("  write      store an object");
                    Console.WriteLine("  get        retrieve an object");
                    Console.WriteLine("  del        delete an object");
                    Console.WriteLine("  md         retrieve object metadata");
                    Console.WriteLine("  list       list 100 objects in the index");
                    Console.WriteLine("  listp      paginated list objects");
                    Console.WriteLine("  exists     check if object exists in the index");
                    Console.WriteLine("  stats      list index stats");
                    Console.WriteLine("");
                    break;

                case "q":
                case "Q":
                    runForever = false;
                    break;

                case "cls":
                    Console.Clear();
                    break;

                case "write":
                    filename = InputString("Input filename:", null, false);
                    key      = InputString("Object key:", null, false);
                    long contentLength = GetContentLength(filename);
                    using (FileStream fs = new FileStream(filename, FileMode.Open))
                    {
                        _Dedupe.Write(key, contentLength, fs);
                    }
                    break;

                case "get":
                    key      = InputString("Object key:", null, false);
                    filename = InputString("Output filename:", null, false);
                    obj      = _Dedupe.Get(key);
                    if (obj != null)
                    {
                        if (obj.Length > 0)
                        {
                            using (FileStream fs = new FileStream(filename, FileMode.OpenOrCreate))
                            {
                                int    bytesRead      = 0;
                                long   bytesRemaining = obj.Length;
                                byte[] readBuffer     = new byte[65536];

                                while (bytesRemaining > 0)
                                {
                                    bytesRead = obj.DataStream.Read(readBuffer, 0, readBuffer.Length);
                                    if (bytesRead > 0)
                                    {
                                        fs.Write(readBuffer, 0, bytesRead);
                                        bytesRemaining -= bytesRead;
                                    }
                                }
                            }

                            Console.WriteLine("Success");
                        }
                        else
                        {
                            Console.WriteLine("Success, (no data)");
                        }
                    }
                    else
                    {
                        Console.WriteLine("Failed");
                    }
                    break;

                case "del":
                    key = InputString("Object key:", null, false);
                    _Dedupe.Delete(key);
                    break;

                case "md":
                    key = InputString("Object key:", null, false);
                    obj = _Dedupe.GetMetadata(key);
                    if (obj != null)
                    {
                        Console.WriteLine("Success");
                        Console.WriteLine(obj.ToString());
                    }
                    else
                    {
                        Console.WriteLine("Failed");
                    }
                    break;

                case "list":
                    _EnumResult = _Dedupe.ListObjects();
                    if (_EnumResult == null)
                    {
                        Console.WriteLine("No objects");
                    }
                    else
                    {
                        Console.WriteLine(_EnumResult.ToTabularString());
                    }
                    break;

                case "exists":
                    key = InputString("Object key:", null, false);
                    if (_Dedupe.Exists(key))
                    {
                        Console.WriteLine("Object exists");
                    }
                    else
                    {
                        Console.WriteLine("Object does not exist");
                    }
                    break;

                case "stats":
                    _Stats = _Dedupe.IndexStats();
                    if (_Stats != null)
                    {
                        Console.WriteLine("Statistics:");
                        Console.WriteLine("  Number of objects : " + _Stats.Objects);
                        Console.WriteLine("  Number of chunks  : " + _Stats.Chunks);
                        Console.WriteLine("  Logical bytes     : " + _Stats.LogicalBytes + " bytes");
                        Console.WriteLine("  Physical bytes    : " + _Stats.PhysicalBytes + " bytes");
                        Console.WriteLine("  Dedupe ratio      : " + DecimalToString(_Stats.RatioX) + "X, " + DecimalToString(_Stats.RatioPercent) + "%");
                        Console.WriteLine("");
                    }
                    else
                    {
                        Console.WriteLine("Failed");
                    }
                    break;

                default:
                    break;
                }
            }
        }
예제 #3
0
        static void Main(string[] args)
        {
            try
            {
                #region Parse-Arguments

                if (args == null || args.Length < 2)
                {
                    Usage("No arguments specified");
                    return;
                }

                _IndexFile = args[0];
                _Command   = args[1];

                for (int i = 2; i < args.Length; i++)
                {
                    if (String.IsNullOrEmpty(args[i]))
                    {
                        continue;
                    }
                    if (args[i].StartsWith("--chunks=") && args[i].Length > 9)
                    {
                        _ChunkDir = args[i].Substring(9);
                        if (!_ChunkDir.EndsWith("\\"))
                        {
                            _ChunkDir += "\\";
                        }
                        if (!Directory.Exists(_ChunkDir))
                        {
                            Directory.CreateDirectory(_ChunkDir);
                        }
                    }
                    else if (args[i].StartsWith("--key=") && args[i].Length > 6)
                    {
                        _Key = args[i].Substring(6);
                    }
                    else if (args[i].StartsWith("--idxstart=") && args[i].Length > 11)
                    {
                        if (!Int32.TryParse(args[i].Substring(11), out _IndexStart))
                        {
                            Usage("Index start must be an integer value.");
                            return;
                        }
                        else
                        {
                            if (_IndexStart < 0)
                            {
                                Usage("Index start must be greater than zero.");
                                return;
                            }
                        }
                    }
                    else if (args[i].StartsWith("--results=") && args[i].Length > 10)
                    {
                        if (!Int32.TryParse(args[i].Substring(10), out _MaxResults))
                        {
                            Usage("Max results must be an integer value.");
                            return;
                        }
                        else
                        {
                            if (_MaxResults < 1 || _MaxResults > 100)
                            {
                                Usage("Max results must be greater than zero and less than or equal to 100.");
                                return;
                            }
                        }
                    }
                    else if (args[i].StartsWith("--params=") && args[i].Length > 9)
                    {
                        _CreateParams = args[i].Substring(9);
                        if (new Regex(@"^\d+,\d+,\d+,\d+$").IsMatch(_CreateParams))
                        {
                            string[] currParams = _CreateParams.Split(',');
                            if (currParams.Length != 4)
                            {
                                Usage("Value for 'params' is invalid");
                                return;
                            }

                            if (!Int32.TryParse(currParams[0], out _MinChunkSize) ||
                                !Int32.TryParse(currParams[1], out _MaxChunkSize) ||
                                !Int32.TryParse(currParams[2], out _ShiftCount) ||
                                !Int32.TryParse(currParams[3], out _BoundaryCheckBytes)
                                )
                            {
                                Usage("Value for 'params' is not of the form int,int,int,int");
                                return;
                            }
                        }
                        else
                        {
                            Usage("Value for 'params' is not of the form int,int,int,int");
                            return;
                        }
                    }
                    else
                    {
                        Usage("Unknown argument: " + args[i]);
                        return;
                    }
                }

                #endregion

                #region Verify-Values

                List <string> validCommands = new List <string>()
                {
                    "create", "stats", "write", "get", "del", "list", "exists", "md"
                };
                if (!validCommands.Contains(_Command))
                {
                    Usage("Invalid command: " + _Command);
                    return;
                }

                #endregion

                #region Create

                if (String.Compare(_Command, "create") == 0)
                {
                    _Settings  = new DedupeSettings(_MinChunkSize, _MaxChunkSize, _ShiftCount, _BoundaryCheckBytes);
                    _Callbacks = new DedupeCallbacks(WriteChunk, ReadChunk, DeleteChunk);
                    _Dedupe    = new DedupeLibrary(_IndexFile, _Settings, _Callbacks);
                    return;
                }

                #endregion

                #region Initialize-Index

                if (!File.Exists(_IndexFile))
                {
                    Console.WriteLine("*** Index file " + _IndexFile + " not found");
                }

                _Settings  = new DedupeSettings();
                _Callbacks = new DedupeCallbacks(WriteChunk, ReadChunk, DeleteChunk);
                _Dedupe    = new DedupeLibrary(_IndexFile, _Settings, _Callbacks);

                #endregion

                #region Process-by-Command

                switch (_Command)
                {
                case "stats":
                    _Stats = _Dedupe.IndexStats();
                    if (_Stats != null)
                    {
                        Console.WriteLine("Statistics:");
                        Console.WriteLine("  Number of objects : " + _Stats.Objects);
                        Console.WriteLine("  Number of chunks  : " + _Stats.Chunks);
                        Console.WriteLine("  Logical bytes     : " + _Stats.LogicalBytes + " bytes");
                        Console.WriteLine("  Physical bytes    : " + _Stats.PhysicalBytes + " bytes");
                        Console.WriteLine("  Dedupe ratio      : " + DecimalToString(_Stats.RatioX) + "X, " + DecimalToString(_Stats.RatioPercent) + "%");
                        return;
                    }
                    else
                    {
                        Console.WriteLine("Failed");
                    }
                    return;

                case "get":
                    if (String.IsNullOrEmpty(_Key))
                    {
                        Usage("Object key must be supplied");
                    }
                    else
                    {
                        _Object = _Dedupe.Get(_Key);
                        if (_Object == null)
                        {
                            Console.WriteLine("Failed");
                        }
                        else
                        {
                            WriteConsoleData(_Object.Data);
                        }
                    }
                    return;

                case "write":
                    if (String.IsNullOrEmpty(_Key))
                    {
                        Usage("Object key must be supplied");
                    }
                    else
                    {
                        if (_Dedupe.Exists(_Key))
                        {
                            Console.WriteLine("Already exists");
                        }
                        else
                        {
                            ReadConsoleData();
                            _Dedupe.Write(_Key, _Request);
                        }
                    }
                    return;

                case "del":
                    if (String.IsNullOrEmpty(_Key))
                    {
                        Usage("Object key must be supplied");
                    }
                    else
                    {
                        _Dedupe.Delete(_Key);
                    }
                    return;

                case "md":
                    if (String.IsNullOrEmpty(_Key))
                    {
                        Usage("Object key must be supplied");
                    }
                    else
                    {
                        _Object = _Dedupe.GetMetadata(_Key);
                        if (_Object == null)
                        {
                            Console.WriteLine("Failed");
                        }
                        else
                        {
                            Console.WriteLine(_Object.ToString());
                        }
                    }
                    return;

                case "list":
                    _EnumResult = _Dedupe.ListObjects(_Key, _IndexStart, _MaxResults);
                    if (_EnumResult == null)
                    {
                        Console.WriteLine("No objects");
                    }
                    else
                    {
                        Console.WriteLine(_EnumResult.ToTabularString());
                    }
                    return;

                case "exists":
                    if (String.IsNullOrEmpty(_Key))
                    {
                        Usage("Object key must be supplied");
                        return;
                    }
                    else
                    {
                        Console.WriteLine(_Dedupe.Exists(_Key));
                    }
                    return;

                default:
                    Usage("Unknown command: " + _Command);
                    return;
                }

                #endregion
            }
            catch (Exception e)
            {
                ExceptionConsole("Dedupe", "Outer exception", e);
            }
        }