Beispiel #1
0
 /// <summary>
 /// Ctor for FileReader
 /// </summary>
 /// <param name="fromFile">File name to read data from</param>
 /// <param name="toDictionary">Collection to store data in</param>
 /// <param name="maxChunksInDict">How many chunks may the collection store at a time (used for not running out of memory)</param>
 /// <param name="chunkSize">It will read data from file in chunks of this size (Bytes). It will add data to collection in chunks of same size if not in magicNumerMode</param>
 /// <param name="magicNumberMode">If this is set to True, will assume that data has GZip headers and scan for those instead of conserving chunks as is.
 /// Will add data to collection in variable size chunks</param>
 public FileReader(string fromFile, MySafeDictionary <int, byte[]> toDictionary, int maxChunksInDict, int chunkSize, bool magicNumberMode = false)
 {
     _fileName        = fromFile;
     _dict            = toDictionary;
     _magicNumberMode = magicNumberMode;
     _chunkSize       = chunkSize;
     _maxChunks       = maxChunksInDict;
 }
Beispiel #2
0
 /// <summary>
 /// Create an instance of Compressor, with all the data it needs in order to process.
 /// </summary>
 /// <param name="fromDictionary">The data to compress/decompess in a collection</param>
 /// <param name="toDictionary">The collection to put resulting data in</param>
 /// <param name="maxThreads">How many worker threads to use while processing data</param>
 /// <param name="throttleOutputCount">Will throttle processing if resulting output collection has this many items. Useful if processing is too fast, writing too slow</param>
 /// <param name="mode">Compress or decompress</param>
 public Compressor(MySafeDictionary <int, byte[]> fromDictionary, MySafeDictionary <int, byte[]> toDictionary, int maxThreads, int throttleOutputCount, CompressionMode mode)
 {
     _inputDict           = fromDictionary;
     _outputDict          = toDictionary;
     _maxThreads          = maxThreads;
     _mode                = mode;
     _threads             = new Thread[maxThreads];
     _throttleOutputCount = throttleOutputCount;
 }
Beispiel #3
0
        // We await input of format GZipTest.exe compress/decompress FromFile ToFile
        private static void Main(string[] args)
        {
            if (!ParseInput(args))
            {
                Console.ReadLine();
                return;
            }
            // Let's figure out technical parameters
            ComputerInfo CI       = new ComputerInfo();
            ulong        mem      = CI.AvailablePhysicalMemory;
            ulong        vmem     = CI.AvailableVirtualMemory;
            int          cpuCount = Environment.ProcessorCount;

            _maxThreads      = cpuCount;
            _maxInputChunks  = Convert.ToInt32((mem * 0.45) / (_chunkSize));
            _maxOutputChunks = _maxInputChunks;
            // Seems like all input was correct, let's continue with actual work
            DateTime start = DateTime.Now;

            Console.WriteLine($"{_mode.ToString()}ing using {_maxThreads} threads and {_maxInputChunks} chunks per dictionary");
            var    inputDictionary  = new MySafeDictionary <int, byte[]>();
            var    outputDictionary = new MySafeDictionary <int, byte[]>();
            var    myCompressor     = new Compressor(inputDictionary, outputDictionary, _maxThreads, _maxOutputChunks, _mode);
            var    myReader         = new FileReader(from, inputDictionary, _maxInputChunks, _chunkSize, _mode == CompressionMode.Compress ? false : true);
            var    myWriter         = new FileWriter(to, outputDictionary);
            Thread readThread       = new Thread(myReader.ReadAsync);
            Thread writeThread      = new Thread(myWriter.Write);

            readThread.Start();
            writeThread.Start();
            myCompressor.LaunchThreads();
            // Ok the work is being done. The first thing we should do is wait for the readThread to be done. The write thread might come down any minute if it runs out of space too
            while (readThread.IsAlive && writeThread.IsAlive)
            {
                // Console.WriteLine($"read = {myReader.ChunksRead}, inputcount = {inputDictionary.Count}, outputcount = {outputDictionary.Count}, written = {myWriter.ChunksWritten}");
                Thread.Sleep(100);
            }
            while (myWriter.ChunksWritten < myReader.ChunksRead && writeThread.IsAlive)
            {
                // Console.WriteLine($"Reading all done. read = {myReader.ChunksRead}, inputcount = {inputDictionary.Count}, outputcount = {outputDictionary.Count}, written = {myWriter.ChunksWritten}");
                Thread.Sleep(100);
            }
            myCompressor.StopThreads();
            if (!writeThread.IsAlive)
            {
                Console.ReadLine();
                return;
            }
            writeThread.Abort();
            long startSize = new FileInfo(from).Length;
            long endSize   = new FileInfo(to).Length;

            Console.WriteLine($"{_mode.ToString()}ed {startSize} Bytes ({(startSize / (1024 * 1024)):N2} MB) into {endSize} Bytes ({(endSize / (1024 * 1024)):N2} MB) in {DateTime.Now.Subtract(start).TotalMilliseconds:N0} milliseconds ({(DateTime.Now.Subtract(start).TotalMilliseconds / (60 * 1000)):N2} minutes)");
            Console.ReadLine();
        }
Beispiel #4
0
 /// <summary>
 /// Ctor for FileWriter
 /// </summary>
 /// <param name="toFile">Output file name, expected to be a valid filename and you should have writing permissions for it</param>
 /// <param name="fromDictionary">Collection with data to write in order</param>
 public FileWriter(string toFile, MySafeDictionary <int, byte[]> fromDictionary)
 {
     _fileName = toFile;
     _dict     = fromDictionary;
 }