/// <summary>
/// Ctor for FileReader
/// </summary>
/// <param name="fromFile">File name to read data from</param>
/// <param name="toDictionary">Collection to store data in</param>
/// <param name="maxChunksInDict">How many chunks may the collection store at a time (used for not running out of memory)</param>
/// <param name="chunkSize">It will read data from file in chunks of this size (Bytes). It will add data to collection in chunks of same size if not in magicNumerMode</param>
/// <param name="magicNumberMode">If this is set to True, will assume that data has GZip headers and scan for those instead of conserving chunks as is.
/// Will add data to collection in variable size chunks</param>
public FileReader(string fromFile, MySafeDictionary <int, byte[]> toDictionary, int maxChunksInDict, int chunkSize, bool magicNumberMode = false)
{
_fileName = fromFile;
_dict = toDictionary;
_magicNumberMode = magicNumberMode;
_chunkSize = chunkSize;
_maxChunks = maxChunksInDict;
}
/// <summary>
/// Create an instance of Compressor, with all the data it needs in order to process.
/// </summary>
/// <param name="fromDictionary">The data to compress/decompess in a collection</param>
/// <param name="toDictionary">The collection to put resulting data in</param>
/// <param name="maxThreads">How many worker threads to use while processing data</param>
/// <param name="throttleOutputCount">Will throttle processing if resulting output collection has this many items. Useful if processing is too fast, writing too slow</param>
/// <param name="mode">Compress or decompress</param>
public Compressor(MySafeDictionary <int, byte[]> fromDictionary, MySafeDictionary <int, byte[]> toDictionary, int maxThreads, int throttleOutputCount, CompressionMode mode)
{
_inputDict = fromDictionary;
_outputDict = toDictionary;
_maxThreads = maxThreads;
_mode = mode;
_threads = new Thread[maxThreads];
_throttleOutputCount = throttleOutputCount;
}
// We await input of format GZipTest.exe compress/decompress FromFile ToFile
private static void Main(string[] args)
{
if (!ParseInput(args))
{
Console.ReadLine();
return;
}
// Let's figure out technical parameters
ComputerInfo CI = new ComputerInfo();
ulong mem = CI.AvailablePhysicalMemory;
ulong vmem = CI.AvailableVirtualMemory;
int cpuCount = Environment.ProcessorCount;
_maxThreads = cpuCount;
_maxInputChunks = Convert.ToInt32((mem * 0.45) / (_chunkSize));
_maxOutputChunks = _maxInputChunks;
// Seems like all input was correct, let's continue with actual work
DateTime start = DateTime.Now;
Console.WriteLine($"{_mode.ToString()}ing using {_maxThreads} threads and {_maxInputChunks} chunks per dictionary");
var inputDictionary = new MySafeDictionary <int, byte[]>();
var outputDictionary = new MySafeDictionary <int, byte[]>();
var myCompressor = new Compressor(inputDictionary, outputDictionary, _maxThreads, _maxOutputChunks, _mode);
var myReader = new FileReader(from, inputDictionary, _maxInputChunks, _chunkSize, _mode == CompressionMode.Compress ? false : true);
var myWriter = new FileWriter(to, outputDictionary);
Thread readThread = new Thread(myReader.ReadAsync);
Thread writeThread = new Thread(myWriter.Write);
readThread.Start();
writeThread.Start();
myCompressor.LaunchThreads();
// Ok the work is being done. The first thing we should do is wait for the readThread to be done. The write thread might come down any minute if it runs out of space too
while (readThread.IsAlive && writeThread.IsAlive)
{
// Console.WriteLine($"read = {myReader.ChunksRead}, inputcount = {inputDictionary.Count}, outputcount = {outputDictionary.Count}, written = {myWriter.ChunksWritten}");
Thread.Sleep(100);
}
while (myWriter.ChunksWritten < myReader.ChunksRead && writeThread.IsAlive)
{
// Console.WriteLine($"Reading all done. read = {myReader.ChunksRead}, inputcount = {inputDictionary.Count}, outputcount = {outputDictionary.Count}, written = {myWriter.ChunksWritten}");
Thread.Sleep(100);
}
myCompressor.StopThreads();
if (!writeThread.IsAlive)
{
Console.ReadLine();
return;
}
writeThread.Abort();
long startSize = new FileInfo(from).Length;
long endSize = new FileInfo(to).Length;
Console.WriteLine($"{_mode.ToString()}ed {startSize} Bytes ({(startSize / (1024 * 1024)):N2} MB) into {endSize} Bytes ({(endSize / (1024 * 1024)):N2} MB) in {DateTime.Now.Subtract(start).TotalMilliseconds:N0} milliseconds ({(DateTime.Now.Subtract(start).TotalMilliseconds / (60 * 1000)):N2} minutes)");
Console.ReadLine();
}
/// <summary>
/// Ctor for FileWriter
/// </summary>
/// <param name="toFile">Output file name, expected to be a valid filename and you should have writing permissions for it</param>
/// <param name="fromDictionary">Collection with data to write in order</param>
public FileWriter(string toFile, MySafeDictionary <int, byte[]> fromDictionary)
{
_fileName = toFile;
_dict = fromDictionary;
}