static unsafe void Main(string[] args)
{
// almost no effect on compression ration when above 10000
// main effect on memory consumption
var chunkSize = 100000;
var date = new DateTime(2015, 8, 5);
var fb = new FixedBuffer();
var tsize = Marshal.SizeOf(typeof(TaqTrade));
Console.WriteLine(tsize);
var zip = ZipFile.OpenRead(path);
var stream = zip.Entries.Single().Open();
using (var reader = new StreamReader(stream, Encoding.ASCII))
using (var bReader = new BinaryReader(stream, Encoding.ASCII)) {
byte[] compressedBuffer = null;
var byteBuffer = new byte[106];
var tradesArray = new TaqTrade[chunkSize];
var line = reader.ReadLine();
Console.WriteLine(line);
Console.WriteLine("Press enter to continue");
Console.ReadLine();
var sw = new Stopwatch();
sw.Start();
var c = 0;
while (!reader.EndOfStream) { // && c < 100
// these two lines take 57% time
line = reader.ReadLine();
Encoding.ASCII.GetBytes(line, 0, 106, byteBuffer, 0);
fb.Wrap(byteBuffer);
var trade = new TaqTrade(date, fb);
tradesArray[c % chunkSize] = trade;
c++;
if (c % chunkSize == 0) {
var comrpessed = Serializer.Serialize(tradesArray);
var averageComprSize = comrpessed.Length * 1.0 / chunkSize;
var ratio = (106.0 * chunkSize) / (comrpessed.Length * 1.0);
//Console.WriteLine(line);
Console.WriteLine($"Read so far: {c}, ratio: {ratio}, comp size: {averageComprSize}");
//Console.WriteLine("Press enter to continue");
//Console.ReadLine();
}
}
sw.Stop();
Console.WriteLine($"Lines read: ${c} in msecs: {sw.ElapsedMilliseconds}");
}
Console.WriteLine("Finished");
Console.ReadLine();
}
