static unsafe void Main(string[] args)
{
// almost no effect on compression ration when above 10000
// main effect on memory consumption
var chunkSize = 100000;
var date = new DateTime(2015, 8, 5);
var fb = new FixedBuffer();
var tsize = Marshal.SizeOf(typeof(TaqTrade));
Console.WriteLine(tsize);
var zip = ZipFile.OpenRead(path);
var stream = zip.Entries.Single().Open();
using (var reader = new StreamReader(stream, Encoding.ASCII))
using (var bReader = new BinaryReader(stream, Encoding.ASCII)) {
byte[] compressedBuffer = null;
var byteBuffer = new byte[106];
var tradesArray = new TaqTrade[chunkSize];
var line = reader.ReadLine();
Console.WriteLine(line);
Console.WriteLine("Press enter to continue");
Console.ReadLine();
var sw = new Stopwatch();
sw.Start();
var c = 0;
while (!reader.EndOfStream) // && c < 100
// these two lines take 57% time
{
line = reader.ReadLine();
Encoding.ASCII.GetBytes(line, 0, 106, byteBuffer, 0);
fb.Wrap(byteBuffer);
var trade = new TaqTrade(date, fb);
tradesArray[c % chunkSize] = trade;
c++;
if (c % chunkSize == 0)
{
var comrpessed = Serializer.Serialize(tradesArray);
var averageComprSize = comrpessed.Length * 1.0 / chunkSize;
var ratio = (106.0 * chunkSize) / (comrpessed.Length * 1.0);
//Console.WriteLine(line);
Console.WriteLine($"Read so far: {c}, ratio: {ratio}, comp size: {averageComprSize}");
//Console.WriteLine("Press enter to continue");
//Console.ReadLine();
}
}
sw.Stop();
Console.WriteLine($"Lines read: ${c} in msecs: {sw.ElapsedMilliseconds}");
}
Console.WriteLine("Finished");
Console.ReadLine();
}
static unsafe void Main(string[] args) {
// almost no effect on compression ration when above 10000
// main effect on memory consumption
var chunkSize = 100000;
var date = new DateTime(2015, 8, 5);
var tsize = Marshal.SizeOf(typeof(TaqTrade));
Console.WriteLine(tsize);
var zip = ZipFile.OpenRead(path);
var stream = zip.Entries.Single().Open();
using (var reader = new StreamReader(stream, Encoding.ASCII))
using (var bReader = new BinaryReader(stream, Encoding.ASCII)) {
byte[] compressedBuffer = null;
var byteBuffer = new byte[106];
var tradesArray = new TaqTrade[chunkSize];
var line = reader.ReadLine();
Console.WriteLine(line);
Console.WriteLine("Press enter to continue");
Console.ReadLine();
var sw = new Stopwatch();
sw.Start();
var c = 0;
while (!reader.EndOfStream) { // && c < 100
// these two lines take 57% time
line = reader.ReadLine();
Encoding.ASCII.GetBytes(line, 0, 106, byteBuffer, 0);
var fb = new FixedBuffer(byteBuffer);
var trade = new TaqTrade(date, fb);
tradesArray[c % chunkSize] = trade;
c++;
if (c % chunkSize == 0) {
var comrpessed = Serializer.Serialize(tradesArray);
var averageComprSize = comrpessed.Length * 1.0 / chunkSize;
var ratio = (106.0 * chunkSize) / (comrpessed.Length * 1.0);
//Console.WriteLine(line);
Console.WriteLine($"Read so far: {c}, ratio: {ratio}, comp size: {averageComprSize}");
//Console.WriteLine("Press enter to continue");
//Console.ReadLine();
}
}
sw.Stop();
Console.WriteLine($"Lines read: ${c} in msecs: {sw.ElapsedMilliseconds}");
}
Console.WriteLine("Finished");
Console.ReadLine();
}
static unsafe void Main(string[] args)
{
GC.Collect(3, GCCollectionMode.Forced, true);
var store = new SeriesStorage(SeriesStorage.GetDefaultConnectionString("TAQSample2.db"));
var date = new DateTime(2015, 8, 5);
var tsize = Marshal.SizeOf(typeof(TaqTrade));
Console.WriteLine(tsize);
var zip = ZipFile.OpenRead(path);
var stream = zip.Entries.Single().Open();
var seriesDictionary = new Dictionary <string, IPersistentOrderedMap <DateTime, TaqTrade> >();
using (BufferedStream bs = new BufferedStream(stream, 2 * 1024 * 1024))
using (var reader = new StreamReader(bs, Encoding.ASCII)) {
byte[] compressedBuffer = null;
var byteBuffer = new byte[106];
int len;
var line = reader.ReadLine();
len = bs.ReadLineIntoBuffer(byteBuffer);
Console.WriteLine(line);
Console.WriteLine("Press enter to continue");
Console.ReadLine();
var sw = new Stopwatch();
sw.Start();
var c = 0;
while ((len = bs.ReadLineIntoBuffer(byteBuffer)) != 0) // && c < 100
{
var fb = new FixedBuffer(byteBuffer, 0, len);
var trade = new TaqTrade(date, fb);
var symbol = trade.Symbol.ToLowerInvariant().Trim();
IPersistentOrderedMap <DateTime, TaqTrade> series;
if (!seriesDictionary.TryGetValue(symbol, out series))
{
series = store.GetPersistentOrderedMap <DateTime, TaqTrade>(symbol);
seriesDictionary[symbol] = series;
}
series[trade.Time] = trade;
c++;
if (c % 100000 == 0)
{
Console.WriteLine($"Read so far: {c}");
foreach (var s in seriesDictionary)
{
s.Value.Flush();
}
}
}
sw.Stop();
foreach (var series in seriesDictionary)
{
series.Value.Flush();
}
Console.WriteLine($"Lines read: ${c} in msecs: {sw.ElapsedMilliseconds}");
}
Console.WriteLine("Finished");
GC.Collect(3, GCCollectionMode.Forced, true);
Console.WriteLine($"Total memory: {GC.GetTotalMemory(true)}");
Console.ReadLine();
}