Example #1
0
        static unsafe void Main(string[] args)
        {
            // almost no effect on compression ration when above 10000
            // main effect on memory consumption
            var chunkSize = 100000;
            var date      = new DateTime(2015, 8, 5);
            var fb        = new FixedBuffer();
            var tsize     = Marshal.SizeOf(typeof(TaqTrade));

            Console.WriteLine(tsize);

            var zip    = ZipFile.OpenRead(path);
            var stream = zip.Entries.Single().Open();

            using (var reader = new StreamReader(stream, Encoding.ASCII))
                using (var bReader = new BinaryReader(stream, Encoding.ASCII)) {
                    byte[] compressedBuffer = null;
                    var    byteBuffer       = new byte[106];
                    var    tradesArray      = new TaqTrade[chunkSize];
                    var    line             = reader.ReadLine();
                    Console.WriteLine(line);
                    Console.WriteLine("Press enter to continue");
                    Console.ReadLine();
                    var sw = new Stopwatch();
                    sw.Start();
                    var c = 0;
                    while (!reader.EndOfStream) // && c < 100
                    // these two lines take 57% time
                    {
                        line = reader.ReadLine();
                        Encoding.ASCII.GetBytes(line, 0, 106, byteBuffer, 0);

                        fb.Wrap(byteBuffer);
                        var trade = new TaqTrade(date, fb);
                        tradesArray[c % chunkSize] = trade;

                        c++;
                        if (c % chunkSize == 0)
                        {
                            var comrpessed       = Serializer.Serialize(tradesArray);
                            var averageComprSize = comrpessed.Length * 1.0 / chunkSize;
                            var ratio            = (106.0 * chunkSize) / (comrpessed.Length * 1.0);
                            //Console.WriteLine(line);
                            Console.WriteLine($"Read so far: {c}, ratio: {ratio}, comp size: {averageComprSize}");
                            //Console.WriteLine("Press enter to continue");
                            //Console.ReadLine();
                        }
                    }
                    sw.Stop();
                    Console.WriteLine($"Lines read: ${c} in msecs: {sw.ElapsedMilliseconds}");
                }

            Console.WriteLine("Finished");
            Console.ReadLine();
        }
Example #2
0
        static unsafe void Main(string[] args) {
            // almost no effect on compression ration when above 10000
            // main effect on memory consumption
            var chunkSize = 100000;
            var date = new DateTime(2015, 8, 5);
            
            var tsize = Marshal.SizeOf(typeof(TaqTrade));
            Console.WriteLine(tsize);

            var zip = ZipFile.OpenRead(path);
            var stream = zip.Entries.Single().Open();

            using (var reader = new StreamReader(stream, Encoding.ASCII))
            using (var bReader = new BinaryReader(stream, Encoding.ASCII)) {
                byte[] compressedBuffer = null;
                var byteBuffer = new byte[106];
                var tradesArray = new TaqTrade[chunkSize];
                var line = reader.ReadLine();
                Console.WriteLine(line);
                Console.WriteLine("Press enter to continue");
                Console.ReadLine();
                var sw = new Stopwatch();
                sw.Start();
                var c = 0;
                while (!reader.EndOfStream) { // && c < 100
                    // these two lines take 57% time
                    line = reader.ReadLine();
                    Encoding.ASCII.GetBytes(line, 0, 106, byteBuffer, 0);

                    var fb = new FixedBuffer(byteBuffer);
                    var trade = new TaqTrade(date, fb);
                    tradesArray[c % chunkSize] = trade;

                    c++;
                    if (c % chunkSize == 0) {
                        var comrpessed = Serializer.Serialize(tradesArray);
                        var averageComprSize = comrpessed.Length * 1.0 / chunkSize;
                        var ratio = (106.0 * chunkSize) / (comrpessed.Length * 1.0);
                        //Console.WriteLine(line);
                        Console.WriteLine($"Read so far: {c}, ratio: {ratio}, comp size: {averageComprSize}");
                        //Console.WriteLine("Press enter to continue");
                        //Console.ReadLine();
                    }
                }
                sw.Stop();
                Console.WriteLine($"Lines read: ${c} in msecs: {sw.ElapsedMilliseconds}");
            }

            Console.WriteLine("Finished");
            Console.ReadLine();

        }
Example #3
0
        static unsafe void Main(string[] args)
        {
            GC.Collect(3, GCCollectionMode.Forced, true);
            var store = new SeriesStorage(SeriesStorage.GetDefaultConnectionString("TAQSample2.db"));


            var date = new DateTime(2015, 8, 5);

            var tsize = Marshal.SizeOf(typeof(TaqTrade));

            Console.WriteLine(tsize);

            var zip    = ZipFile.OpenRead(path);
            var stream = zip.Entries.Single().Open();

            var seriesDictionary = new Dictionary <string, IPersistentOrderedMap <DateTime, TaqTrade> >();

            using (BufferedStream bs = new BufferedStream(stream, 2 * 1024 * 1024))
                using (var reader = new StreamReader(bs, Encoding.ASCII)) {
                    byte[] compressedBuffer = null;
                    var    byteBuffer       = new byte[106];
                    int    len;
                    var    line = reader.ReadLine();
                    len = bs.ReadLineIntoBuffer(byteBuffer);
                    Console.WriteLine(line);
                    Console.WriteLine("Press enter to continue");
                    Console.ReadLine();
                    var sw = new Stopwatch();
                    sw.Start();
                    var c = 0;
                    while ((len = bs.ReadLineIntoBuffer(byteBuffer)) != 0) // && c < 100
                    {
                        var fb    = new FixedBuffer(byteBuffer, 0, len);
                        var trade = new TaqTrade(date, fb);

                        var symbol = trade.Symbol.ToLowerInvariant().Trim();

                        IPersistentOrderedMap <DateTime, TaqTrade> series;
                        if (!seriesDictionary.TryGetValue(symbol, out series))
                        {
                            series = store.GetPersistentOrderedMap <DateTime, TaqTrade>(symbol);
                            seriesDictionary[symbol] = series;
                        }

                        series[trade.Time] = trade;

                        c++;
                        if (c % 100000 == 0)
                        {
                            Console.WriteLine($"Read so far: {c}");
                            foreach (var s in seriesDictionary)
                            {
                                s.Value.Flush();
                            }
                        }
                    }
                    sw.Stop();
                    foreach (var series in seriesDictionary)
                    {
                        series.Value.Flush();
                    }
                    Console.WriteLine($"Lines read: ${c} in msecs: {sw.ElapsedMilliseconds}");
                }

            Console.WriteLine("Finished");
            GC.Collect(3, GCCollectionMode.Forced, true);
            Console.WriteLine($"Total memory: {GC.GetTotalMemory(true)}");
            Console.ReadLine();
        }