/// <summary>
///
/// </summary>
public static UnmanagedMemoryStream GetDirectStream(this ArraySegment <byte> arraySegment)
{
var fb = new FixedBuffer(arraySegment.Array, arraySegment.Offset, arraySegment.Count);
// NB offet/len are applied to fb
return(fb.CreateStream());
}
public SafeFixedBuffer(ref FixedBuffer fixedBuffer) : base(false)
{
_fixedBuffer = fixedBuffer;
_fixedBuffer.PinBuffer();
base.SetHandle(new IntPtr(_fixedBuffer._unpinner.PinnedGCHandle.AddrOfPinnedObject().ToInt64() + _fixedBuffer._offset));
base.Initialize((uint)_fixedBuffer._length);
}
internal FixedBufferStream(FixedBuffer buffer, long offset, long length, bool readOnly, bool unsafePointer)
{
Debug.Assert(buffer != null, "buffer is null");
if (unsafePointer)
{
Initialize(buffer, offset, length, readOnly ? FileAccess.Read : FileAccess.ReadWrite);
}
else
{
Initialize((byte *)buffer.DirectBuffer.data, offset, length, readOnly ? FileAccess.Read : FileAccess.ReadWrite);
}
}
static unsafe void Main(string[] args) {
// almost no effect on compression ration when above 10000
// main effect on memory consumption
var chunkSize = 100000;
var date = new DateTime(2015, 8, 5);
var tsize = Marshal.SizeOf(typeof(TaqTrade));
Console.WriteLine(tsize);
var zip = ZipFile.OpenRead(path);
var stream = zip.Entries.Single().Open();
using (var reader = new StreamReader(stream, Encoding.ASCII))
using (var bReader = new BinaryReader(stream, Encoding.ASCII)) {
byte[] compressedBuffer = null;
var byteBuffer = new byte[106];
var tradesArray = new TaqTrade[chunkSize];
var line = reader.ReadLine();
Console.WriteLine(line);
Console.WriteLine("Press enter to continue");
Console.ReadLine();
var sw = new Stopwatch();
sw.Start();
var c = 0;
while (!reader.EndOfStream) { // && c < 100
// these two lines take 57% time
line = reader.ReadLine();
Encoding.ASCII.GetBytes(line, 0, 106, byteBuffer, 0);
var fb = new FixedBuffer(byteBuffer);
var trade = new TaqTrade(date, fb);
tradesArray[c % chunkSize] = trade;
c++;
if (c % chunkSize == 0) {
var comrpessed = Serializer.Serialize(tradesArray);
var averageComprSize = comrpessed.Length * 1.0 / chunkSize;
var ratio = (106.0 * chunkSize) / (comrpessed.Length * 1.0);
//Console.WriteLine(line);
Console.WriteLine($"Read so far: {c}, ratio: {ratio}, comp size: {averageComprSize}");
//Console.WriteLine("Press enter to continue");
//Console.ReadLine();
}
}
sw.Stop();
Console.WriteLine($"Lines read: ${c} in msecs: {sw.ElapsedMilliseconds}");
}
Console.WriteLine("Finished");
Console.ReadLine();
}
public TaqTrade(DateTime date, FixedBuffer fb)
{
// Read<> over fb is much slower
var db = fb;
// Time
_timeUTCTicks = ReadHHMMSSXXXXXXAsUtcTicks(date, fb, 0);
// Exchange
Exchange = fb.ReadByte(12);
// Symbol
fixed (void* ptr = _symbol)
{
fb.Copy((IntPtr)ptr, 13, 16);
}
// Sale Condition
fixed (void* ptr = _saleCondition)
{
fb.Copy((IntPtr)ptr, 29, 4);
}
TradeVolume = (uint)ReadUInt64(fb, 33, 9);
TradePrice = ReadUInt64(fb, 42, 11);
TradeStopStockIndicator = db.ReadByte(53);
TradeCorrectionIndicator = (byte)ReadUInt64(fb, 54, 2);
TradeSequenceNumber = ReadUInt64(fb, 56, 16);
SourceOfTrade = db.ReadByte(72);
TradeReportingFacility = db.ReadByte(73);
_participantTimestampUtcTicks = ReadHHMMSSXXXXXXAsUtcTicks(date, fb, 74);
fixed (void* ptr = _rnn)
{
fb.Copy((IntPtr)ptr, 86, 8);
}
_TRFTimestampUtcTicks = ReadHHMMSSXXXXXXAsUtcTicks(date, fb, 94);
}
/// <summary>
///
/// </summary>
public static UnmanagedMemoryStream GetDirectStream(this ArraySegment<byte> arraySegment) {
var fb = new FixedBuffer(arraySegment.Array, arraySegment.Offset, arraySegment.Count);
// NB offet/len are applied to fb
return fb.CreateStream();
}
internal FixedBufferStream(FixedBuffer buffer, long offset, long length, bool readOnly, bool unsafePointer)
{
Debug.Assert(buffer != null, "buffer is null");
if (unsafePointer) {
Initialize(buffer, offset, length, readOnly ? FileAccess.Read : FileAccess.ReadWrite);
} else {
Initialize((byte*)buffer.DirectBuffer.data, offset, length, readOnly ? FileAccess.Read : FileAccess.ReadWrite);
}
}
internal FixedBufferAccessor(FixedBuffer buffer, long offset, long length, bool readOnly)
{
Debug.Assert(buffer != null, "buffer is null");
Initialize(buffer, offset, length, readOnly ? FileAccess.Read : FileAccess.ReadWrite);
}
/// <summary>
///
/// </summary>
public static UnmanagedMemoryStream GetDirectStream(this ArraySegment<byte> arraySegment)
{
var db = new FixedBuffer(arraySegment.Array);
return db.CreateStream(arraySegment.Offset, arraySegment.Count, false, true);
}
/// <summary>
///
/// </summary>
public static UnmanagedMemoryAccessor GetDirectAccessor(this ArraySegment<byte> arraySegment)
{
var db = new FixedBuffer(arraySegment.Array);
return db.CreateAccessor(arraySegment.Offset, arraySegment.Count, false);
}
/// <summary>
///
/// </summary>
public static UnmanagedMemoryStream GetDirectStream(this ArraySegment <byte> arraySegment)
{
var db = new FixedBuffer(arraySegment.Array);
return(db.CreateStream(arraySegment.Offset, arraySegment.Count, false, true));
}
/// <summary>
///
/// </summary>
public static UnmanagedMemoryAccessor GetDirectAccessor(this ArraySegment <byte> arraySegment)
{
var db = new FixedBuffer(arraySegment.Array);
return(db.CreateAccessor(arraySegment.Offset, arraySegment.Count, false));
}
internal FixedBufferAccessor(FixedBuffer buffer, long offset, long length, bool readOnly)
{
Debug.Assert(buffer != null, "buffer is null");
Initialize(buffer, offset, length, readOnly ? FileAccess.Read : FileAccess.ReadWrite);
}
