public int Parse(string str, int i, DType cur)
{
Whitespace(str, i);
int k = 0, kk;
if ((kk = AtChar(str, i, '[')) > 0)
{
cur.Type = typeof(IDictionary <string, object>);
i += kk; k += kk;
int nfield = 0;
for (;;)
{
nfield++;
if ((kk = Whitespace(str, i)) >= 0)
{
i += kk; k += kk;
}
if ((kk = AtChar(str, i, '(')) <= 0)
{
break;
}
DType child = new DType(null, cur);
cur.Fields.Add(child);
kk = Field(str, i, child);
i += kk; k += kk;
if ((kk = Whitespace(str, i)) >= 0)
{
i += kk; k += kk;
}
if ((kk = AtChar(str, i, ',')) >= 0)
{
i += kk; k += kk;
continue;
}
}
if ((kk = Whitespace(str, i)) >= 0)
{
i += kk; k += kk;
}
kk = AtChar(str, i, ']', true);
i += kk; k += kk;
return(k);
}
if ((kk = String(str, i, out string val)) > 0)
{
if (val.StartsWith("<"))
{
cur.Endian = EndianType.Little;
}
else if (val.StartsWith(">"))
{
cur.Endian = EndianType.Big;
}
else if (val.StartsWith("="))
{
cur.Endian = EndianType.Native;
}
if (val == "<f8")
{
cur.Type = typeof(double);
if (sizeof(double) != 8)
{
throw new InvalidOperationException();
}
cur.Size = 8;
}
else if (val == "<i8")
{
cur.Type = typeof(long);
if (sizeof(long) != 8)
{
throw new InvalidOperationException();
}
cur.Size = 8;
}
else if (val == "<i4")
{
cur.Type = typeof(int);
if (sizeof(int) != 4)
{
throw new InvalidOperationException();
}
cur.Size = 4;
}
else if (val == "<M8[ns]")
{
cur.Type = typeof(DateTime);
if (sizeof(long) != 8)
{
throw new InvalidOperationException();
}
cur.Size = 8;
}
else if (val.ToUpper().Contains("S"))
{
cur.Type = typeof(string);
if (!int.TryParse(val.Remove(0, 1), out int size))
{
throw new InvalidOperationException();
}
cur.Size = size;
cur.EncodingStyle = Encoding.UTF8;
}
else if (val.ToUpper().Contains("U"))
{
cur.Type = typeof(string);
if (!int.TryParse(val.Remove(0, 2), out int size))
{
throw new InvalidOperationException();
}
cur.Size = size * 4;
cur.EncodingStyle = Encoding.Unicode;
}
else
{
throw new InvalidOperationException("unknown numpy dtype '{0}'".Args(val));
}
i += kk; k += kk;
return(k);
}
BadChar(str, i, "\'[");
return(k);
}
public static Series FromBuffer(byte[] buf, DType buftype, int iheight, int icol, Func <long, DateTime> getter, Func <DateTime, long> setter)
{
return(new DateTimeSeries(Series <long> .FromBuffer(buf, buftype, iheight, icol) as Series <long>));
}
public DType(Type type = null, DType parent = null)
{
Type = type;
Parent = null;
}
//public DateTimeSeries(NdArray<long> array, DType dtype=null)
// : this(new Series<long>(array))
//{
//}
public DateTimeSeries(BaseSeries <long> source, DType dtype = null)
: base(source, getter: DateTime64.ToDateTime, setter: DateTime64.ToDateTime64, dtype: dtype ?? DType.DateTime64)
{
}
public override Series Clone()
{
return(new DateTimeSeries(Source.Copy(), DType.Clone()));
}
public abstract void ToBuffer(byte[] buf, DType buftype, int iheight, int icol);
public DateTimeSeries(long count, DType dtype = null)
: this(new Series <long>(count))
{
}
public StringSeries(string[] data, DType dtype = null, string name = null) : base(data, dtype, name)
{
}
public StringSeries(NdArray <string> values, DType dtype = null) : base(values, dtype)
{
}
public StringSeries(long count, DType dtype = null) : base(count, dtype)
{
}
public async Task <DataFrame> ReadAsync(string symbol, DateRange range = null, BsonDocument version = null)
{
version = version ?? await ReadVersionAsync(symbol);
if (version == null)
{
return(null);
}
Log.Debug("version: {0}".Args(version));
var buf = new ByteBuffer();
var index = GetSegmentsIndex(version);
var id = version["_id"];
var parent = version.GetValue("base_version_id", null);
if (parent == null)
{
parent = id;
}
var filter = BF.Eq("symbol", symbol) & BF.Eq("parent", parent);
int start_segment = 0;
int end_segment = -1;
if (range != null)
{
foreach (var t in index)
{
if (range.StartDate != default(DateTime) && range.StartDate > t.Item1) // t.Item1 is inclusive end date of segment
{
start_segment = (int)t.Item2 + 1; // should start from next segment
}
if (range.StopDate != default(DateTime) && range.StopDate <= t.Item1 && end_segment == -1)
{
end_segment = (int)t.Item2; // should stop at this segment
}
}
if (start_segment != 0)
{
filter = filter & BF.Gte("segment", start_segment);
}
if (end_segment != -1)
{
filter = filter & BF.Lte("segment", end_segment);
}
}
if (end_segment == -1)
{
end_segment = version["up_to"].AsInt32 - 1;
}
var segments = await this._segments.FindAsync(filter);
int segcount = 0;
while (await segments.MoveNextAsync())
{
foreach (var segment in segments.Current)
{
#if DEBUG
//Log.Debug ("read segment: {0}".Args(segment));
#endif
var chunk = segment["data"].AsByteArray;
if (segment ["compressed"].AsBoolean)
{
buf.AppendDecompress(chunk);
}
else
{
buf.Append(chunk);
}
segcount++;
}
}
var metadata = version.GetValue("metadata", new BsonDocument()).AsBsonDocument;
if (segcount == 0)
{
//var df1 = new DataFrame();
//df1.Metadata = metadata;
//return df1;
//throw new InvalidOperationException("No segments found for {0}".Args(version));
}
var nrows = end_segment - start_segment + 1;
var dtype = version ["dtype"].AsString;
var buftype = new DType(dtype);
var bytes = buf.GetBytes();
Log.Debug("converting to dataframe up_to={0} dtype={1} len={2}".Args(nrows, dtype, bytes.Length));
var df = DataFrame.FromBuffer(buf.GetBytes(), buftype, nrows);
var meta = version["dtype_metadata"].AsBsonDocument;
var index_name = meta.GetValue("index", new BsonArray()).AsBsonArray[0];
if (index_name != null)
{
df.Index = df.Columns[index_name.AsString];
}
df.Metadata = metadata;
df.Name = symbol;
df.FilledCount = df.Rows.Count;
// TODO: Filter first/last segment
return(df);
}
public int Parse(string str, int i, DType cur)
{
Whitespace(str, i);
int k = 0, kk;
if ((kk = AtChar(str, i, '[')) > 0)
{
cur.Type = typeof(IDictionary <string, object>);
i += kk; k += kk;
int nfield = 0;
for (;;)
{
nfield++;
if ((kk = Whitespace(str, i)) >= 0)
{
i += kk; k += kk;
}
if ((kk = AtChar(str, i, '(')) <= 0)
{
break;
}
DType child = new DType(null, cur);
cur.Fields.Add(child);
kk = Field(str, i, child);
i += kk; k += kk;
if ((kk = Whitespace(str, i)) >= 0)
{
i += kk; k += kk;
}
if ((kk = AtChar(str, i, ',')) >= 0)
{
i += kk; k += kk;
continue;
}
}
if ((kk = Whitespace(str, i)) >= 0)
{
i += kk; k += kk;
}
kk = AtChar(str, i, ']', true);
i += kk; k += kk;
return(k);
}
string val;
if ((kk = String(str, i, out val)) > 0)
{
if (val == "<f8")
{
cur.Type = typeof(double);
if (sizeof(double) != 8)
{
throw new InvalidOperationException();
}
cur.Size = 8;
}
else if (val == "<i8")
{
cur.Type = typeof(long);
if (sizeof(long) != 8)
{
throw new InvalidOperationException();
}
cur.Size = 8;
}
else if (val == "<i4")
{
cur.Type = typeof(int);
if (sizeof(int) != 4)
{
throw new InvalidOperationException();
}
cur.Size = 4;
}
else if (val == "<M8[ns]")
{
cur.Type = typeof(DateTime);
if (sizeof(long) != 8)
{
throw new InvalidOperationException();
}
cur.Size = 8;
}
else
{
throw new InvalidOperationException("unknown numpy dtype '{0}'".Args(val));
}
i += kk; k += kk;
return(k);
}
BadChar(str, i, "\'[");
return(k);
}