static bool TryMakeColumnSpec(string name, feather.fbs.Type effectiveType, ref PrimitiveArray arrayDetails, string[] categoryLevels, DateTimePrecisionType precision, out ColumnSpec columnSpec, out string errorMessage)
{
var arrayOffset = arrayDetails.Offset;
var arrayLength = arrayDetails.Length;
var arrayNulls = arrayDetails.NullCount;
var arrayEncoding = arrayDetails.Encoding;
// TODO (Dictionary Encoding)
if (arrayEncoding != feather.fbs.Encoding.PLAIN)
{
throw new NotImplementedException();
}
// END TODO
ColumnType type;
bool isNullable;
if (!TryGetType(effectiveType, ref arrayDetails, precision, out type, out isNullable, out errorMessage))
{
columnSpec = default(ColumnSpec);
return(false);
}
long numNullBytes = 0;
if (isNullable)
{
numNullBytes = arrayLength / 8;
if (arrayLength % 8 != 0)
{
numNullBytes++;
}
}
// a naive reading of the spec suggests that the null bitmask should be
// aligned based on the _type_ but it appears to always be long
// aligned.
// this may be a bug in the spec
int nullPadding = 0;
if ((numNullBytes % FeatherMagic.NULL_BITMASK_ALIGNMENT) != 0)
{
nullPadding = FeatherMagic.NULL_BITMASK_ALIGNMENT - (int)(numNullBytes % FeatherMagic.NULL_BITMASK_ALIGNMENT);
}
var nullOffset = isNullable ? (arrayOffset) : -1;
var dataOffset = !isNullable ? (arrayOffset) : (nullOffset + numNullBytes + nullPadding);
columnSpec =
new ColumnSpec
{
Name = name,
NullBitmaskOffset = nullOffset,
DataOffset = dataOffset,
Length = arrayLength,
Type = type,
CategoryLevels = categoryLevels,
// only spin up this map if we've got categories to potentially map to
CategoryEnumMap = categoryLevels != null ? new Dictionary <System.Type, CategoryEnumMapType>() : null
};
errorMessage = null;
return(true);
}
static bool TryGetType(feather.fbs.Type effectiveType, ref PrimitiveArray array, DateTimePrecisionType precision, out ColumnType type, out bool isNullable, out string errorMessage)
{
isNullable = array.NullCount != 0;
if (!isNullable)
{
switch (effectiveType)
{
case feather.fbs.Type.BINARY:
type = ColumnType.Binary;
errorMessage = null;
return(true);
case feather.fbs.Type.BOOL:
type = ColumnType.Bool;
errorMessage = null;
return(true);
case feather.fbs.Type.CATEGORY:
type = ColumnType.Category;
errorMessage = null;
return(true);
case feather.fbs.Type.DATE:
type = ColumnType.Date;
errorMessage = null;
return(true);
case feather.fbs.Type.DOUBLE:
type = ColumnType.Double;
errorMessage = null;
return(true);
case feather.fbs.Type.FLOAT:
type = ColumnType.Float;
errorMessage = null;
return(true);
case feather.fbs.Type.INT16:
type = ColumnType.Int16;
errorMessage = null;
return(true);
case feather.fbs.Type.INT32:
type = ColumnType.Int32;
errorMessage = null;
return(true);
case feather.fbs.Type.INT64:
type = ColumnType.Int64;
errorMessage = null;
return(true);
case feather.fbs.Type.INT8:
type = ColumnType.Int8;
errorMessage = null;
return(true);
case feather.fbs.Type.TIMESTAMP:
switch (precision)
{
case DateTimePrecisionType.Microsecond:
type = ColumnType.Timestamp_Microsecond;
errorMessage = null;
return(true);
case DateTimePrecisionType.Millisecond:
type = ColumnType.Timestamp_Millisecond;
errorMessage = null;
return(true);
case DateTimePrecisionType.Nanosecond:
type = ColumnType.Timestamp_Nanosecond;
errorMessage = null;
return(true);
case DateTimePrecisionType.Second:
type = ColumnType.Timestamp_Second;
errorMessage = null;
return(true);
default:
errorMessage = $"Unknown precision {precision}";
type = ColumnType.NONE;
return(false);
}
case feather.fbs.Type.TIME:
switch (precision)
{
case DateTimePrecisionType.Microsecond:
type = ColumnType.Time_Microsecond;
errorMessage = null;
return(true);
case DateTimePrecisionType.Millisecond:
type = ColumnType.Time_Millisecond;
errorMessage = null;
return(true);
case DateTimePrecisionType.Nanosecond:
type = ColumnType.Time_Nanosecond;
errorMessage = null;
return(true);
case DateTimePrecisionType.Second:
type = ColumnType.Time_Second;
errorMessage = null;
return(true);
default:
errorMessage = $"Unknown precision {precision}";
type = ColumnType.NONE;
return(false);
}
case feather.fbs.Type.UINT16:
type = ColumnType.Uint16;
errorMessage = null;
return(true);
case feather.fbs.Type.UINT32:
type = ColumnType.Uint32;
errorMessage = null;
return(true);
case feather.fbs.Type.UINT64:
type = ColumnType.Uint64;
errorMessage = null;
return(true);
case feather.fbs.Type.UINT8:
type = ColumnType.Uint8;
errorMessage = null;
return(true);
case feather.fbs.Type.UTF8:
type = ColumnType.String;
errorMessage = null;
return(true);
default:
errorMessage = $"Unknown column type {array.Type}";
type = ColumnType.NONE;
return(false);
}
}
else
{
switch (effectiveType)
{
case feather.fbs.Type.BINARY:
type = ColumnType.NullableBinary;
errorMessage = null;
return(true);
case feather.fbs.Type.BOOL:
type = ColumnType.NullableBool;
errorMessage = null;
return(true);
case feather.fbs.Type.CATEGORY:
type = ColumnType.NullableCategory;
errorMessage = null;
return(true);
case feather.fbs.Type.DATE:
type = ColumnType.NullableDate;
errorMessage = null;
return(true);
case feather.fbs.Type.DOUBLE:
type = ColumnType.NullableDouble;
errorMessage = null;
return(true);
case feather.fbs.Type.FLOAT:
type = ColumnType.NullableFloat;
errorMessage = null;
return(true);
case feather.fbs.Type.INT16:
type = ColumnType.NullableInt16;
errorMessage = null;
return(true);
case feather.fbs.Type.INT32:
type = ColumnType.NullableInt32;
errorMessage = null;
return(true);
case feather.fbs.Type.INT64:
type = ColumnType.NullableInt64;
errorMessage = null;
return(true);
case feather.fbs.Type.INT8:
type = ColumnType.NullableInt8;
errorMessage = null;
return(true);
case feather.fbs.Type.TIMESTAMP:
switch (precision)
{
case DateTimePrecisionType.Microsecond:
type = ColumnType.NullableTimestamp_Microsecond;
errorMessage = null;
return(true);
case DateTimePrecisionType.Millisecond:
type = ColumnType.NullableTimestamp_Millisecond;
errorMessage = null;
return(true);
case DateTimePrecisionType.Nanosecond:
type = ColumnType.NullableTimestamp_Nanosecond;
errorMessage = null;
return(true);
case DateTimePrecisionType.Second:
type = ColumnType.NullableTimestamp_Second;
errorMessage = null;
return(true);
default:
errorMessage = $"Unknown precision {precision}";
type = ColumnType.NONE;
return(false);
}
case feather.fbs.Type.TIME:
switch (precision)
{
case DateTimePrecisionType.Microsecond:
type = ColumnType.NullableTime_Microsecond;
errorMessage = null;
return(true);
case DateTimePrecisionType.Millisecond:
type = ColumnType.NullableTime_Millisecond;
errorMessage = null;
return(true);
case DateTimePrecisionType.Nanosecond:
type = ColumnType.NullableTime_Nanosecond;
errorMessage = null;
return(true);
case DateTimePrecisionType.Second:
type = ColumnType.NullableTime_Second;
errorMessage = null;
return(true);
default:
errorMessage = $"Unknown precision {precision}";
type = ColumnType.NONE;
return(false);
}
case feather.fbs.Type.UINT16:
type = ColumnType.NullableUint16;
errorMessage = null;
return(true);
case feather.fbs.Type.UINT32:
type = ColumnType.NullableUint32;
errorMessage = null;
return(true);
case feather.fbs.Type.UINT64:
type = ColumnType.NullableUint64;
errorMessage = null;
return(true);
case feather.fbs.Type.UINT8:
type = ColumnType.NullableUint8;
errorMessage = null;
return(true);
case feather.fbs.Type.UTF8:
type = ColumnType.NullableString;
errorMessage = null;
return(true);
default:
errorMessage = $"Unknown column type {array.Type}";
type = ColumnType.NONE;
return(false);
}
}
}
