static IList GetValues(Dictionary <string, IList> columns, DataField field, bool createIfMissing, bool isNested = false)
{
if (field.Path == null)
{
throw new ArgumentNullException(nameof(field.Path));
}
if (!columns.TryGetValue(field.Path, out IList values) || values == null)
{
if (createIfMissing)
{
IDataTypeHandler handler = DataTypeFactory.Match(field);
values = isNested
? new List <IEnumerable>()
: handler.CreateEmptyList(field.HasNulls, field.IsArray, 0);
columns[field.Path] = values;
}
else
{
throw new ArgumentException($"column does not exist by path '{field.Path}'", nameof(field));
}
}
return(values);
}
private DataColumn(DataField field)
{
Field = field ?? throw new ArgumentNullException(nameof(field));
_dataTypeHandler = DataTypeFactory.Match(field.DataType);
HasRepetitions = field.IsArray;
}
public DataColumn(DataField field)
{
_field = field ?? throw new ArgumentNullException(nameof(field));
IDataTypeHandler handler = DataTypeFactory.Match(field.DataType);
_definedData = handler.CreateEmptyList(false, false, 0); // always a plain list, always non-nullable when possible
_definitionLevels = field.HasNulls ? new List <int>() : null; // do not create an instance when not required
}
public DataColumn(DataField field)
{
_field = field ?? throw new ArgumentNullException(nameof(field));
IDataTypeHandler handler = DataTypeFactory.Match(field.DataType);
_definedData = handler.CreateEmptyList(false, false, 0); // always a plain list, always non-nullable when possible
_definitionLevels = new List <int>();
HasRepetitions = field.IsArray;
_repetitionLevels = HasRepetitions ? new List <int>() : null;
}
public DataField(string name, DataType dataType, bool hasNulls = true, bool isArray = false) : base(name, SchemaType.Data)
{
DataType = dataType;
HasNulls = hasNulls;
IsArray = isArray;
IDataTypeHandler handler = DataTypeFactory.Match(dataType);
if (handler != null)
{
ClrType = handler.ClrType;
}
}
/// <summary>
/// Creates a new instance of <see cref="DataField"/> by specifying all the required attributes.
/// </summary>
/// <param name="name">Field name.</param>
/// <param name="dataType">Native Parquet type</param>
/// <param name="hasNulls">When true, the field accepts null values. Note that nullable values take slightly more disk space and computing comparing to non-nullable, but are more common.</param>
/// <param name="isArray">When true, each value of this field can have multiple values, similar to array in C#.</param>
public DataField(string name, DataType dataType, bool hasNulls = true, bool isArray = false) : base(name, SchemaType.Data)
{
DataType = dataType;
HasNulls = hasNulls;
IsArray = isArray;
MaxRepetitionLevel = isArray ? 1 : 0;
IDataTypeHandler handler = DataTypeFactory.Match(dataType);
if (handler != null)
{
ClrType = handler.ClrType;
ClrNullableIfHasNullsType = hasNulls ? ClrType.GetNullable() : ClrType;
}
}
private static CInfo Discover()
{
Type t = typeof(T);
Type baseType = t;
bool isArray = false;
bool hasNulls = false;
//throw a useful hint
if (t.TryExtractDictionaryType(out Type dKey, out Type dValue))
{
throw new ArgumentException($"cannot declare a dictionary this way, please use {nameof(MapField)}.");
}
if (t.TryExtractEnumerableType(out Type enumItemType))
{
baseType = enumItemType;
isArray = true;
}
if (baseType.IsNullable())
{
baseType = baseType.GetNonNullable();
hasNulls = true;
}
if (typeof(Row) == baseType)
{
throw new ArgumentException($"{typeof(Row)} is not supported. If you tried to declare a struct please use {typeof(StructField)} instead.");
}
IDataTypeHandler handler = DataTypeFactory.Match(baseType);
if (handler == null)
{
DataTypeFactory.ThrowClrTypeNotSupported(baseType);
}
return(new CInfo
{
dataType = handler.DataType,
baseType = baseType,
isArray = isArray,
hasNulls = hasNulls
});
}
internal void AddElement(IList keys, IList values, IDictionary dictionary)
{
IDataTypeHandler keyHandler = DataTypeFactory.Match(Key.DataType);
IDataTypeHandler valueHandler = DataTypeFactory.Match(Value.DataType);
IList keysList = keyHandler.CreateEmptyList(Key.HasNulls, false, dictionary.Count);
IList valuesList = valueHandler.CreateEmptyList(Value.HasNulls, false, dictionary.Count);
foreach (object v in dictionary.Keys)
{
keysList.Add(v);
}
foreach (object v in dictionary.Values)
{
valuesList.Add(v);
}
keys.Add(keysList);
values.Add(valuesList);
}
static void AddList(Dictionary <string, IList> columns, ListField listField, object value)
{
/*
* Value slicing can happen only when entering a list and in no other cases.
* Only list is changing hierarchy dramatically.
*/
switch (listField.Item.SchemaType)
{
case SchemaType.Struct:
StructField structField = (StructField)listField.Item;
IEnumerable <Row> rows = value as IEnumerable <Row>;
var deepColumns = new Dictionary <string, IList>();
foreach (Row row in rows)
{
Append(deepColumns, structField.Fields, row);
}
SliceIn(columns, deepColumns);
break;
case SchemaType.Data:
DataField dataField = (DataField)listField.Item;
IDataTypeHandler handler = DataTypeFactory.Match(dataField);
IList values = handler.CreateEmptyList(dataField.HasNulls, dataField.IsArray, 0);
foreach (object v in (IEnumerable)value)
{
values.Add(v);
}
GetValues(columns, dataField, true, true).Add(values);
break;
default:
throw OtherExtensions.NotImplementedForPotentialAssholesAndMoaners($"adding {listField.Item.SchemaType} to list");
}
}
/// <summary>
/// To be deprecated soon
/// </summary>
public IList CreateEmptyList(bool isNullable, bool isArray)
{
IDataTypeHandler handler = DataTypeFactory.Match(DataType);
return(handler.CreateEmptyList(isNullable, isArray, 0));
}
