private static string[] findNested(DataType field, string fieldName = "")
{
if (!(field is StructType structField))
{
return(new [] { fieldName });
}
return(structField.Fields
.SelectMany(x => findNested(x.DataType, fieldName + "." + x.Name))
.ToArray());
}
private static KeyValuePair <string, string>[] getNestedTypes(DataType field, string fieldName = "")
{
if (!(field is StructType structField))
{
return(new [] { new KeyValuePair <string, string>(fieldName, field.TypeName) });
}
return(structField.Fields
.SelectMany(x => getNestedTypes(x.DataType, fieldName + "." + x.Name))
.ToArray());
}
public static Action <StructType> IsNotNested(string column) =>
schema =>
{
DataType columnDataType = StructField(column, schema).DataType;
if (
columnDataType.TypeName == "StructType" ||
columnDataType.TypeName == "MapType" ||
columnDataType.TypeName == "ArrayType"
)
{
throw new Exception($"Unsupported nested column type of column {column}: {columnDataType}!");
}
};
public static Action <StructType> IsNumeric(string column) =>
schema =>
{
DataType columnDataType = StructField(column, schema).DataType;
bool hasNumericType = columnDataType.TypeName == new ByteType().TypeName ||
columnDataType.TypeName == new ShortType().TypeName ||
columnDataType.TypeName == new IntegerType().TypeName ||
columnDataType.TypeName == new LongType().TypeName ||
columnDataType.TypeName == new FloatType().TypeName ||
columnDataType.TypeName == new DecimalType().TypeName;
if (!hasNumericType)
{
throw new Exception(
$"Expected type of column $column to be one of ${string.Join(',', NumericDataType)}), but found ${columnDataType} instead!");
}
};