private IEnumerable <IColumn> CreateColumnsFromColumnMetadata(IArray <ColumnMetadata> columns,
IColumnStorage columnStorage)
{
foreach (var cType in columns)
{
// Build.
IColumn column;
if (cType.FieldType == EFieldType.String)
{
// String.
var data = columnStorage.GetFile(cType.FieldName, EDataType.Data);
var index = columnStorage.GetFile(cType.FieldName, EDataType.Index);
int maxLen = cType.MaxLength ?? DEFAULT_STRING_MAX_LEN;
column = new StringColumn(data, index, maxLen, GetPropertyName(cType.FieldName));
}
else if (cType.FieldType == EFieldType.BitSet)
{
var data = columnStorage.GetFile(cType.FieldName, EDataType.Data);
column = new BitsetColumn(data, columns);
}
else
{
// Fixed size.
var data = columnStorage.GetFile(cType.FieldName, EDataType.Data);
column = new FixedColumn(data, cType.FieldType, GetPropertyName(cType.FieldName));
}
yield return(column);
}
}
private IList <ColumnMetadata> ParseColumns(IEnumerable <IColumnSerializerMetadata> columnsMetadata, List <ColumnElement> columns)
{
// Build.
var cols = new List <ColumnMetadata>();
int nullIndex = 0;
foreach (IColumnSerializerMetadata field in columnsMetadata)
{
var nIndex = field.Nullable ? nullIndex++ : -1;
// Type.
switch (field.ColumnType)
{
case EFieldType.Byte:
case EFieldType.Bool:
case EFieldType.Int16:
case EFieldType.Int32:
case EFieldType.Int64:
case EFieldType.Double:
cols.Add(ColumnMetadata.FromFixedField(field, cols.Count, nIndex));
break;
case EFieldType.DateTime:
case EFieldType.DateTimeEpochMs:
// Check config.
var dateTimeConfig = columns
.FirstOrDefault(c => c.Name.Equals(field.PropertyName, StringComparison.OrdinalIgnoreCase));
if (dateTimeConfig != null && dateTimeConfig.ColumnType == EFieldType.DateTimeEpochMs)
{
field.ColumnType = EFieldType.DateTimeEpochMs;
}
cols.Add(ColumnMetadata.FromFixedField(field, cols.Count, nIndex));
break;
case EFieldType.Symbol:
case EFieldType.String:
// Check config.
var stringConfig = (VarLenColumnElement)columns
.FirstOrDefault(c => c.Name.Equals(field.PropertyName,
StringComparison.OrdinalIgnoreCase));
if (stringConfig != null)
{
cols.Add(ColumnMetadata.FromColumnElement(field, stringConfig, cols.Count, nIndex));
}
else
{
// No config.
cols.Add(ColumnMetadata.FromStringField(field,
MetadataConstants.DEFAULT_STRING_AVG_SIZE,
MetadataConstants.DEFAULT_STRING_MAX_SIZE,
cols.Count, nIndex));
}
break;
case EFieldType.Binary:
var binaryConfig = (VarLenColumnElement)columns
.FirstOrDefault(c => c.Name.Equals(field.PropertyName,
StringComparison.OrdinalIgnoreCase));
if (binaryConfig != null)
{
cols.Add(ColumnMetadata.FromColumnElement(field, binaryConfig, cols.Count, nIndex));
}
else
{
// No config.
cols.Add(ColumnMetadata.FromBinaryField(field,
MetadataConstants.DEFAULT_BINARY_AVG_SIZE,
MetadataConstants.DEFAULT_BINARY_MAX_SIZE,
cols.Count, nIndex));
}
break;
case EFieldType.BitSet:
var fieldSize = BitsetColumn.CalculateSize(field.Size);
cols.Add(ColumnMetadata.FromBitsetField(field, fieldSize, cols.Count));
break;
default:
throw new ArgumentOutOfRangeException();
}
}
return(cols);
}
private ColumnSource[] CreateColumnsFromColumnMetadata(IList <ColumnMetadata> columns,
IColumnStorage columnStorage, PartitionConfig configOverride, int partitionID)
{
var resultColumns = new ColumnSource[columns.Count];
var recordHint = _settings.RecordHint;
if (configOverride != null)
{
columns = ParseColumns(columns.Select(c => c.SerializerMetadata), configOverride.Columns);
recordHint = configOverride.RecordHint;
}
int fileID = 0;
for (int columnID = 0; columnID < columns.Count; columnID++)
{
var cType = columns[columnID];
// Build.
IColumn column;
if (cType.ColumnType == EFieldType.String)
{
// String.
var data = columnStorage.GetFile(cType, fileID++, EDataType.Data, recordHint);
var index = columnStorage.GetFile(cType, fileID++, EDataType.Index, recordHint);
column = new StringColumn(data, index, cType.MaxSize, GetPropertyName(cType.FileName));
}
else if (cType.ColumnType == EFieldType.BitSet)
{
var data = columnStorage.GetFile(cType, fileID++, EDataType.Data, recordHint);
column = new BitsetColumn(data, cType.MaxSize);
}
else if (cType.ColumnType == EFieldType.Symbol)
{
var colData = columnStorage.GetFile(cType, fileID++, EDataType.Data, recordHint);
var symData = columnStorage.GetFile(cType, fileID++, EDataType.Symd, recordHint);
var symi = columnStorage.GetFile(cType, fileID++, EDataType.Symi, recordHint);
var symk = columnStorage.GetFile(cType, fileID++, EDataType.Symrk, recordHint);
var symr = columnStorage.GetFile(cType, fileID++, EDataType.Symrr, recordHint);
int maxLen = cType.MaxSize;
int distinctHintCount = cType.HintDistinctCount;
if (cType.Indexed)
{
var colDataK = columnStorage.GetFile(cType, fileID++, EDataType.Datak, recordHint);
var colDataR = columnStorage.GetFile(cType, fileID++, EDataType.Datar, recordHint);
column = new SymbolMapColumn(
columnID,
partitionID,
data: colData,
datak: colDataK,
datar: colDataR,
symd: symData,
symi: symi,
symk: symk,
symr: symr,
propertyName: GetPropertyName(cType.FileName),
capacity: distinctHintCount,
recordCountHint: _settings.RecordHint,
maxLen: maxLen);
}
else
{
column = new SymbolMapColumn(
columnID,
partitionID,
data: colData,
symd: symData,
symi: symi,
symk: symk,
symr: symr,
propertyName: GetPropertyName(cType.FileName),
capacity: distinctHintCount,
maxLen: maxLen);
}
}
else if (cType.ColumnType == EFieldType.Binary)
{
// Byte array.
var data = columnStorage.GetFile(cType, fileID++, EDataType.Data, recordHint);
var index = columnStorage.GetFile(cType, fileID++, EDataType.Index, recordHint);
column = new BinaryColumn(data, index, cType.MaxSize, GetPropertyName(cType.FileName));
}
else
{
// Fixed size.
var data = columnStorage.GetFile(cType, fileID++, EDataType.Data, recordHint);
column = new FixedColumn(data, cType.ColumnType, GetPropertyName(cType.FileName));
}
resultColumns[columnID] = new ColumnSource(cType, column, fileID);
}
return(resultColumns);
}
