private IEnumerable <IColumn> CreateColumnsFromColumnMetadata(IArray <ColumnMetadata> columns, IColumnStorage columnStorage) { foreach (var cType in columns) { // Build. IColumn column; if (cType.FieldType == EFieldType.String) { // String. var data = columnStorage.GetFile(cType.FieldName, EDataType.Data); var index = columnStorage.GetFile(cType.FieldName, EDataType.Index); int maxLen = cType.MaxLength ?? DEFAULT_STRING_MAX_LEN; column = new StringColumn(data, index, maxLen, GetPropertyName(cType.FieldName)); } else if (cType.FieldType == EFieldType.BitSet) { var data = columnStorage.GetFile(cType.FieldName, EDataType.Data); column = new BitsetColumn(data, columns); } else { // Fixed size. var data = columnStorage.GetFile(cType.FieldName, EDataType.Data); column = new FixedColumn(data, cType.FieldType, GetPropertyName(cType.FieldName)); } yield return(column); } }
private IList <ColumnMetadata> ParseColumns(IEnumerable <IColumnSerializerMetadata> columnsMetadata, List <ColumnElement> columns) { // Build. var cols = new List <ColumnMetadata>(); int nullIndex = 0; foreach (IColumnSerializerMetadata field in columnsMetadata) { var nIndex = field.Nullable ? nullIndex++ : -1; // Type. switch (field.ColumnType) { case EFieldType.Byte: case EFieldType.Bool: case EFieldType.Int16: case EFieldType.Int32: case EFieldType.Int64: case EFieldType.Double: cols.Add(ColumnMetadata.FromFixedField(field, cols.Count, nIndex)); break; case EFieldType.DateTime: case EFieldType.DateTimeEpochMs: // Check config. var dateTimeConfig = columns .FirstOrDefault(c => c.Name.Equals(field.PropertyName, StringComparison.OrdinalIgnoreCase)); if (dateTimeConfig != null && dateTimeConfig.ColumnType == EFieldType.DateTimeEpochMs) { field.ColumnType = EFieldType.DateTimeEpochMs; } cols.Add(ColumnMetadata.FromFixedField(field, cols.Count, nIndex)); break; case EFieldType.Symbol: case EFieldType.String: // Check config. var stringConfig = (VarLenColumnElement)columns .FirstOrDefault(c => c.Name.Equals(field.PropertyName, StringComparison.OrdinalIgnoreCase)); if (stringConfig != null) { cols.Add(ColumnMetadata.FromColumnElement(field, stringConfig, cols.Count, nIndex)); } else { // No config. cols.Add(ColumnMetadata.FromStringField(field, MetadataConstants.DEFAULT_STRING_AVG_SIZE, MetadataConstants.DEFAULT_STRING_MAX_SIZE, cols.Count, nIndex)); } break; case EFieldType.Binary: var binaryConfig = (VarLenColumnElement)columns .FirstOrDefault(c => c.Name.Equals(field.PropertyName, StringComparison.OrdinalIgnoreCase)); if (binaryConfig != null) { cols.Add(ColumnMetadata.FromColumnElement(field, binaryConfig, cols.Count, nIndex)); } else { // No config. cols.Add(ColumnMetadata.FromBinaryField(field, MetadataConstants.DEFAULT_BINARY_AVG_SIZE, MetadataConstants.DEFAULT_BINARY_MAX_SIZE, cols.Count, nIndex)); } break; case EFieldType.BitSet: var fieldSize = BitsetColumn.CalculateSize(field.Size); cols.Add(ColumnMetadata.FromBitsetField(field, fieldSize, cols.Count)); break; default: throw new ArgumentOutOfRangeException(); } } return(cols); }
private ColumnSource[] CreateColumnsFromColumnMetadata(IList <ColumnMetadata> columns, IColumnStorage columnStorage, PartitionConfig configOverride, int partitionID) { var resultColumns = new ColumnSource[columns.Count]; var recordHint = _settings.RecordHint; if (configOverride != null) { columns = ParseColumns(columns.Select(c => c.SerializerMetadata), configOverride.Columns); recordHint = configOverride.RecordHint; } int fileID = 0; for (int columnID = 0; columnID < columns.Count; columnID++) { var cType = columns[columnID]; // Build. IColumn column; if (cType.ColumnType == EFieldType.String) { // String. var data = columnStorage.GetFile(cType, fileID++, EDataType.Data, recordHint); var index = columnStorage.GetFile(cType, fileID++, EDataType.Index, recordHint); column = new StringColumn(data, index, cType.MaxSize, GetPropertyName(cType.FileName)); } else if (cType.ColumnType == EFieldType.BitSet) { var data = columnStorage.GetFile(cType, fileID++, EDataType.Data, recordHint); column = new BitsetColumn(data, cType.MaxSize); } else if (cType.ColumnType == EFieldType.Symbol) { var colData = columnStorage.GetFile(cType, fileID++, EDataType.Data, recordHint); var symData = columnStorage.GetFile(cType, fileID++, EDataType.Symd, recordHint); var symi = columnStorage.GetFile(cType, fileID++, EDataType.Symi, recordHint); var symk = columnStorage.GetFile(cType, fileID++, EDataType.Symrk, recordHint); var symr = columnStorage.GetFile(cType, fileID++, EDataType.Symrr, recordHint); int maxLen = cType.MaxSize; int distinctHintCount = cType.HintDistinctCount; if (cType.Indexed) { var colDataK = columnStorage.GetFile(cType, fileID++, EDataType.Datak, recordHint); var colDataR = columnStorage.GetFile(cType, fileID++, EDataType.Datar, recordHint); column = new SymbolMapColumn( columnID, partitionID, data: colData, datak: colDataK, datar: colDataR, symd: symData, symi: symi, symk: symk, symr: symr, propertyName: GetPropertyName(cType.FileName), capacity: distinctHintCount, recordCountHint: _settings.RecordHint, maxLen: maxLen); } else { column = new SymbolMapColumn( columnID, partitionID, data: colData, symd: symData, symi: symi, symk: symk, symr: symr, propertyName: GetPropertyName(cType.FileName), capacity: distinctHintCount, maxLen: maxLen); } } else if (cType.ColumnType == EFieldType.Binary) { // Byte array. var data = columnStorage.GetFile(cType, fileID++, EDataType.Data, recordHint); var index = columnStorage.GetFile(cType, fileID++, EDataType.Index, recordHint); column = new BinaryColumn(data, index, cType.MaxSize, GetPropertyName(cType.FileName)); } else { // Fixed size. var data = columnStorage.GetFile(cType, fileID++, EDataType.Data, recordHint); column = new FixedColumn(data, cType.ColumnType, GetPropertyName(cType.FileName)); } resultColumns[columnID] = new ColumnSource(cType, column, fileID); } return(resultColumns); }