public static TypeDescription getDesiredRowTypeDescr(Configuration conf) { string columnNameProperty = null; string columnTypeProperty = null; IList <string> schemaEvolutionColumnNames = null; List <TypeDescription> schemaEvolutionTypeDescrs = null; bool haveSchemaEvolutionProperties = false; if (HiveConf.getBoolVar(conf, ConfVars.HIVE_SCHEMA_EVOLUTION)) { columnNameProperty = conf.get(IOConstants.SCHEMA_EVOLUTION_COLUMNS); columnTypeProperty = conf.get(IOConstants.SCHEMA_EVOLUTION_COLUMNS_TYPES); haveSchemaEvolutionProperties = (columnNameProperty != null && columnTypeProperty != null); if (haveSchemaEvolutionProperties) { schemaEvolutionColumnNames = columnNameProperty.Split(','); if (schemaEvolutionColumnNames.Count == 0) { haveSchemaEvolutionProperties = false; } else { schemaEvolutionTypeDescrs = OrcUtils.typeDescriptionsFromHiveTypeProperty(columnTypeProperty); if (schemaEvolutionTypeDescrs.Count != schemaEvolutionColumnNames.Count) { haveSchemaEvolutionProperties = false; } } } } if (!haveSchemaEvolutionProperties) { // Try regular properties; columnNameProperty = conf.get(serdeConstants.LIST_COLUMNS); columnTypeProperty = conf.get(serdeConstants.LIST_COLUMN_TYPES); if (columnTypeProperty == null || columnNameProperty == null) { return(null); } schemaEvolutionColumnNames = columnNameProperty.Split(','); if (schemaEvolutionColumnNames.Count == 0) { return(null); } schemaEvolutionTypeDescrs = OrcUtils.typeDescriptionsFromHiveTypeProperty(columnTypeProperty); if (schemaEvolutionTypeDescrs.Count != schemaEvolutionColumnNames.Count) { return(null); } } // Desired schema does not include virtual columns or partition columns. TypeDescription result = TypeDescription.createStruct(); for (int i = 0; i < schemaEvolutionColumnNames.Count; i++) { result.addField(schemaEvolutionColumnNames[i], schemaEvolutionTypeDescrs[i]); } return(result); }
public static TypeDescription convertTypeInfo(TypeInfo info) { switch (info.getCategory()) { case ObjectInspectorCategory.PRIMITIVE: { PrimitiveTypeInfo pinfo = (PrimitiveTypeInfo)info; switch (pinfo.getPrimitiveCategory()) { case PrimitiveCategory.BOOLEAN: return(TypeDescription.createBoolean()); case PrimitiveCategory.BYTE: return(TypeDescription.createByte()); case PrimitiveCategory.SHORT: return(TypeDescription.createShort()); case PrimitiveCategory.INT: return(TypeDescription.createInt()); case PrimitiveCategory.LONG: return(TypeDescription.createLong()); case PrimitiveCategory.FLOAT: return(TypeDescription.createFloat()); case PrimitiveCategory.DOUBLE: return(TypeDescription.createDouble()); case PrimitiveCategory.STRING: return(TypeDescription.createString()); case PrimitiveCategory.DATE: return(TypeDescription.createDate()); case PrimitiveCategory.TIMESTAMP: return(TypeDescription.createTimestamp()); case PrimitiveCategory.BINARY: return(TypeDescription.createBinary()); case PrimitiveCategory.DECIMAL: { DecimalTypeInfo dinfo = (DecimalTypeInfo)pinfo; return(TypeDescription.createDecimal() .withScale(dinfo.scale()) .withPrecision(dinfo.precision())); } case PrimitiveCategory.VARCHAR: { BaseCharTypeInfo cinfo = (BaseCharTypeInfo)pinfo; return(TypeDescription.createVarchar() .withMaxLength(cinfo.getLength())); } case PrimitiveCategory.CHAR: { BaseCharTypeInfo cinfo = (BaseCharTypeInfo)pinfo; return(TypeDescription.createChar() .withMaxLength(cinfo.getLength())); } default: throw new ArgumentException("ORC doesn't handle primitive" + " category " + pinfo.getPrimitiveCategory()); } } case ObjectInspectorCategory.LIST: { ListTypeInfo linfo = (ListTypeInfo)info; return(TypeDescription.createList (convertTypeInfo(linfo.getListElementTypeInfo()))); } case ObjectInspectorCategory.MAP: { MapTypeInfo minfo = (MapTypeInfo)info; return(TypeDescription.createMap (convertTypeInfo(minfo.getMapKeyTypeInfo()), convertTypeInfo(minfo.getMapValueTypeInfo()))); } case ObjectInspectorCategory.UNION: { UnionTypeInfo minfo = (UnionTypeInfo)info; TypeDescription result = TypeDescription.createUnion(); foreach (TypeInfo child in minfo.getAllUnionObjectTypeInfos()) { result.addUnionChild(convertTypeInfo(child)); } return(result); } case ObjectInspectorCategory.STRUCT: { StructTypeInfo sinfo = (StructTypeInfo)info; TypeDescription result = TypeDescription.createStruct(); foreach (string fieldName in sinfo.getAllStructFieldNames()) { result.addField(fieldName, convertTypeInfo(sinfo.getStructFieldTypeInfo(fieldName))); } return(result); } default: throw new ArgumentException("ORC doesn't handle " + info.getCategory()); } }