private static void writeTypes(OrcProto.Footer.Builder builder, TypeDescription schema) { OrcProto.Type.Builder type = OrcProto.Type.CreateBuilder(); IList<TypeDescription> children = schema.getChildren(); switch (schema.getCategory()) { case Category.BOOLEAN: type.Kind = OrcProto.Type.Types.Kind.BOOLEAN; break; case Category.BYTE: type.Kind = OrcProto.Type.Types.Kind.BYTE; break; case Category.SHORT: type.Kind = OrcProto.Type.Types.Kind.SHORT; break; case Category.INT: type.Kind = OrcProto.Type.Types.Kind.INT; break; case Category.LONG: type.Kind = OrcProto.Type.Types.Kind.LONG; break; case Category.FLOAT: type.Kind = OrcProto.Type.Types.Kind.FLOAT; break; case Category.DOUBLE: type.Kind = OrcProto.Type.Types.Kind.DOUBLE; break; case Category.STRING: type.Kind = OrcProto.Type.Types.Kind.STRING; break; case Category.CHAR: type.Kind = OrcProto.Type.Types.Kind.CHAR; type.MaximumLength = (uint)schema.getMaxLength(); break; case Category.VARCHAR: type.Kind = OrcProto.Type.Types.Kind.VARCHAR; type.MaximumLength = (uint)schema.getMaxLength(); break; case Category.BINARY: type.Kind = OrcProto.Type.Types.Kind.BINARY; break; case Category.TIMESTAMP: type.Kind = OrcProto.Type.Types.Kind.TIMESTAMP; break; case Category.DATE: type.Kind = OrcProto.Type.Types.Kind.DATE; break; case Category.DECIMAL: type.Kind = OrcProto.Type.Types.Kind.DECIMAL; type.Precision = (uint)schema.getPrecision(); type.Scale = (uint)schema.getScale(); break; case Category.LIST: type.Kind = OrcProto.Type.Types.Kind.LIST; type.AddSubtypes((uint)children[0].getId()); break; case Category.MAP: type.Kind = OrcProto.Type.Types.Kind.MAP; foreach (TypeDescription t in children) { type.AddSubtypes((uint)t.getId()); } break; case Category.STRUCT: type.Kind = OrcProto.Type.Types.Kind.STRUCT; foreach (TypeDescription t in children) { type.AddSubtypes((uint)t.getId()); } foreach (string field in schema.getFieldNames()) { type.AddFieldNames(field); } break; case Category.UNION: type.Kind = OrcProto.Type.Types.Kind.UNION; foreach (TypeDescription t in children) { type.AddSubtypes((uint)t.getId()); } break; default: throw new ArgumentException("Unknown category: " + schema.getCategory()); } builder.AddTypes(type); if (children != null) { foreach (TypeDescription child in children) { writeTypes(builder, child); } } }
private long getRawDataSize(TreeWriter child, TypeDescription schema) { long total = 0; long numVals = child.fileStatistics.getNumberOfValues(); switch (schema.getCategory()) { case Category.BOOLEAN: case Category.BYTE: case Category.SHORT: case Category.INT: case Category.FLOAT: return numVals * JavaDataModel.Four; case Category.LONG: case Category.DOUBLE: return numVals * JavaDataModel.Eight; case Category.STRING: case Category.VARCHAR: case Category.CHAR: // ORC strings are converted to java Strings. so use JavaDataModel to // compute the overall size of strings StringColumnStatistics scs = (StringColumnStatistics)child.fileStatistics; numVals = numVals == 0 ? 1 : numVals; int avgStringLen = (int)(scs.getSum() / numVals); return numVals * JavaDataModel.lengthForStringOfLength(avgStringLen); case Category.DECIMAL: return numVals * JavaDataModel.lengthOfDecimal(); case Category.DATE: return numVals * JavaDataModel.lengthOfDate(); case Category.BINARY: // get total length of binary blob BinaryColumnStatistics bcs = (BinaryColumnStatistics)child.fileStatistics; return bcs.getSum(); case Category.TIMESTAMP: return numVals * JavaDataModel.lengthOfTimestamp(); case Category.LIST: case Category.MAP: case Category.UNION: case Category.STRUCT: { TreeWriter[] childWriters = child.getChildrenWriters(); IList<TypeDescription> childTypes = schema.getChildren(); for (int i = 0; i < childWriters.Length; ++i) { total += getRawDataSize(childWriters[i], childTypes[i]); } break; } default: LOG.debug("Unknown object inspector category."); break; } return total; }
public StructTreeWriter(int columnId, ObjectInspector inspector, TypeDescription schema, StreamFactory writer, bool nullable) : base(columnId, inspector, schema, writer, nullable) { IList<TypeDescription> children = schema.getChildren(); if (inspector != null) { StructObjectInspector structObjectInspector = (StructObjectInspector)inspector; fields = structObjectInspector.getAllStructFieldRefs(); } childrenWriters = new TreeWriter[children.Count]; for (int i = 0; i < childrenWriters.Length; ++i) { ObjectInspector childOI; if (fields != null && i < fields.Count) { childOI = fields[i].getFieldObjectInspector(); } else { childOI = null; } childrenWriters[i] = createTreeWriter( childOI, children[i], writer, true); } recordPosition(rowIndexPosition); }
public UnionTreeWriter(int columnId, ObjectInspector inspector, TypeDescription schema, StreamFactory writer, bool nullable) : base(columnId, inspector, schema, writer, nullable) { IList<ObjectInspector> choices = null; if (inspector != null) { UnionObjectInspector insp = (UnionObjectInspector)inspector; choices = insp.getObjectInspectors(); } IList<TypeDescription> children = schema.getChildren(); childrenWriters = new TreeWriter[children.Count]; for (int i = 0; i < childrenWriters.Length; ++i) { childrenWriters[i] = createTreeWriter( choices != null ? choices[i] : null, children[i], writer, true); } tags = new RunLengthByteWriter(writer.createStream(columnId, OrcProto.Stream.Types.Kind.DATA)); recordPosition(rowIndexPosition); }
public MapTreeWriter(int columnId, ObjectInspector inspector, TypeDescription schema, StreamFactory writer, bool nullable) : base(columnId, inspector, schema, writer, nullable) { this.isDirectV2 = isNewWriteFormat(writer); ObjectInspector keyInspector = null; ObjectInspector valueInspector = null; if (inspector != null) { MapObjectInspector insp = (MapObjectInspector)inspector; keyInspector = insp.getMapKeyObjectInspector(); valueInspector = insp.getMapValueObjectInspector(); } childrenWriters = new TreeWriter[2]; IList<TypeDescription> children = schema.getChildren(); childrenWriters[0] = createTreeWriter(keyInspector, children[0], writer, true); childrenWriters[1] = createTreeWriter(valueInspector, children[1], writer, true); lengths = createIntegerWriter(writer.createStream(columnId, OrcProto.Stream.Types.Kind.LENGTH), false, isDirectV2, writer); recordPosition(rowIndexPosition); }
public ListTreeWriter(int columnId, ObjectInspector inspector, TypeDescription schema, StreamFactory writer, bool nullable) : base(columnId, inspector, schema, writer, nullable) { this.isDirectV2 = isNewWriteFormat(writer); ObjectInspector childOI = inspector == null ? null : ((ListObjectInspector)inspector).getListElementObjectInspector(); childrenWriters = new TreeWriter[1]; childrenWriters[0] = createTreeWriter(childOI, schema.getChildren()[0], writer, true); lengths = createIntegerWriter(writer.createStream(columnId, OrcProto.Stream.Types.Kind.LENGTH), false, isDirectV2, writer); recordPosition(rowIndexPosition); }
private static void appendOrcTypes(List<OrcProto.Type> result, TypeDescription typeDescr) { OrcProto.Type.Builder type = OrcProto.Type.CreateBuilder(); IList<TypeDescription> children = typeDescr.getChildren(); switch (typeDescr.getCategory()) { case Category.BOOLEAN: type.SetKind(OrcProto.Type.Types.Kind.BOOLEAN); break; case Category.BYTE: type.SetKind(OrcProto.Type.Types.Kind.BYTE); break; case Category.SHORT: type.SetKind(OrcProto.Type.Types.Kind.SHORT); break; case Category.INT: type.SetKind(OrcProto.Type.Types.Kind.INT); break; case Category.LONG: type.SetKind(OrcProto.Type.Types.Kind.LONG); break; case Category.FLOAT: type.SetKind(OrcProto.Type.Types.Kind.FLOAT); break; case Category.DOUBLE: type.SetKind(OrcProto.Type.Types.Kind.DOUBLE); break; case Category.STRING: type.SetKind(OrcProto.Type.Types.Kind.STRING); break; case Category.CHAR: type.SetKind(OrcProto.Type.Types.Kind.CHAR); type.SetMaximumLength((uint)typeDescr.getMaxLength()); break; case Category.VARCHAR: type.SetKind(OrcProto.Type.Types.Kind.VARCHAR); type.SetMaximumLength((uint)typeDescr.getMaxLength()); break; case Category.BINARY: type.SetKind(OrcProto.Type.Types.Kind.BINARY); break; case Category.TIMESTAMP: type.SetKind(OrcProto.Type.Types.Kind.TIMESTAMP); break; case Category.DATE: type.SetKind(OrcProto.Type.Types.Kind.DATE); break; case Category.DECIMAL: type.SetKind(OrcProto.Type.Types.Kind.DECIMAL); type.SetPrecision((uint)typeDescr.getPrecision()); type.SetScale((uint)typeDescr.getScale()); break; case Category.LIST: type.SetKind(OrcProto.Type.Types.Kind.LIST); type.AddSubtypes((uint)children[0].getId()); break; case Category.MAP: type.SetKind(OrcProto.Type.Types.Kind.MAP); foreach (TypeDescription t in children) { type.AddSubtypes((uint)t.getId()); } break; case Category.STRUCT: type.SetKind(OrcProto.Type.Types.Kind.STRUCT); foreach (TypeDescription t in children) { type.AddSubtypes((uint)t.getId()); } foreach (string field in typeDescr.getFieldNames()) { type.AddFieldNames(field); } break; case Category.UNION: type.SetKind(OrcProto.Type.Types.Kind.UNION); foreach (TypeDescription t in children) { type.AddSubtypes((uint)t.getId()); } break; default: throw new ArgumentException("Unknown category: " + typeDescr.getCategory()); } result.Add(type.Build()); if (children != null) { foreach (TypeDescription child in children) { appendOrcTypes(result, child); } } }
/** * Returns selected columns as a bool array with true value set for specified column names. * The result will contain number of elements equal to flattened number of columns. * For example: * selectedColumns - a,b,c * allColumns - a,b,c,d * If column c is a complex type, say list<string> and other types are primitives then result will * be [false, true, true, true, true, true, false] * Index 0 is the root element of the struct which is set to false by default, index 1,2 * corresponds to columns a and b. Index 3,4 correspond to column c which is list<string> and * index 5 correspond to column d. After flattening list<string> gets 2 columns. * * @param selectedColumns - comma separated list of selected column names * @param schema - object schema * @return - bool array with true value set for the specified column names */ public static bool[] includeColumns(string selectedColumns, TypeDescription schema) { int numFlattenedCols = schema.getMaximumId(); bool[] results = new bool[numFlattenedCols + 1]; if ("*".Equals(selectedColumns)) { for (int i = 0; i < results.Length; i++) { results[i] = true; } return results; } if (selectedColumns != null && schema.getCategory() == Category.STRUCT) { IList<string> fieldNames = schema.getFieldNames(); IList<TypeDescription> fields = schema.getChildren(); foreach (string column in selectedColumns.Split((','))) { TypeDescription col = findColumn(column, fieldNames, fields); if (col != null) { for (int i = col.getId(); i <= col.getMaximumId(); ++i) { results[i] = true; } } } } return results; }
/** * NOTE: This method ignores the subtype numbers in the TypeDescription rebuilds the subtype * numbers based on the length of the result list being appended. * * @param result * @param typeInfo */ public static void appendOrcTypesRebuildSubtypes( IList<OrcProto.Type> result, TypeDescription typeDescr) { int subtype = result.Count; OrcProto.Type.Builder type = OrcProto.Type.CreateBuilder(); bool needsAdd = true; IList<TypeDescription> children = typeDescr.getChildren(); switch (typeDescr.getCategory()) { case Category.BOOLEAN: type.SetKind(OrcProto.Type.Types.Kind.BOOLEAN); break; case Category.BYTE: type.SetKind(OrcProto.Type.Types.Kind.BYTE); break; case Category.SHORT: type.SetKind(OrcProto.Type.Types.Kind.SHORT); break; case Category.INT: type.SetKind(OrcProto.Type.Types.Kind.INT); break; case Category.LONG: type.SetKind(OrcProto.Type.Types.Kind.LONG); break; case Category.FLOAT: type.SetKind(OrcProto.Type.Types.Kind.FLOAT); break; case Category.DOUBLE: type.SetKind(OrcProto.Type.Types.Kind.DOUBLE); break; case Category.STRING: type.SetKind(OrcProto.Type.Types.Kind.STRING); break; case Category.CHAR: type.SetKind(OrcProto.Type.Types.Kind.CHAR); type.SetMaximumLength((uint)typeDescr.getMaxLength()); break; case Category.VARCHAR: type.SetKind(OrcProto.Type.Types.Kind.VARCHAR); type.SetMaximumLength((uint)typeDescr.getMaxLength()); break; case Category.BINARY: type.SetKind(OrcProto.Type.Types.Kind.BINARY); break; case Category.TIMESTAMP: type.SetKind(OrcProto.Type.Types.Kind.TIMESTAMP); break; case Category.DATE: type.SetKind(OrcProto.Type.Types.Kind.DATE); break; case Category.DECIMAL: type.SetKind(OrcProto.Type.Types.Kind.DECIMAL); type.SetPrecision((uint)typeDescr.getPrecision()); type.SetScale((uint)typeDescr.getScale()); break; case Category.LIST: type.SetKind(OrcProto.Type.Types.Kind.LIST); type.AddSubtypes((uint)++subtype); result.Add(type.Build()); needsAdd = false; appendOrcTypesRebuildSubtypes(result, children[0]); break; case Category.MAP: { // Make room for MAP type. result.Add(null); // Add MAP type pair in order to determine their subtype values. appendOrcTypesRebuildSubtypes(result, children[0]); int subtype2 = result.Count; appendOrcTypesRebuildSubtypes(result, children[1]); type.SetKind(OrcProto.Type.Types.Kind.MAP); type.AddSubtypes((uint)subtype + 1); type.AddSubtypes((uint)subtype2); result[subtype] = type.Build(); needsAdd = false; } break; case Category.STRUCT: { IList<String> fieldNames = typeDescr.getFieldNames(); // Make room for STRUCT type. result.Add(null); List<int> fieldSubtypes = new List<int>(fieldNames.Count); foreach (TypeDescription child in children) { int fieldSubtype = result.Count; fieldSubtypes.Add(fieldSubtype); appendOrcTypesRebuildSubtypes(result, child); } type.SetKind(OrcProto.Type.Types.Kind.STRUCT); for (int i = 0; i < fieldNames.Count; i++) { type.AddSubtypes((uint)fieldSubtypes[i]); type.AddFieldNames(fieldNames[i]); } result[subtype] = type.Build(); needsAdd = false; } break; case Category.UNION: { // Make room for UNION type. result.Add(null); List<int> unionSubtypes = new List<int>(children.Count); foreach (TypeDescription child in children) { int unionSubtype = result.Count; unionSubtypes.Add(unionSubtype); appendOrcTypesRebuildSubtypes(result, child); } type.SetKind(OrcProto.Type.Types.Kind.UNION); for (int i = 0; i < children.Count; i++) { type.AddSubtypes((uint)unionSubtypes[i]); } result[subtype] = type.Build(); needsAdd = false; } break; default: throw new ArgumentException("Unknown category: " + typeDescr.getCategory()); } if (needsAdd) { result.Add(type.Build()); } }