public void nextVector(LongColumnVector previous, long previousLen) { previous.isRepeating = true; for (int i = 0; i < previousLen; i++) { if (!previous.isNull[i]) { previous.vector[i] = next(); } else { // The default value of null for int type in vectorized // processing is 1, so set that if the value is null previous.vector[i] = 1; } // The default value for nulls in Vectorization for int types is 1 // and given that non null value can also be 1, we need to check for isNull also // when determining the isRepeating flag. if (previous.isRepeating && i > 0 && (previous.vector[i - 1] != previous.vector[i] || previous.isNull[i - 1] != previous.isNull[i])) { previous.isRepeating = false; } } }
// Copy the current object contents into the output. Only copy selected entries, // as indicated by selectedInUse and the sel array. public void copySelected(bool selectedInUse, int[] sel, int size, LongColumnVector output) { // Output has nulls if and only if input has nulls. output.noNulls = noNulls; output.isRepeating = false; // Handle repeating case if (isRepeating) { output.vector[0] = vector[0]; output.isNull[0] = isNull[0]; output.isRepeating = true; return; } // Handle normal case // Copy data values over if (selectedInUse) { for (int j = 0; j < size; j++) { int i = sel[j]; output.vector[i] = vector[i]; } } else { Array.Copy(vector, 0, output.vector, 0, size); } // Copy nulls over if needed if (!noNulls) { if (selectedInUse) { for (int j = 0; j < size; j++) { int i = sel[j]; output.isNull[i] = isNull[i]; } } else { Array.Copy(isNull, 0, output.isNull, 0, size); } } }
// Copy the current object contents into the output. Only copy selected entries, // as indicated by selectedInUse and the sel array. public void copySelected(bool selectedInUse, int[] sel, int size, LongColumnVector output) { // Output has nulls if and only if input has nulls. output.noNulls = noNulls; output.isRepeating = false; // Handle repeating case if (isRepeating) { output.vector[0] = vector[0]; output.isNull[0] = isNull[0]; output.isRepeating = true; return; } // Handle normal case // Copy data values over if (selectedInUse) { for (int j = 0; j < size; j++) { int i = sel[j]; output.vector[i] = vector[i]; } } else { Array.Copy(vector, 0, output.vector, 0, size); } // Copy nulls over if needed if (!noNulls) { if (selectedInUse) { for (int j = 0; j < size; j++) { int i = sel[j]; output.isNull[i] = isNull[i]; } } else { Array.Copy(isNull, 0, output.isNull, 0, size); } } }
public void testTimestamp() { ObjectInspector inspector = ObjectInspectorFactory.getReflectionObjectInspector(typeof(Timestamp)); TypeDescription schema = TypeDescription.createTimestamp(); List<Timestamp> tslist = new List<Timestamp>(); using (Stream file = File.OpenWrite(TestFilePath)) using (Writer writer = OrcFile.createWriter(TestFilePath, file, OrcFile.writerOptions(conf) .setSchema(schema) .stripeSize(100000) .bufferSize(10000) .version(OrcFile.Version.V_0_11))) { tslist.Add(Timestamp.Parse("2037-01-01 00:00:00.000999")); tslist.Add(Timestamp.Parse("2003-01-01 00:00:00.000000222")); tslist.Add(Timestamp.Parse("1999-01-01 00:00:00.999999999")); tslist.Add(Timestamp.Parse("1995-01-01 00:00:00.688888888")); tslist.Add(Timestamp.Parse("2002-01-01 00:00:00.1")); tslist.Add(Timestamp.Parse("2010-03-02 00:00:00.000009001")); tslist.Add(Timestamp.Parse("2005-01-01 00:00:00.000002229")); tslist.Add(Timestamp.Parse("2006-01-01 00:00:00.900203003")); tslist.Add(Timestamp.Parse("2003-01-01 00:00:00.800000007")); tslist.Add(Timestamp.Parse("1996-08-02 00:00:00.723100809")); tslist.Add(Timestamp.Parse("1998-11-02 00:00:00.857340643")); tslist.Add(Timestamp.Parse("2008-10-02 00:00:00")); VectorizedRowBatch batch = new VectorizedRowBatch(1, 1024); LongColumnVector vec = new LongColumnVector(1024); batch.cols[0] = vec; batch.reset(); batch.size = tslist.Count; for (int i = 0; i < tslist.Count; ++i) { Timestamp ts = tslist[i]; vec.vector[i] = ts.Nanoseconds; } writer.addRowBatch(batch); schema = writer.getSchema(); } Reader reader = OrcFile.createReader(TestFilePath, OrcFile.readerOptions(conf)); using (RecordReader rows = reader.rows()) { int idx = 0; while (rows.hasNext()) { object row = rows.next(); Assert.Equal(tslist[idx++].getNanos(), ((Timestamp)row).getNanos()); } Assert.Equal(tslist.Count, rows.getRowNumber()); Assert.Equal(0, schema.getMaximumId()); bool[] expected = new bool[] { false }; bool[] included = OrcUtils.includeColumns("", schema); Assert.Equal(expected, included); } }