private static void writeStripeInformation(JsonWriter writer, StripeInformation stripe)
{
writer.newObject();
writer.key("offset").value(stripe.getOffset());
writer.key("indexLength").value(stripe.getIndexLength());
writer.key("dataLength").value(stripe.getDataLength());
writer.key("footerLength").value(stripe.getFooterLength());
writer.key("rowCount").value(stripe.getNumberOfRows());
writer.endObject();
}
public OrcProto.StripeFooter readStripeFooter(StripeInformation stripe)
{
long offset = stripe.getOffset() + stripe.getIndexLength() + stripe.getDataLength();
int tailLength = (int)stripe.getFooterLength();
// read the footer
ByteBuffer tailBuf = ByteBuffer.allocate(tailLength);
file.readFully(offset, tailBuf.array(), tailBuf.arrayOffset(), tailLength);
return(OrcProto.StripeFooter.ParseFrom(InStream.createCodedInputStream(null, "footer",
new List <DiskRange> {
new RecordReaderImpl.BufferChunk(tailBuf, 0)
},
tailLength, codec, bufferSize)));
}
public void appendStripe(byte[] stripe, int offset, int length,
StripeInformation stripeInfo,
OrcProto.StripeStatistics stripeStatistics)
{
checkArgument(stripe != null, "Stripe must not be null");
checkArgument(length <= stripe.Length,
"Specified length must not be greater specified array length");
checkArgument(stripeInfo != null, "Stripe information must not be null");
checkArgument(stripeStatistics != null,
"Stripe statistics must not be null");
getStream();
long start = rawWriter.Position;
long availBlockSpace = blockSize - (start % blockSize);
// see if stripe can fit in the current hdfs block, else pad the remaining
// space in the block
if (length < blockSize && length > availBlockSpace &&
addBlockPadding)
{
byte[] pad = new byte[(int)Math.Min(HDFS_BUFFER_SIZE, availBlockSpace)];
LOG.info(String.Format("Padding ORC by {0} bytes while merging..",
availBlockSpace));
start += availBlockSpace;
while (availBlockSpace > 0)
{
int writeLen = (int)Math.Min(availBlockSpace, pad.Length);
rawWriter.Write(pad, 0, writeLen);
availBlockSpace -= writeLen;
}
}
rawWriter.Write(stripe, 0, stripe.Length);
rowsInStripe = (long)stripeStatistics.ColStatsList[0].NumberOfValues;
rowCount += rowsInStripe;
// since we have already written the stripe, just update stripe statistics
treeWriter.stripeStatsBuilders.Add(stripeStatistics.ToBuilder());
// update file level statistics
updateFileStatistics(stripeStatistics);
// update stripe information
OrcProto.StripeInformation.Builder dirEntry = OrcProto.StripeInformation.CreateBuilder();
dirEntry.Offset = (ulong)start;
dirEntry.NumberOfRows = (ulong)rowsInStripe;
dirEntry.IndexLength = (ulong)stripeInfo.getIndexLength();
dirEntry.DataLength = (ulong)stripeInfo.getDataLength();
dirEntry.FooterLength = (ulong)stripeInfo.getFooterLength();
stripes.Add(dirEntry.Build());
// reset it after writing the stripe
rowsInStripe = 0;
}
public OrcProto.StripeFooter readStripeFooter(StripeInformation stripe)
{
long offset = stripe.getOffset() + stripe.getIndexLength() + stripe.getDataLength();
int tailLength = (int)stripe.getFooterLength();
// read the footer
ByteBuffer tailBuf = ByteBuffer.allocate(tailLength);
file.readFully(offset, tailBuf.array(), tailBuf.arrayOffset(), tailLength);
return OrcProto.StripeFooter.ParseFrom(InStream.createCodedInputStream(null, "footer",
new List<DiskRange> { new RecordReaderImpl.BufferChunk(tailBuf, 0) },
tailLength, codec, bufferSize));
}
private static void writeStripeInformation(JsonWriter writer, StripeInformation stripe)
{
writer.newObject();
writer.key("offset").value(stripe.getOffset());
writer.key("indexLength").value(stripe.getIndexLength());
writer.key("dataLength").value(stripe.getDataLength());
writer.key("footerLength").value(stripe.getFooterLength());
writer.key("rowCount").value(stripe.getNumberOfRows());
writer.endObject();
}
