/// <summary>
/// Create a new Blob from an existing ByteBuffer. IMPORTANT: If copy is
/// false, after calling this constructor, if you keep a pointer to the buffer
/// then you must treat it as immutable and promise not to change it.
/// </summary>
///
/// <param name="buffer"></param>
/// <param name="copy"></param>
public Blob(ByteBuffer buffer, bool copy)
{
this.haveHashCode_ = false;
if (buffer != null) {
if (copy) {
buffer_ = ILOG.J2CsMapping.NIO.ByteBuffer.allocate(buffer.remaining());
// Put updates buffer.position(), so save and restore it.
int savePosition = buffer.position();
buffer_.put(buffer);
buffer.position(savePosition);
buffer_.flip();
} else
buffer_ = buffer.slice();
} else
buffer_ = null;
}
private void Write(ByteBuffer buffer, int size)
{
buffer.position(0);
buffer.limit(size);
if (Channel != null)
{
Channel.write(buffer);
}
else if (ResultBuffer != null)
{
ResultBuffer.put(buffer);
}
TotalSize += size;
}
/* slices a read only contiguous buffer of chunkLength */
private ByteBuffer slice(int chunkLength)
{
int len = chunkLength;
long oldOffset = currentOffset;
ByteBuffer slice;
if (compressed.remaining() >= len)
{
slice = compressed.slice();
// simple case
slice.limit(len);
currentOffset += len;
compressed.position(compressed.position() + len);
return slice;
}
else if (currentRange >= (bytes.Count - 1))
{
// nothing has been modified yet
throw new IOException("EOF in " + this + " while trying to read " +
chunkLength + " bytes");
}
if (LOG.isDebugEnabled())
{
LOG.debug(String.Format(
"Crossing into next BufferChunk because compressed only has %d bytes (needs %d)",
compressed.remaining(), len));
}
// we need to consolidate 2 or more buffers into 1
// first copy out compressed buffers
ByteBuffer copy = allocateBuffer(chunkLength, compressed.isDirect());
currentOffset += compressed.remaining();
len -= compressed.remaining();
copy.put(compressed);
for (int i = currentRange; i < bytes.Count && len > 0; i++)
{
++currentRange;
if (LOG.isDebugEnabled())
{
LOG.debug(String.Format("Read slow-path, >1 cross block reads with {0}", this.ToString()));
}
DiskRange range = bytes[i];
compressed = range.getData().duplicate();
if (compressed.remaining() >= len)
{
slice = compressed.slice();
slice.limit(len);
copy.put(slice);
currentOffset += len;
compressed.position(compressed.position() + len);
return copy;
}
currentOffset += compressed.remaining();
len -= compressed.remaining();
copy.put(compressed);
}
// restore offsets for exception clarity
seek(oldOffset);
throw new IOException("EOF in " + this + " while trying to read " +
chunkLength + " bytes");
}
/// <summary>
/// Continue to read data until the end of an element, then call
/// elementListener.onReceivedElement(element ). The buffer passed to
/// onReceivedElement is only valid during this call. If you need the data
/// later, you must copy.
/// </summary>
///
/// <param name="data"></param>
/// <exception cref="EncodingException">For invalid encoding.</exception>
public void onReceivedData(ByteBuffer data)
{
// We may repeatedly set data to a slice as we read elements.
data = data.slice();
// Process multiple objects in the data.
while (true) {
bool gotElementEnd;
int offset;
try {
if (!usePartialData_) {
// This is the beginning of an element.
if (data.remaining() <= 0)
// Wait for more data.
return;
}
// Scan the input to check if a whole TLV object has been read.
tlvStructureDecoder_.seek(0);
gotElementEnd = tlvStructureDecoder_.findElementEnd(data);
offset = tlvStructureDecoder_.getOffset();
} catch (EncodingException ex) {
// Reset to read a new element on the next call.
usePartialData_ = false;
tlvStructureDecoder_ = new TlvStructureDecoder();
throw ex;
}
if (gotElementEnd) {
// Got the remainder of an element. Report to the caller.
ByteBuffer element;
if (usePartialData_) {
// We have partial data from a previous call, so append this data and point to partialData.
partialData_.ensuredPut(data, 0, offset);
element = partialData_.flippedBuffer();
// Assume we don't need to use partialData anymore until needed.
usePartialData_ = false;
} else {
// We are not using partialData, so just point to the input data buffer.
element = data.duplicate();
element.limit(offset);
}
// Reset to read a new object. Do this before calling onReceivedElement
// in case it throws an exception.
data.position(offset);
data = data.slice();
tlvStructureDecoder_ = new TlvStructureDecoder();
elementListener_.onReceivedElement(element);
if (data.remaining() <= 0)
// No more data in the packet.
return;
// else loop back to decode.
} else {
// Save remaining data for a later call.
if (!usePartialData_) {
usePartialData_ = true;
partialData_.position(0);
}
if (partialData_.buffer().position() + data.remaining() > net.named_data.jndn.util.Common.MAX_NDN_PACKET_SIZE) {
// Reset to read a new element on the next call.
usePartialData_ = false;
tlvStructureDecoder_ = new TlvStructureDecoder();
throw new EncodingException(
"The incoming packet exceeds the maximum limit Face.getMaxNdnPacketSize()");
}
partialData_.ensuredPut(data);
return;
}
}
}
public void seek(long desired)
{
if (desired == 0 && bytes.Count == 0)
{
logEmptySeek(name);
return;
}
int i = 0;
foreach (DiskRange curRange in bytes)
{
if (desired == 0 && curRange.getData().remaining() == 0)
{
logEmptySeek(name);
return;
}
if (curRange.getOffset() <= desired &&
(desired - curRange.getOffset()) < curRange.getLength())
{
currentOffset = desired;
currentRange = i;
this.range = curRange.getData().duplicate();
int pos = range.position();
pos += (int)(desired - curRange.getOffset()); // this is why we duplicate
this.range.position(pos);
return;
}
++i;
}
// if they are seeking to the precise end, go ahead and let them go there
int segments = bytes.Count;
if (segments != 0 && desired == bytes[segments - 1].getEnd())
{
currentOffset = desired;
currentRange = segments - 1;
DiskRange curRange = bytes[currentRange];
this.range = curRange.getData().duplicate();
int pos = range.position();
pos += (int)(desired - curRange.getOffset()); // this is why we duplicate
this.range.position(pos);
return;
}
throw new ArgumentException("Seek in " + name + " to " +
desired + " is outside of the data");
}
private void seek(long desired)
{
if (desired == 0 && bytes.Count == 0)
{
logEmptySeek(name);
return;
}
int i = 0;
foreach (DiskRange range in bytes)
{
if (range.getOffset() <= desired && desired < range.getEnd())
{
currentRange = i;
compressed = range.getData().duplicate();
int pos = compressed.position();
pos += (int)(desired - range.getOffset());
compressed.position(pos);
currentOffset = desired;
return;
}
++i;
}
// if they are seeking to the precise end, go ahead and let them go there
int segments = bytes.Count;
if (segments != 0 && desired == bytes[segments - 1].getEnd())
{
DiskRange range = bytes[segments - 1];
currentRange = segments - 1;
compressed = range.getData().duplicate();
compressed.position(compressed.limit());
currentOffset = desired;
return;
}
throw new IOException("Seek outside of data in " + this + " to " + desired);
}
/// <summary>
/// Call ensureCapacity to ensure there is capacity for buffer.remaining() more
/// bytes and use buffer().put to copy.
/// This increments the position by buffer.remaining().
/// This does update buffer's position to its limit.
/// </summary>
///
/// <param name="buffer"></param>
public void ensuredPut(ByteBuffer buffer)
{
ensureRemainingCapacity(buffer.remaining());
int savePosition = buffer.position();
buffer_.put(buffer);
buffer.position(savePosition);
}
/// <summary>
/// Write the buffer from its position() to limit() to the output just
/// before getLength() from the back. Advance getLength() of the output. This
/// does NOT change buffer.position(). Note that this does not encode a type
/// and length; for that see writeBlobTlv.
/// </summary>
///
/// <param name="buffer"></param>
public void writeBuffer(ByteBuffer buffer)
{
if (buffer == null)
return;
// Write backwards.
int position = output_.setRemainingFromBack(output_.remaining()
+ buffer.remaining());
int saveBufferValuePosition = buffer.position();
output_.buffer().put(buffer);
// Restore positions after put.
output_.position(position);
buffer.position(saveBufferValuePosition);
}
/// <summary>
/// Parse the data from the input buffer recursively and return the root as an
/// object of a subclass of DerNode.
/// </summary>
///
/// <param name="inputBuf"></param>
/// <returns>An object of a subclass of DerNode.</returns>
public static DerNode parse(ByteBuffer inputBuf)
{
return parse(inputBuf, inputBuf.position());
}
/// <summary>
/// Write the value to result, escaping characters according to the NDN URI
/// Scheme.
/// This also adds "..." to a value with zero or more ".".
/// This does not add a type code prefix such as "sha256digest=".
/// </summary>
///
/// <param name="value"></param>
/// <param name="result">The StringBuffer to write to.</param>
public static void toEscapedString(ByteBuffer value_ren, StringBuilder result)
{
bool gotNonDot = false;
for (int i = value_ren.position(); i < value_ren.limit(); ++i) {
if (value_ren.get(i) != 0x2e) {
gotNonDot = true;
break;
}
}
if (!gotNonDot) {
// Special case for component of zero or more periods. Add 3 periods.
result.append("...");
for (int i_0 = value_ren.position(); i_0 < value_ren.limit(); ++i_0)
result.append('.');
} else {
for (int i_1 = value_ren.position(); i_1 < value_ren.limit(); ++i_1) {
int x = ((int) value_ren.get(i_1) & 0xff);
// Check for 0-9, A-Z, a-z, (+), (-), (.), (_)
if (x >= 0x30 && x <= 0x39 || x >= 0x41 && x <= 0x5a
|| x >= 0x61 && x <= 0x7a || x == 0x2b || x == 0x2d
|| x == 0x2e || x == 0x5f)
result.append((char) x);
else {
result.append('%');
if (x < 16)
result.append('0');
result.append(ILOG.J2CsMapping.Util.IlNumber.ToString(x,16).ToUpper());
}
}
}
}
/// <summary>
/// Write a hex string of the contents of buffer from position to limit to the
/// output.
/// </summary>
///
/// <param name="buffer">The buffer.</param>
/// <returns>A string of hex bytes.</returns>
/// <param name="output">The StringBuffer to write to.</param>
public static void toHex(ByteBuffer buffer, StringBuilder output)
{
for (int i = buffer.position(); i < buffer.limit(); ++i) {
String hex = ILOG.J2CsMapping.Util.IlNumber.ToString((int) buffer.get(i) & 0xff,16);
if (hex.Length <= 1)
// Append the leading zero.
output.append("0");
output.append(hex);
}
}
public void output(ByteBuffer buffer)
{
_output.Write(buffer.array(), buffer.arrayOffset() + buffer.position(), buffer.remaining());
}
private static int readByteBuffer(Stream file, ByteBuffer dest)
{
int pos = dest.position();
int result = dest.readRemaining(file);
if (result > 0)
{
// Ensure this explicitly since versions before 2.7 read doesn't do it.
dest.position(pos + result);
}
return result;
}
public static void readDirect(Stream file, int len, ByteBuffer directBuf)
{
// TODO: HDFS API is a mess, so handle all kinds of cases.
// Before 2.7, read() also doesn't adjust position correctly, so track it separately.
int pos = directBuf.position(), startPos = pos, endPos = pos + len;
try
{
while (pos < endPos)
{
int count = readByteBuffer(file, directBuf);
if (count < 0) throw new EndOfStreamException();
Debug.Assert(count != 0, "0-length read: " + (endPos - pos) + "@" + (pos - startPos));
pos += count;
Debug.Assert(pos <= endPos, "Position " + pos + " > " + endPos + " after reading " + count);
directBuf.position(pos);
}
}
catch (NotSupportedException)
{
Debug.Assert(pos == startPos);
// Happens in q files and such.
RecordReaderImpl.LOG.error("Stream does not support direct read; we will copy.");
byte[] buffer = new byte[len];
file.readFully(buffer, 0, buffer.Length);
directBuf.put(buffer);
}
directBuf.position(startPos);
directBuf.limit(startPos + len);
}
/// <summary>
/// Compute the HMAC with SHA-256 of data, as defined in
/// http://tools.ietf.org/html/rfc2104#section-2 .
/// </summary>
///
/// <param name="key">The key byte array.</param>
/// <param name="data">The input byte buffer. This does not change the position.</param>
/// <returns>The HMAC result.</returns>
public static byte[] computeHmacWithSha256(byte[] key, ByteBuffer data)
{
using (var hmac = new HMACSHA256(key)) {
// Copy the buffer to an array.
var array = new byte[data.remaining()];
int savePosition = data.position();
data.get(array);
data.position(savePosition);
return hmac.ComputeHash(array);
}
}
/// <summary>
/// Compute the sha-256 digest of data.
/// </summary>
///
/// <param name="data">The input byte buffer. This does not change the position.</param>
/// <returns>The digest.</returns>
public static byte[] digestSha256(ByteBuffer data)
{
// Copy the buffer to an array.
var array = new byte[data.remaining()];
int savePosition = data.position();
data.get(array);
data.position(savePosition);
return sha256_.ComputeHash(array);
}
/// <summary>
/// Call ensureCapacity to ensure there is capacity for (limit - position) more
/// bytes and use buffer().put to copy.
/// This increments the position by (limit - position).
/// </summary>
///
/// <param name="buffer"></param>
/// <param name="position">The position in buffer to copy from.</param>
/// <param name="limit">The limit in buffer to copy from.</param>
public void ensuredPut(ByteBuffer buffer, int position, int limit)
{
ensureRemainingCapacity(limit - position);
int savePosition = buffer.position();
int saveLimit = buffer.limit();
try {
buffer.position(position);
buffer.limit(limit);
buffer_.put(buffer);
} finally {
// put updates buffer's position and limit, so restore.
buffer.position(savePosition);
buffer.limit(saveLimit);
}
}
// Doubles the size of the ByteBuffer, and copies the old data towards the
// end of the new buffer (since we build the buffer backwards).
internal virtual ByteBuffer growByteBuffer(ByteBuffer bb)
{
sbyte[] old_buf = bb.array();
int old_buf_size = old_buf.Length;
if ((old_buf_size & 0xC0000000) != 0) // Ensure we don't grow beyond what fits in an int.
{
throw new AssertionError("FlatBuffers: cannot grow buffer beyond 2 gigabytes.");
}
int new_buf_size = old_buf_size << 1;
sbyte[] new_buf = new sbyte[new_buf_size];
Array.Copy(old_buf, 0, new_buf, new_buf_size - old_buf_size, old_buf_size);
ByteBuffer nbb = newByteBuffer(new_buf);
nbb.position(bb.position());
return nbb;
}
private void getNewInputBuffer()
{
if (codec == null)
{
current = ByteBuffer.allocate(_bufferSize);
}
else
{
current = ByteBuffer.allocate(_bufferSize + HEADER_SIZE);
writeHeader(current, 0, _bufferSize, true);
current.position(HEADER_SIZE);
}
}
/// <summary>
/// Create a new DerInteger from the bytes in the buffer. If bytes represent
/// a positive integer, you must ensure that the first byte is less than 0x80.
/// </summary>
///
/// <param name="buffer"></param>
/// <exception cref="DerEncodingException">if the first byte is not less than 0x80.</exception>
public DerInteger(ByteBuffer buffer)
: base(net.named_data.jndn.encoding.der.DerNodeType.Integer)
{
if (buffer.remaining() > 0
&& (((int) buffer.get(buffer.position())) & 0xff) >= 0x80)
throw new DerEncodingException(
"DerInteger: Negative integers are not currently supported");
if (buffer.remaining() == 0)
payload_.ensuredPut((byte) 0);
else
payload_.ensuredPut(buffer);
encodeHeader(payload_.position());
}
/// <summary>
/// Set data to the host.
/// </summary>
/// <param name="data">The buffer of data to send. This reads from position() to
/// limit(), but does not change the position.</param>
public override void send(ByteBuffer data)
{
if (socket_ == null)
throw new System.IO.IOException
("Cannot send because the socket is not open. Use connect.");
// ByteBuffer is readonly so we can't call array(), and we can't do a low-level
// operation on the array, so we have to copy.
var buffer = new byte[data.remaining()];
int savePosition = data.position();
data.get(buffer);
data.position(savePosition);
socket_.Send(buffer);
}
/**
* Decodes bytes starting at the current position of the given input buffer,
* and writes the equivalent character sequence into the given output buffer
* from its current position.
* <p>
* The buffers' position will be changed with the reading and writing
* operation, but their limits and marks will be kept intact.
* <p>
* A <code>CoderResult</code> instance will be returned according to
* following rules:
* <ul>
* <li>{@link CoderResult#OVERFLOW CoderResult.OVERFLOW} indicates that
* even though not all of the input has been processed, the buffer the
* output is being written to has reached its capacity. In the event of this
* code being returned this method should be called once more with an
* <code>out</code> argument that has not already been filled.</li>
* <li>{@link CoderResult#UNDERFLOW CoderResult.UNDERFLOW} indicates that
* as many bytes as possible in the input buffer have been decoded. If there
* is no further input and no remaining bytes in the input buffer then this
* operation may be regarded as complete. Otherwise, this method should be
* called once more with additional input.</li>
* <li>A {@link CoderResult#malformedForLength(int) malformed input} result
* indicates that some malformed input error has been encountered, and the
* erroneous bytes start at the input buffer's position and their number can
* be got by result's {@link CoderResult#length() length}. This kind of
* result can be returned only if the malformed action is
* {@link CodingErrorAction#REPORT CodingErrorAction.REPORT}. </li>
* <li>A {@link CoderResult#unmappableForLength(int) unmappable character}
* result indicates that some unmappable character error has been
* encountered, and the erroneous bytes start at the input buffer's position
* and their number can be got by result's
* {@link CoderResult#length() length}. This kind of result can be returned
* only if the unmappable character action is
* {@link CodingErrorAction#REPORT CodingErrorAction.REPORT}. </li>
* </ul>
* <p>
* The <code>endOfInput</code> parameter indicates that the invoker cannot
* provide further input. This parameter is true if and only if the bytes in
* current input buffer are all inputs for this decoding operation. Note
* that it is common and won't cause an error if the invoker sets false and
* then can't provide more input, while it may cause an error if the invoker
* always sets true in several consecutive invocations. This would make the
* remaining input to be treated as malformed input.
* <p>
* This method invokes the
* {@link #decodeLoop(ByteBuffer, CharBuffer) decodeLoop} method to
* implement the basic decode logic for a specific charset.
*
* @param in
* the input buffer.
* @param out
* the output buffer.
* @param endOfInput
* true if all the input characters have been provided.
* @return a <code>CoderResult</code> instance which indicates the reason
* of termination.
* @throws IllegalStateException
* if decoding has started or no more input is needed in this
* decoding progress.
* @throws CoderMalfunctionError
* if the {@link #decodeLoop(ByteBuffer, CharBuffer) decodeLoop}
* method threw an <code>BufferUnderflowException</code> or
* <code>BufferOverflowException</code>.
*/
public CoderResult decode(ByteBuffer inJ, CharBuffer outJ,
bool endOfInput)
{
/*
* status check
*/
if ((status == FLUSH) || (!endOfInput && status == END)) {
throw new java.lang.IllegalStateException();
}
CoderResult result = null;
// begin to decode
while (true) {
CodingErrorAction action = null;
try {
result = decodeLoop(inJ, outJ);
} catch (BufferOverflowException ex) {
// unexpected exception
throw new CoderMalfunctionError(ex);
} catch (BufferUnderflowException ex) {
// unexpected exception
throw new CoderMalfunctionError(ex);
}
/*
* result handling
*/
if (result.isUnderflow()) {
int remaining = inJ.remaining(); // WHY inJ.remaining() == 1?
status = endOfInput ? END : ONGOING;
if (endOfInput && remaining > 0) {
result = CoderResult.malformedForLength(remaining);
} else {
return result;
}
}
if (result.isOverflow()) {
return result;
}
// set coding error handle action
action = malformAction;
if (result.isUnmappable()) {
action = unmapAction;
}
// If the action is IGNORE or REPLACE, we should continue decoding.
if (action == CodingErrorAction.REPLACE) {
if (outJ.remaining() < replace.length()) {
return CoderResult.OVERFLOW;
}
outJ.put(replace);
} else {
if (action != CodingErrorAction.IGNORE)
return result;
}
inJ.position(inJ.position() + result.length());
}
}
/**
* Send an object through the telemetry link.
* @throws IOException
* @param[in] obj Object handle to send
* @param[in] type Transaction type \return Success (true), Failure (false)
*/
private bool transmitSingleObject(UAVObject obj, int type, bool allInstances)
{
int length;
int allInstId = uavConsts.ALL_INSTANCES;
ByteBuffer bbuf = new ByteBuffer(uavConsts.MAX_PACKET_LENGTH);
// Determine data length
if (type == uavConsts.TYPE_OBJ_REQ || type == uavConsts.TYPE_ACK)
{
length = 0;
}
else
{
length = obj.getNumBytes();
}
// Setup type and object id fields
bbuf.put((byte)(uavConsts.SYNC_VAL & 0xff));
bbuf.put((byte)(type & 0xff));
bbuf.putShort((UInt16)(length + 2 /* SYNC, Type */+ 2 /* Size */+ 4 /* ObjID */+ (obj
.isSingleInstance() ? 0 : 2)));
bbuf.putUint32((UInt32)obj.getObjID());
// Setup instance ID if one is required
if (!obj.isSingleInstance())
{
// Check if all instances are requested
if (allInstances)
bbuf.putShort((UInt16)(allInstId & 0xffff));
else
bbuf.putShort((UInt16)(obj.getInstID() & 0xffff));
}
// Check length
if (length >= uavConsts.MAX_PAYLOAD_LENGTH)
return false;
// Copy data (if any)
if (length > 0)
try
{
if (obj.pack(bbuf) == 0)
return false;
}
catch (Exception e)
{
// TODO Auto-generated catch block
Debug.Write(e.Message);
return false;
}
// Calculate checksum
bbuf.put((byte)(CRC.updateCRC(0, bbuf.array(), bbuf.position()) & 0xff));
int packlen = bbuf.position();
bbuf.position(0);
byte[] dst = new byte[packlen];
bbuf.get(dst, 0, packlen);
if (type == uavConsts.TYPE_OBJ_ACK || type == uavConsts.TYPE_OBJ_REQ)
{
// Once we send a UAVTalk packet that requires an ack or object let's set up
// the transaction here
setupTransaction(obj, allInstances, type);
}
ch.write(dst);
// Update stats
++txStats.Objects;
txStats.Bytes += bbuf.position();
txStats.ObjectBytes += length;
// Done
return true;
}