/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
private static JobSplit.SplitMetaInfo[] WriteNewSplits <T>(Configuration conf, T[]
array, FSDataOutputStream @out)
where T : InputSplit
{
JobSplit.SplitMetaInfo[] info = new JobSplit.SplitMetaInfo[array.Length];
if (array.Length != 0)
{
SerializationFactory factory = new SerializationFactory(conf);
int i = 0;
int maxBlockLocations = conf.GetInt(MRConfig.MaxBlockLocationsKey, MRConfig.MaxBlockLocationsDefault
);
long offset = @out.GetPos();
foreach (T split in array)
{
long prevCount = @out.GetPos();
Text.WriteString(@out, split.GetType().FullName);
Org.Apache.Hadoop.IO.Serializer.Serializer <T> serializer = factory.GetSerializer(
(Type)split.GetType());
serializer.Open(@out);
serializer.Serialize(split);
long currCount = @out.GetPos();
string[] locations = split.GetLocations();
if (locations.Length > maxBlockLocations)
{
Log.Warn("Max block location exceeded for split: " + split + " splitsize: " + locations
.Length + " maxsize: " + maxBlockLocations);
locations = Arrays.CopyOf(locations, maxBlockLocations);
}
info[i++] = new JobSplit.SplitMetaInfo(locations, offset, split.GetLength());
offset += currCount - prevCount;
}
}
return(info);
}
/// <exception cref="System.IO.IOException"/>
public virtual string ToString(T obj)
{
outBuf.Reset();
serializer.Serialize(obj);
byte[] buf = new byte[outBuf.GetLength()];
System.Array.Copy(outBuf.GetData(), 0, buf, 0, buf.Length);
return(new string(Base64.EncodeBase64(buf), Charsets.Utf8));
}
/// <summary>
/// This method is called to write the record that was most recently
/// served (before a call to the mark).
/// </summary>
/// <remarks>
/// This method is called to write the record that was most recently
/// served (before a call to the mark). Since the framework reads one
/// record in advance, to get this record, we serialize the current key
/// and value
/// </remarks>
/// <param name="out"/>
/// <exception cref="System.IO.IOException"/>
private void WriteFirstKeyValueBytes(DataOutputStream @out)
{
System.Diagnostics.Debug.Assert((this._enclosing.GetCurrentKey() != null && this.
_enclosing.GetCurrentValue() != null));
WritableUtils.WriteVInt(@out, this._enclosing.currentKeyLength);
WritableUtils.WriteVInt(@out, this._enclosing.currentValueLength);
Org.Apache.Hadoop.IO.Serializer.Serializer <KEYIN> keySerializer = this._enclosing
.serializationFactory.GetSerializer(this._enclosing.keyClass);
keySerializer.Open(@out);
keySerializer.Serialize(this._enclosing.GetCurrentKey());
Org.Apache.Hadoop.IO.Serializer.Serializer <VALUEIN> valueSerializer = this._enclosing
.serializationFactory.GetSerializer(this._enclosing.valueClass);
valueSerializer.Open(@out);
valueSerializer.Serialize(this._enclosing.GetCurrentValue());
}
/// <summary>Make a copy of the writable object using serialization to a buffer</summary>
/// <param name="src">the object to copy from</param>
/// <param name="dst">the object to copy into, which is destroyed</param>
/// <returns>dst param (the copy)</returns>
/// <exception cref="System.IO.IOException"/>
public static T Copy <T>(Configuration conf, T src, T dst)
{
ReflectionUtils.CopyInCopyOutBuffer buffer = cloneBuffers.Get();
buffer.outBuffer.Reset();
SerializationFactory factory = GetFactory(conf);
Type cls = (Type)src.GetType();
Org.Apache.Hadoop.IO.Serializer.Serializer <T> serializer = factory.GetSerializer(
cls);
serializer.Open(buffer.outBuffer);
serializer.Serialize(src);
buffer.MoveData();
Deserializer <T> deserializer = factory.GetDeserializer(cls);
deserializer.Open(buffer.inBuffer);
dst = deserializer.Deserialize(dst);
return(dst);
}
/// <exception cref="System.IO.IOException"/>
public virtual void Append(K key, V value)
{
if (key.GetType() != keyClass)
{
throw new IOException("wrong key class: " + key.GetType() + " is not " + keyClass
);
}
if (value.GetType() != valueClass)
{
throw new IOException("wrong value class: " + value.GetType() + " is not " + valueClass
);
}
// Append the 'key'
keySerializer.Serialize(key);
int keyLength = buffer.GetLength();
if (keyLength < 0)
{
throw new IOException("Negative key-length not allowed: " + keyLength + " for " +
key);
}
// Append the 'value'
valueSerializer.Serialize(value);
int valueLength = buffer.GetLength() - keyLength;
if (valueLength < 0)
{
throw new IOException("Negative value-length not allowed: " + valueLength + " for "
+ value);
}
// Write the record out
WritableUtils.WriteVInt(@out, keyLength);
// key length
WritableUtils.WriteVInt(@out, valueLength);
// value length
@out.Write(buffer.GetData(), 0, buffer.GetLength());
// data
// Reset
buffer.Reset();
// Update bytes written
decompressedBytesWritten += keyLength + valueLength + WritableUtils.GetVIntSize(keyLength
) + WritableUtils.GetVIntSize(valueLength);
++numRecordsWritten;
}
/// <exception cref="System.IO.IOException"/>
private E MakeCopyForPassByValue <E>(Serialization <E> serialization, E obj)
{
Org.Apache.Hadoop.IO.Serializer.Serializer <E> ser = serialization.GetSerializer(GenericsUtil
.GetClass(obj));
Deserializer <E> deser = serialization.GetDeserializer(GenericsUtil.GetClass(obj));
DataOutputBuffer dof = this._enclosing.threadLocalDataOutputBuffer.Get();
dof.Reset();
ser.Open(dof);
ser.Serialize(obj);
ser.Close();
obj = ReflectionUtils.NewInstance(GenericsUtil.GetClass(obj), this._enclosing.GetChainJobConf
());
ByteArrayInputStream bais = new ByteArrayInputStream(dof.GetData(), 0, dof.GetLength
());
deser.Open(bais);
deser.Deserialize(obj);
deser.Close();
return(obj);
}
/// <exception cref="System.Exception"/>
private K SerDeser <K>(K conf)
{
SerializationFactory factory = new SerializationFactory(Conf);
Org.Apache.Hadoop.IO.Serializer.Serializer <K> serializer = factory.GetSerializer(
GenericsUtil.GetClass(conf));
Deserializer <K> deserializer = factory.GetDeserializer(GenericsUtil.GetClass(conf
));
DataOutputBuffer @out = new DataOutputBuffer();
serializer.Open(@out);
serializer.Serialize(conf);
serializer.Close();
DataInputBuffer @in = new DataInputBuffer();
@in.Reset(@out.GetData(), @out.GetLength());
deserializer.Open(@in);
K after = deserializer.Deserialize(null);
deserializer.Close();
return(after);
}
/// <exception cref="System.IO.IOException"/>
public virtual void Write(K key, V value)
{
if (key.GetType() != keyClass)
{
throw new IOException("wrong key class: " + key.GetType() + " is not " + keyClass
);
}
if (value.GetType() != valueClass)
{
throw new IOException("wrong value class: " + value.GetType() + " is not " + valueClass
);
}
// Append the 'key'
keySerializer.Serialize(key);
int keyLength = dataBuffer.GetLength();
if (keyLength < 0)
{
throw new IOException("Negative key-length not allowed: " + keyLength + " for " +
key);
}
// Append the 'value'
valueSerializer.Serialize(value);
int valueLength = dataBuffer.GetLength() - keyLength;
if (valueLength < 0)
{
throw new IOException("Negative value-length not allowed: " + valueLength + " for "
+ value);
}
// Write the record out
WritableUtils.WriteVInt(outputStream, keyLength);
WritableUtils.WriteVInt(outputStream, valueLength);
outputStream.Write(dataBuffer.GetData(), 0, dataBuffer.GetLength());
// Reset
dataBuffer.Reset();
}
