/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
private static JobSplit.SplitMetaInfo[] WriteNewSplits <T>(Configuration conf, T[]
array, FSDataOutputStream @out)
where T : InputSplit
{
JobSplit.SplitMetaInfo[] info = new JobSplit.SplitMetaInfo[array.Length];
if (array.Length != 0)
{
SerializationFactory factory = new SerializationFactory(conf);
int i = 0;
int maxBlockLocations = conf.GetInt(MRConfig.MaxBlockLocationsKey, MRConfig.MaxBlockLocationsDefault
);
long offset = @out.GetPos();
foreach (T split in array)
{
long prevCount = @out.GetPos();
Text.WriteString(@out, split.GetType().FullName);
Org.Apache.Hadoop.IO.Serializer.Serializer <T> serializer = factory.GetSerializer(
(Type)split.GetType());
serializer.Open(@out);
serializer.Serialize(split);
long currCount = @out.GetPos();
string[] locations = split.GetLocations();
if (locations.Length > maxBlockLocations)
{
Log.Warn("Max block location exceeded for split: " + split + " splitsize: " + locations
.Length + " maxsize: " + maxBlockLocations);
locations = Arrays.CopyOf(locations, maxBlockLocations);
}
info[i++] = new JobSplit.SplitMetaInfo(locations, offset, split.GetLength());
offset += currCount - prevCount;
}
}
return(info);
}
/// <summary>
/// This method is called to write the record that was most recently
/// served (before a call to the mark).
/// </summary>
/// <remarks>
/// This method is called to write the record that was most recently
/// served (before a call to the mark). Since the framework reads one
/// record in advance, to get this record, we serialize the current key
/// and value
/// </remarks>
/// <param name="out"/>
/// <exception cref="System.IO.IOException"/>
private void WriteFirstKeyValueBytes(DataOutputStream @out)
{
System.Diagnostics.Debug.Assert((this._enclosing.GetCurrentKey() != null && this.
_enclosing.GetCurrentValue() != null));
WritableUtils.WriteVInt(@out, this._enclosing.currentKeyLength);
WritableUtils.WriteVInt(@out, this._enclosing.currentValueLength);
Org.Apache.Hadoop.IO.Serializer.Serializer <KEYIN> keySerializer = this._enclosing
.serializationFactory.GetSerializer(this._enclosing.keyClass);
keySerializer.Open(@out);
keySerializer.Serialize(this._enclosing.GetCurrentKey());
Org.Apache.Hadoop.IO.Serializer.Serializer <VALUEIN> valueSerializer = this._enclosing
.serializationFactory.GetSerializer(this._enclosing.valueClass);
valueSerializer.Open(@out);
valueSerializer.Serialize(this._enclosing.GetCurrentValue());
}
/// <exception cref="System.IO.IOException"/>
public KeyValueWriter(Configuration conf, OutputStream output, Type kyClass, Type
valClass)
{
keyClass = kyClass;
valueClass = valClass;
dataBuffer = new DataOutputBuffer();
SerializationFactory serializationFactory = new SerializationFactory(conf);
keySerializer = (Org.Apache.Hadoop.IO.Serializer.Serializer <K>)serializationFactory
.GetSerializer(keyClass);
keySerializer.Open(dataBuffer);
valueSerializer = (Org.Apache.Hadoop.IO.Serializer.Serializer <V>)serializationFactory
.GetSerializer(valueClass);
valueSerializer.Open(dataBuffer);
outputStream = new DataOutputStream(output);
}
/// <summary>Make a copy of the writable object using serialization to a buffer</summary>
/// <param name="src">the object to copy from</param>
/// <param name="dst">the object to copy into, which is destroyed</param>
/// <returns>dst param (the copy)</returns>
/// <exception cref="System.IO.IOException"/>
public static T Copy <T>(Configuration conf, T src, T dst)
{
ReflectionUtils.CopyInCopyOutBuffer buffer = cloneBuffers.Get();
buffer.outBuffer.Reset();
SerializationFactory factory = GetFactory(conf);
Type cls = (Type)src.GetType();
Org.Apache.Hadoop.IO.Serializer.Serializer <T> serializer = factory.GetSerializer(
cls);
serializer.Open(buffer.outBuffer);
serializer.Serialize(src);
buffer.MoveData();
Deserializer <T> deserializer = factory.GetDeserializer(cls);
deserializer.Open(buffer.inBuffer);
dst = deserializer.Deserialize(dst);
return(dst);
}
public DefaultStringifier(Configuration conf, Type c)
{
SerializationFactory factory = new SerializationFactory(conf);
this.serializer = factory.GetSerializer(c);
this.deserializer = factory.GetDeserializer(c);
this.inBuf = new DataInputBuffer();
this.outBuf = new DataOutputBuffer();
try
{
serializer.Open(outBuf);
deserializer.Open(inBuf);
}
catch (IOException ex)
{
throw new RuntimeException(ex);
}
}
/// <exception cref="System.IO.IOException"/>
private E MakeCopyForPassByValue <E>(Serialization <E> serialization, E obj)
{
Org.Apache.Hadoop.IO.Serializer.Serializer <E> ser = serialization.GetSerializer(GenericsUtil
.GetClass(obj));
Deserializer <E> deser = serialization.GetDeserializer(GenericsUtil.GetClass(obj));
DataOutputBuffer dof = this._enclosing.threadLocalDataOutputBuffer.Get();
dof.Reset();
ser.Open(dof);
ser.Serialize(obj);
ser.Close();
obj = ReflectionUtils.NewInstance(GenericsUtil.GetClass(obj), this._enclosing.GetChainJobConf
());
ByteArrayInputStream bais = new ByteArrayInputStream(dof.GetData(), 0, dof.GetLength
());
deser.Open(bais);
deser.Deserialize(obj);
deser.Close();
return(obj);
}
/// <exception cref="System.Exception"/>
private K SerDeser <K>(K conf)
{
SerializationFactory factory = new SerializationFactory(Conf);
Org.Apache.Hadoop.IO.Serializer.Serializer <K> serializer = factory.GetSerializer(
GenericsUtil.GetClass(conf));
Deserializer <K> deserializer = factory.GetDeserializer(GenericsUtil.GetClass(conf
));
DataOutputBuffer @out = new DataOutputBuffer();
serializer.Open(@out);
serializer.Serialize(conf);
serializer.Close();
DataInputBuffer @in = new DataInputBuffer();
@in.Reset(@out.GetData(), @out.GetLength());
deserializer.Open(@in);
K after = deserializer.Deserialize(null);
deserializer.Close();
return(after);
}
/// <exception cref="System.IO.IOException"/>
public Writer(Configuration conf, FSDataOutputStream @out, Type keyClass, Type valueClass
, CompressionCodec codec, Counters.Counter writesCounter, bool ownOutputStream)
{
this.writtenRecordsCounter = writesCounter;
this.checksumOut = new IFileOutputStream(@out);
this.rawOut = @out;
this.start = this.rawOut.GetPos();
if (codec != null)
{
this.compressor = CodecPool.GetCompressor(codec);
if (this.compressor != null)
{
this.compressor.Reset();
this.compressedOut = codec.CreateOutputStream(checksumOut, compressor);
this.@out = new FSDataOutputStream(this.compressedOut, null);
this.compressOutput = true;
}
else
{
Log.Warn("Could not obtain compressor from CodecPool");
this.@out = new FSDataOutputStream(checksumOut, null);
}
}
else
{
this.@out = new FSDataOutputStream(checksumOut, null);
}
this.keyClass = keyClass;
this.valueClass = valueClass;
if (keyClass != null)
{
SerializationFactory serializationFactory = new SerializationFactory(conf);
this.keySerializer = serializationFactory.GetSerializer(keyClass);
this.keySerializer.Open(buffer);
this.valueSerializer = serializationFactory.GetSerializer(valueClass);
this.valueSerializer.Open(buffer);
}
this.ownOutputStream = ownOutputStream;
}
