/// <exception cref="System.Exception"/>
public virtual void TestAcceptHandlingPrimitivesAndArrays()
{
SerializationFactory factory = new SerializationFactory(conf);
NUnit.Framework.Assert.IsNull(factory.GetSerializer <byte[]>());
NUnit.Framework.Assert.IsNull(factory.GetSerializer <byte>());
}
/// <exception cref="System.IO.IOException"/>
public KeyValueWriter(Configuration conf, OutputStream output, Type kyClass, Type
valClass)
{
keyClass = kyClass;
valueClass = valClass;
dataBuffer = new DataOutputBuffer();
SerializationFactory serializationFactory = new SerializationFactory(conf);
keySerializer = (Org.Apache.Hadoop.IO.Serializer.Serializer <K>)serializationFactory
.GetSerializer(keyClass);
keySerializer.Open(dataBuffer);
valueSerializer = (Org.Apache.Hadoop.IO.Serializer.Serializer <V>)serializationFactory
.GetSerializer(valueClass);
valueSerializer.Open(dataBuffer);
outputStream = new DataOutputStream(output);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
private static JobSplit.SplitMetaInfo[] WriteNewSplits <T>(Configuration conf, T[]
array, FSDataOutputStream @out)
where T : InputSplit
{
JobSplit.SplitMetaInfo[] info = new JobSplit.SplitMetaInfo[array.Length];
if (array.Length != 0)
{
SerializationFactory factory = new SerializationFactory(conf);
int i = 0;
int maxBlockLocations = conf.GetInt(MRConfig.MaxBlockLocationsKey, MRConfig.MaxBlockLocationsDefault
);
long offset = @out.GetPos();
foreach (T split in array)
{
long prevCount = @out.GetPos();
Text.WriteString(@out, split.GetType().FullName);
Org.Apache.Hadoop.IO.Serializer.Serializer <T> serializer = factory.GetSerializer(
(Type)split.GetType());
serializer.Open(@out);
serializer.Serialize(split);
long currCount = @out.GetPos();
string[] locations = split.GetLocations();
if (locations.Length > maxBlockLocations)
{
Log.Warn("Max block location exceeded for split: " + split + " splitsize: " + locations
.Length + " maxsize: " + maxBlockLocations);
locations = Arrays.CopyOf(locations, maxBlockLocations);
}
info[i++] = new JobSplit.SplitMetaInfo(locations, offset, split.GetLength());
offset += currCount - prevCount;
}
}
return(info);
}
/// <exception cref="System.IO.IOException"/>
public virtual void Write(DataOutput @out)
{
Text.WriteString(@out, inputSplitClass.FullName);
Text.WriteString(@out, inputFormatClass.FullName);
Text.WriteString(@out, mapperClass.FullName);
SerializationFactory factory = new SerializationFactory(conf);
Org.Apache.Hadoop.IO.Serializer.Serializer serializer = factory.GetSerializer(inputSplitClass
);
serializer.Open((DataOutputStream)@out);
serializer.Serialize(inputSplit);
}
/// <exception cref="System.IO.IOException"/>
public Writer(Configuration conf, FSDataOutputStream @out, Type keyClass, Type valueClass
, CompressionCodec codec, Counters.Counter writesCounter, bool ownOutputStream)
{
this.writtenRecordsCounter = writesCounter;
this.checksumOut = new IFileOutputStream(@out);
this.rawOut = @out;
this.start = this.rawOut.GetPos();
if (codec != null)
{
this.compressor = CodecPool.GetCompressor(codec);
if (this.compressor != null)
{
this.compressor.Reset();
this.compressedOut = codec.CreateOutputStream(checksumOut, compressor);
this.@out = new FSDataOutputStream(this.compressedOut, null);
this.compressOutput = true;
}
else
{
Log.Warn("Could not obtain compressor from CodecPool");
this.@out = new FSDataOutputStream(checksumOut, null);
}
}
else
{
this.@out = new FSDataOutputStream(checksumOut, null);
}
this.keyClass = keyClass;
this.valueClass = valueClass;
if (keyClass != null)
{
SerializationFactory serializationFactory = new SerializationFactory(conf);
this.keySerializer = serializationFactory.GetSerializer(keyClass);
this.keySerializer.Open(buffer);
this.valueSerializer = serializationFactory.GetSerializer(valueClass);
this.valueSerializer.Open(buffer);
}
this.ownOutputStream = ownOutputStream;
}
/// <summary>Write splits in the following format.</summary>
/// <remarks>
/// Write splits in the following format.
/// <c><count><class1><class2>...<classn><split1><split2>...<splitn>
/// </c>
/// </remarks>
/// <exception cref="System.IO.IOException"/>
public virtual void Write(DataOutput @out)
{
WritableUtils.WriteVInt(@out, splits.Length);
foreach (InputSplit s in splits)
{
Text.WriteString(@out, s.GetType().FullName);
}
foreach (InputSplit s_1 in splits)
{
SerializationFactory factory = new SerializationFactory(conf);
Org.Apache.Hadoop.IO.Serializer.Serializer serializer = factory.GetSerializer(s_1
.GetType());
serializer.Open((DataOutputStream)@out);
serializer.Serialize(s_1);
}
}
/// <summary>Make a copy of the writable object using serialization to a buffer</summary>
/// <param name="src">the object to copy from</param>
/// <param name="dst">the object to copy into, which is destroyed</param>
/// <returns>dst param (the copy)</returns>
/// <exception cref="System.IO.IOException"/>
public static T Copy <T>(Configuration conf, T src, T dst)
{
ReflectionUtils.CopyInCopyOutBuffer buffer = cloneBuffers.Get();
buffer.outBuffer.Reset();
SerializationFactory factory = GetFactory(conf);
Type cls = (Type)src.GetType();
Org.Apache.Hadoop.IO.Serializer.Serializer <T> serializer = factory.GetSerializer(
cls);
serializer.Open(buffer.outBuffer);
serializer.Serialize(src);
buffer.MoveData();
Deserializer <T> deserializer = factory.GetDeserializer(cls);
deserializer.Open(buffer.inBuffer);
dst = deserializer.Deserialize(dst);
return(dst);
}
public DefaultStringifier(Configuration conf, Type c)
{
SerializationFactory factory = new SerializationFactory(conf);
this.serializer = factory.GetSerializer(c);
this.deserializer = factory.GetDeserializer(c);
this.inBuf = new DataInputBuffer();
this.outBuf = new DataOutputBuffer();
try
{
serializer.Open(outBuf);
deserializer.Open(inBuf);
}
catch (IOException ex)
{
throw new RuntimeException(ex);
}
}
/// <exception cref="System.Exception"/>
private K SerDeser <K>(K conf)
{
SerializationFactory factory = new SerializationFactory(Conf);
Org.Apache.Hadoop.IO.Serializer.Serializer <K> serializer = factory.GetSerializer(
GenericsUtil.GetClass(conf));
Deserializer <K> deserializer = factory.GetDeserializer(GenericsUtil.GetClass(conf
));
DataOutputBuffer @out = new DataOutputBuffer();
serializer.Open(@out);
serializer.Serialize(conf);
serializer.Close();
DataInputBuffer @in = new DataInputBuffer();
@in.Reset(@out.GetData(), @out.GetLength());
deserializer.Open(@in);
K after = deserializer.Deserialize(null);
deserializer.Close();
return(after);
}