/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> private static JobSplit.SplitMetaInfo[] WriteNewSplits <T>(Configuration conf, T[] array, FSDataOutputStream @out) where T : InputSplit { JobSplit.SplitMetaInfo[] info = new JobSplit.SplitMetaInfo[array.Length]; if (array.Length != 0) { SerializationFactory factory = new SerializationFactory(conf); int i = 0; int maxBlockLocations = conf.GetInt(MRConfig.MaxBlockLocationsKey, MRConfig.MaxBlockLocationsDefault ); long offset = @out.GetPos(); foreach (T split in array) { long prevCount = @out.GetPos(); Text.WriteString(@out, split.GetType().FullName); Org.Apache.Hadoop.IO.Serializer.Serializer <T> serializer = factory.GetSerializer( (Type)split.GetType()); serializer.Open(@out); serializer.Serialize(split); long currCount = @out.GetPos(); string[] locations = split.GetLocations(); if (locations.Length > maxBlockLocations) { Log.Warn("Max block location exceeded for split: " + split + " splitsize: " + locations .Length + " maxsize: " + maxBlockLocations); locations = Arrays.CopyOf(locations, maxBlockLocations); } info[i++] = new JobSplit.SplitMetaInfo(locations, offset, split.GetLength()); offset += currCount - prevCount; } } return(info); }
/// <summary> /// This method is called to write the record that was most recently /// served (before a call to the mark). /// </summary> /// <remarks> /// This method is called to write the record that was most recently /// served (before a call to the mark). Since the framework reads one /// record in advance, to get this record, we serialize the current key /// and value /// </remarks> /// <param name="out"/> /// <exception cref="System.IO.IOException"/> private void WriteFirstKeyValueBytes(DataOutputStream @out) { System.Diagnostics.Debug.Assert((this._enclosing.GetCurrentKey() != null && this. _enclosing.GetCurrentValue() != null)); WritableUtils.WriteVInt(@out, this._enclosing.currentKeyLength); WritableUtils.WriteVInt(@out, this._enclosing.currentValueLength); Org.Apache.Hadoop.IO.Serializer.Serializer <KEYIN> keySerializer = this._enclosing .serializationFactory.GetSerializer(this._enclosing.keyClass); keySerializer.Open(@out); keySerializer.Serialize(this._enclosing.GetCurrentKey()); Org.Apache.Hadoop.IO.Serializer.Serializer <VALUEIN> valueSerializer = this._enclosing .serializationFactory.GetSerializer(this._enclosing.valueClass); valueSerializer.Open(@out); valueSerializer.Serialize(this._enclosing.GetCurrentValue()); }
/// <exception cref="System.IO.IOException"/> public KeyValueWriter(Configuration conf, OutputStream output, Type kyClass, Type valClass) { keyClass = kyClass; valueClass = valClass; dataBuffer = new DataOutputBuffer(); SerializationFactory serializationFactory = new SerializationFactory(conf); keySerializer = (Org.Apache.Hadoop.IO.Serializer.Serializer <K>)serializationFactory .GetSerializer(keyClass); keySerializer.Open(dataBuffer); valueSerializer = (Org.Apache.Hadoop.IO.Serializer.Serializer <V>)serializationFactory .GetSerializer(valueClass); valueSerializer.Open(dataBuffer); outputStream = new DataOutputStream(output); }
/// <summary>Make a copy of the writable object using serialization to a buffer</summary> /// <param name="src">the object to copy from</param> /// <param name="dst">the object to copy into, which is destroyed</param> /// <returns>dst param (the copy)</returns> /// <exception cref="System.IO.IOException"/> public static T Copy <T>(Configuration conf, T src, T dst) { ReflectionUtils.CopyInCopyOutBuffer buffer = cloneBuffers.Get(); buffer.outBuffer.Reset(); SerializationFactory factory = GetFactory(conf); Type cls = (Type)src.GetType(); Org.Apache.Hadoop.IO.Serializer.Serializer <T> serializer = factory.GetSerializer( cls); serializer.Open(buffer.outBuffer); serializer.Serialize(src); buffer.MoveData(); Deserializer <T> deserializer = factory.GetDeserializer(cls); deserializer.Open(buffer.inBuffer); dst = deserializer.Deserialize(dst); return(dst); }
public DefaultStringifier(Configuration conf, Type c) { SerializationFactory factory = new SerializationFactory(conf); this.serializer = factory.GetSerializer(c); this.deserializer = factory.GetDeserializer(c); this.inBuf = new DataInputBuffer(); this.outBuf = new DataOutputBuffer(); try { serializer.Open(outBuf); deserializer.Open(inBuf); } catch (IOException ex) { throw new RuntimeException(ex); } }
/// <exception cref="System.IO.IOException"/> private E MakeCopyForPassByValue <E>(Serialization <E> serialization, E obj) { Org.Apache.Hadoop.IO.Serializer.Serializer <E> ser = serialization.GetSerializer(GenericsUtil .GetClass(obj)); Deserializer <E> deser = serialization.GetDeserializer(GenericsUtil.GetClass(obj)); DataOutputBuffer dof = this._enclosing.threadLocalDataOutputBuffer.Get(); dof.Reset(); ser.Open(dof); ser.Serialize(obj); ser.Close(); obj = ReflectionUtils.NewInstance(GenericsUtil.GetClass(obj), this._enclosing.GetChainJobConf ()); ByteArrayInputStream bais = new ByteArrayInputStream(dof.GetData(), 0, dof.GetLength ()); deser.Open(bais); deser.Deserialize(obj); deser.Close(); return(obj); }
/// <exception cref="System.Exception"/> private K SerDeser <K>(K conf) { SerializationFactory factory = new SerializationFactory(Conf); Org.Apache.Hadoop.IO.Serializer.Serializer <K> serializer = factory.GetSerializer( GenericsUtil.GetClass(conf)); Deserializer <K> deserializer = factory.GetDeserializer(GenericsUtil.GetClass(conf )); DataOutputBuffer @out = new DataOutputBuffer(); serializer.Open(@out); serializer.Serialize(conf); serializer.Close(); DataInputBuffer @in = new DataInputBuffer(); @in.Reset(@out.GetData(), @out.GetLength()); deserializer.Open(@in); K after = deserializer.Deserialize(null); deserializer.Close(); return(after); }
/// <exception cref="System.IO.IOException"/> public Writer(Configuration conf, FSDataOutputStream @out, Type keyClass, Type valueClass , CompressionCodec codec, Counters.Counter writesCounter, bool ownOutputStream) { this.writtenRecordsCounter = writesCounter; this.checksumOut = new IFileOutputStream(@out); this.rawOut = @out; this.start = this.rawOut.GetPos(); if (codec != null) { this.compressor = CodecPool.GetCompressor(codec); if (this.compressor != null) { this.compressor.Reset(); this.compressedOut = codec.CreateOutputStream(checksumOut, compressor); this.@out = new FSDataOutputStream(this.compressedOut, null); this.compressOutput = true; } else { Log.Warn("Could not obtain compressor from CodecPool"); this.@out = new FSDataOutputStream(checksumOut, null); } } else { this.@out = new FSDataOutputStream(checksumOut, null); } this.keyClass = keyClass; this.valueClass = valueClass; if (keyClass != null) { SerializationFactory serializationFactory = new SerializationFactory(conf); this.keySerializer = serializationFactory.GetSerializer(keyClass); this.keySerializer.Open(buffer); this.valueSerializer = serializationFactory.GetSerializer(valueClass); this.valueSerializer.Open(buffer); } this.ownOutputStream = ownOutputStream; }