/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> private static JobSplit.SplitMetaInfo[] WriteNewSplits <T>(Configuration conf, T[] array, FSDataOutputStream @out) where T : InputSplit { JobSplit.SplitMetaInfo[] info = new JobSplit.SplitMetaInfo[array.Length]; if (array.Length != 0) { SerializationFactory factory = new SerializationFactory(conf); int i = 0; int maxBlockLocations = conf.GetInt(MRConfig.MaxBlockLocationsKey, MRConfig.MaxBlockLocationsDefault ); long offset = @out.GetPos(); foreach (T split in array) { long prevCount = @out.GetPos(); Text.WriteString(@out, split.GetType().FullName); Org.Apache.Hadoop.IO.Serializer.Serializer <T> serializer = factory.GetSerializer( (Type)split.GetType()); serializer.Open(@out); serializer.Serialize(split); long currCount = @out.GetPos(); string[] locations = split.GetLocations(); if (locations.Length > maxBlockLocations) { Log.Warn("Max block location exceeded for split: " + split + " splitsize: " + locations .Length + " maxsize: " + maxBlockLocations); locations = Arrays.CopyOf(locations, maxBlockLocations); } info[i++] = new JobSplit.SplitMetaInfo(locations, offset, split.GetLength()); offset += currCount - prevCount; } } return(info); }
/// <exception cref="System.IO.IOException"/> public virtual string ToString(T obj) { outBuf.Reset(); serializer.Serialize(obj); byte[] buf = new byte[outBuf.GetLength()]; System.Array.Copy(outBuf.GetData(), 0, buf, 0, buf.Length); return(new string(Base64.EncodeBase64(buf), Charsets.Utf8)); }
/// <summary> /// This method is called to write the record that was most recently /// served (before a call to the mark). /// </summary> /// <remarks> /// This method is called to write the record that was most recently /// served (before a call to the mark). Since the framework reads one /// record in advance, to get this record, we serialize the current key /// and value /// </remarks> /// <param name="out"/> /// <exception cref="System.IO.IOException"/> private void WriteFirstKeyValueBytes(DataOutputStream @out) { System.Diagnostics.Debug.Assert((this._enclosing.GetCurrentKey() != null && this. _enclosing.GetCurrentValue() != null)); WritableUtils.WriteVInt(@out, this._enclosing.currentKeyLength); WritableUtils.WriteVInt(@out, this._enclosing.currentValueLength); Org.Apache.Hadoop.IO.Serializer.Serializer <KEYIN> keySerializer = this._enclosing .serializationFactory.GetSerializer(this._enclosing.keyClass); keySerializer.Open(@out); keySerializer.Serialize(this._enclosing.GetCurrentKey()); Org.Apache.Hadoop.IO.Serializer.Serializer <VALUEIN> valueSerializer = this._enclosing .serializationFactory.GetSerializer(this._enclosing.valueClass); valueSerializer.Open(@out); valueSerializer.Serialize(this._enclosing.GetCurrentValue()); }
/// <summary>Make a copy of the writable object using serialization to a buffer</summary> /// <param name="src">the object to copy from</param> /// <param name="dst">the object to copy into, which is destroyed</param> /// <returns>dst param (the copy)</returns> /// <exception cref="System.IO.IOException"/> public static T Copy <T>(Configuration conf, T src, T dst) { ReflectionUtils.CopyInCopyOutBuffer buffer = cloneBuffers.Get(); buffer.outBuffer.Reset(); SerializationFactory factory = GetFactory(conf); Type cls = (Type)src.GetType(); Org.Apache.Hadoop.IO.Serializer.Serializer <T> serializer = factory.GetSerializer( cls); serializer.Open(buffer.outBuffer); serializer.Serialize(src); buffer.MoveData(); Deserializer <T> deserializer = factory.GetDeserializer(cls); deserializer.Open(buffer.inBuffer); dst = deserializer.Deserialize(dst); return(dst); }
/// <exception cref="System.IO.IOException"/> public virtual void Append(K key, V value) { if (key.GetType() != keyClass) { throw new IOException("wrong key class: " + key.GetType() + " is not " + keyClass ); } if (value.GetType() != valueClass) { throw new IOException("wrong value class: " + value.GetType() + " is not " + valueClass ); } // Append the 'key' keySerializer.Serialize(key); int keyLength = buffer.GetLength(); if (keyLength < 0) { throw new IOException("Negative key-length not allowed: " + keyLength + " for " + key); } // Append the 'value' valueSerializer.Serialize(value); int valueLength = buffer.GetLength() - keyLength; if (valueLength < 0) { throw new IOException("Negative value-length not allowed: " + valueLength + " for " + value); } // Write the record out WritableUtils.WriteVInt(@out, keyLength); // key length WritableUtils.WriteVInt(@out, valueLength); // value length @out.Write(buffer.GetData(), 0, buffer.GetLength()); // data // Reset buffer.Reset(); // Update bytes written decompressedBytesWritten += keyLength + valueLength + WritableUtils.GetVIntSize(keyLength ) + WritableUtils.GetVIntSize(valueLength); ++numRecordsWritten; }
/// <exception cref="System.IO.IOException"/> private E MakeCopyForPassByValue <E>(Serialization <E> serialization, E obj) { Org.Apache.Hadoop.IO.Serializer.Serializer <E> ser = serialization.GetSerializer(GenericsUtil .GetClass(obj)); Deserializer <E> deser = serialization.GetDeserializer(GenericsUtil.GetClass(obj)); DataOutputBuffer dof = this._enclosing.threadLocalDataOutputBuffer.Get(); dof.Reset(); ser.Open(dof); ser.Serialize(obj); ser.Close(); obj = ReflectionUtils.NewInstance(GenericsUtil.GetClass(obj), this._enclosing.GetChainJobConf ()); ByteArrayInputStream bais = new ByteArrayInputStream(dof.GetData(), 0, dof.GetLength ()); deser.Open(bais); deser.Deserialize(obj); deser.Close(); return(obj); }
/// <exception cref="System.Exception"/> private K SerDeser <K>(K conf) { SerializationFactory factory = new SerializationFactory(Conf); Org.Apache.Hadoop.IO.Serializer.Serializer <K> serializer = factory.GetSerializer( GenericsUtil.GetClass(conf)); Deserializer <K> deserializer = factory.GetDeserializer(GenericsUtil.GetClass(conf )); DataOutputBuffer @out = new DataOutputBuffer(); serializer.Open(@out); serializer.Serialize(conf); serializer.Close(); DataInputBuffer @in = new DataInputBuffer(); @in.Reset(@out.GetData(), @out.GetLength()); deserializer.Open(@in); K after = deserializer.Deserialize(null); deserializer.Close(); return(after); }
/// <exception cref="System.IO.IOException"/> public virtual void Write(K key, V value) { if (key.GetType() != keyClass) { throw new IOException("wrong key class: " + key.GetType() + " is not " + keyClass ); } if (value.GetType() != valueClass) { throw new IOException("wrong value class: " + value.GetType() + " is not " + valueClass ); } // Append the 'key' keySerializer.Serialize(key); int keyLength = dataBuffer.GetLength(); if (keyLength < 0) { throw new IOException("Negative key-length not allowed: " + keyLength + " for " + key); } // Append the 'value' valueSerializer.Serialize(value); int valueLength = dataBuffer.GetLength() - keyLength; if (valueLength < 0) { throw new IOException("Negative value-length not allowed: " + valueLength + " for " + value); } // Write the record out WritableUtils.WriteVInt(outputStream, keyLength); WritableUtils.WriteVInt(outputStream, valueLength); outputStream.Write(dataBuffer.GetData(), 0, dataBuffer.GetLength()); // Reset dataBuffer.Reset(); }