/// <exception cref="System.IO.IOException"/>
protected internal virtual void ReadObject(Writable obj, DataInputStream inStream
)
{
int numBytes = WritableUtils.ReadVInt(inStream);
byte[] buffer;
// For BytesWritable and Text, use the specified length to set the length
// this causes the "obvious" translations to work. So that if you emit
// a string "abc" from C++, it shows up as "abc".
if (obj is BytesWritable)
{
buffer = new byte[numBytes];
inStream.ReadFully(buffer);
((BytesWritable)obj).Set(buffer, 0, numBytes);
}
else
{
if (obj is Text)
{
buffer = new byte[numBytes];
inStream.ReadFully(buffer);
((Text)obj).Set(buffer);
}
else
{
obj.ReadFields(inStream);
}
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void ReadFields(DataInput @in)
{
this.taskid.ReadFields(@in);
this.progress = @in.ReadFloat();
this.state = StringInterner.WeakIntern(Text.ReadString(@in));
this.startTime = @in.ReadLong();
this.finishTime = @in.ReadLong();
diagnostics = WritableUtils.ReadStringArray(@in);
counters = new Counters();
counters.ReadFields(@in);
currentStatus = WritableUtils.ReadEnum <TIPStatus>(@in);
if (currentStatus == TIPStatus.Running)
{
int num = WritableUtils.ReadVInt(@in);
for (int i = 0; i < num; i++)
{
TaskAttemptID t = new TaskAttemptID();
t.ReadFields(@in);
runningAttempts.AddItem(t);
}
}
else
{
if (currentStatus == TIPStatus.Complete)
{
successfulAttempt.ReadFields(@in);
}
}
}
/// <summary>Loads all the keys</summary>
/// <param name="in"/>
/// <exception cref="System.IO.IOException"/>
public virtual void ReadFields(BinaryReader reader)
{
secretKeysMap.Clear();
tokenMap.Clear();
int size = WritableUtils.ReadVInt(@in);
for (int i = 0; i < size; i++)
{
Text alias = new Text();
alias.ReadFields(@in);
Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> t = new Org.Apache.Hadoop.Security.Token.Token
<TokenIdentifier>();
t.ReadFields(@in);
tokenMap[alias] = t;
}
size = WritableUtils.ReadVInt(@in);
for (int i_1 = 0; i_1 < size; i_1++)
{
Text alias = new Text();
alias.ReadFields(@in);
int len = WritableUtils.ReadVInt(@in);
byte[] value = new byte[len];
@in.ReadFully(value);
secretKeysMap[alias] = value;
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void ReadFields(DataInput @in)
{
mapId = WritableUtils.ReadStringSafely(@in, MaxIdLength);
compressedLength = WritableUtils.ReadVLong(@in);
uncompressedLength = WritableUtils.ReadVLong(@in);
forReduce = WritableUtils.ReadVInt(@in);
}
/// <exception cref="System.IO.IOException"/>
protected internal virtual bool PositionToNextRecord(DataInput dIn)
{
// Sanity check
if (eof)
{
throw new EOFException("Completed reading " + bytesRead);
}
// Read key and value lengths
currentKeyLength = WritableUtils.ReadVInt(dIn);
currentValueLength = WritableUtils.ReadVInt(dIn);
bytesRead += WritableUtils.GetVIntSize(currentKeyLength) + WritableUtils.GetVIntSize
(currentValueLength);
// Check for EOF
if (currentKeyLength == EofMarker && currentValueLength == EofMarker)
{
eof = true;
return(false);
}
// Sanity check
if (currentKeyLength < 0)
{
throw new IOException("Rec# " + recNo + ": Negative key-length: " + currentKeyLength
);
}
if (currentValueLength < 0)
{
throw new IOException("Rec# " + recNo + ": Negative value-length: " + currentValueLength
);
}
return(true);
}
/// <summary><inheritDoc/></summary>
/// <exception cref="System.IO.IOException">
/// If the child InputSplit cannot be read, typically
/// for faliing access checks.
/// </exception>
public virtual void ReadFields(DataInput @in)
{
// Generic array assignment
int card = WritableUtils.ReadVInt(@in);
if (splits == null || splits.Length != card)
{
splits = new InputSplit[card];
}
Type[] cls = new Type[card];
try
{
for (int i = 0; i < card; ++i)
{
cls[i] = Sharpen.Runtime.GetType(Text.ReadString(@in)).AsSubclass <InputSplit>();
}
for (int i_1 = 0; i_1 < card; ++i_1)
{
splits[i_1] = ReflectionUtils.NewInstance(cls[i_1], null);
splits[i_1].ReadFields(@in);
}
}
catch (TypeLoadException e)
{
throw (IOException)Sharpen.Extensions.InitCause(new IOException("Failed split init"
), e);
}
}
/// <summary><inheritDoc/></summary>
/// <exception cref="System.IO.IOException">
/// If the child InputSplit cannot be read, typically
/// for failing access checks.
/// </exception>
public virtual void ReadFields(DataInput @in)
{
// Generic array assignment
int card = WritableUtils.ReadVInt(@in);
if (splits == null || splits.Length != card)
{
splits = new InputSplit[card];
}
Type[] cls = new Type[card];
try
{
for (int i = 0; i < card; ++i)
{
cls[i] = Sharpen.Runtime.GetType(Text.ReadString(@in)).AsSubclass <InputSplit>();
}
for (int i_1 = 0; i_1 < card; ++i_1)
{
splits[i_1] = ReflectionUtils.NewInstance(cls[i_1], null);
SerializationFactory factory = new SerializationFactory(conf);
Deserializer deserializer = factory.GetDeserializer(cls[i_1]);
deserializer.Open((DataInputStream)@in);
splits[i_1] = (InputSplit)deserializer.Deserialize(splits[i_1]);
}
}
catch (TypeLoadException e)
{
throw new IOException("Failed split init", e);
}
}
public virtual void BinaryProtocolStub()
{
try
{
InitSoket();
// output code
WritableUtils.WriteVInt(dataOut, 50);
IntWritable wt = new IntWritable();
wt.Set(123);
WriteObject(wt, dataOut);
WriteObject(new Text("value"), dataOut);
// PARTITIONED_OUTPUT
WritableUtils.WriteVInt(dataOut, 51);
WritableUtils.WriteVInt(dataOut, 0);
WriteObject(wt, dataOut);
WriteObject(new Text("value"), dataOut);
// STATUS
WritableUtils.WriteVInt(dataOut, 52);
Text.WriteString(dataOut, "PROGRESS");
dataOut.Flush();
// progress
WritableUtils.WriteVInt(dataOut, 53);
dataOut.WriteFloat(0.55f);
// register counter
WritableUtils.WriteVInt(dataOut, 55);
// id
WritableUtils.WriteVInt(dataOut, 0);
Text.WriteString(dataOut, "group");
Text.WriteString(dataOut, "name");
// increment counter
WritableUtils.WriteVInt(dataOut, 56);
WritableUtils.WriteVInt(dataOut, 0);
WritableUtils.WriteVLong(dataOut, 2);
// map item
int intValue = WritableUtils.ReadVInt(dataInput);
System.Console.Out.WriteLine("intValue:" + intValue);
IntWritable iw = new IntWritable();
ReadObject(iw, dataInput);
System.Console.Out.WriteLine("key:" + iw.Get());
Text txt = new Text();
ReadObject(txt, dataInput);
System.Console.Out.WriteLine("value:" + txt.ToString());
// done
// end of session
WritableUtils.WriteVInt(dataOut, 54);
System.Console.Out.WriteLine("finish");
dataOut.Flush();
dataOut.Close();
}
catch (Exception x)
{
Sharpen.Runtime.PrintStackTrace(x);
}
finally
{
CloseSoket();
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void ReadFields(DataInput @in)
{
taskId.ReadFields(@in);
idWithinJob = WritableUtils.ReadVInt(@in);
isMap = @in.ReadBoolean();
status = WritableUtils.ReadEnum <TaskCompletionEvent.Status>(@in);
taskTrackerHttp = WritableUtils.ReadString(@in);
taskRunTime = WritableUtils.ReadVInt(@in);
eventId = WritableUtils.ReadVInt(@in);
}
/// <exception cref="System.IO.IOException"/>
public virtual void ReadFields(DataInput @in)
{
int len = WritableUtils.ReadVInt(@in);
locations = new string[len];
for (int i = 0; i < locations.Length; i++)
{
locations[i] = Text.ReadString(@in);
}
startOffset = WritableUtils.ReadVLong(@in);
inputDataLength = WritableUtils.ReadVLong(@in);
}
/// <exception cref="System.IO.IOException"/>
public virtual void ReadFields(DataInput @in)
{
lock (this)
{
int version = WritableUtils.ReadVInt(@in);
if (version != groupFactory.Version())
{
throw new IOException("Counters version mismatch, expected " + groupFactory.Version
() + " got " + version);
}
int numFGroups = WritableUtils.ReadVInt(@in);
fgroups.Clear();
AbstractCounters.GroupType[] groupTypes = AbstractCounters.GroupType.Values();
while (numFGroups-- > 0)
{
AbstractCounters.GroupType groupType = groupTypes[WritableUtils.ReadVInt(@in)];
G group;
switch (groupType)
{
case AbstractCounters.GroupType.Filesystem:
{
// with nothing
group = groupFactory.NewFileSystemGroup();
break;
}
case AbstractCounters.GroupType.Framework:
{
// with group id
group = groupFactory.NewFrameworkGroup(WritableUtils.ReadVInt(@in));
break;
}
default:
{
// Silence dumb compiler, as it would've thrown earlier
throw new IOException("Unexpected counter group type: " + groupType);
}
}
group.ReadFields(@in);
fgroups[group.GetName()] = group;
}
int numGroups = WritableUtils.ReadVInt(@in);
while (numGroups-- > 0)
{
limits.CheckGroups(groups.Count + 1);
G group = groupFactory.NewGenericGroup(StringInterner.WeakIntern(Text.ReadString(
@in)), null, limits);
group.ReadFields(@in);
groups[group.GetName()] = group;
}
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void ReadFields(DataInput @in)
{
if (disableRead)
{
return;
}
len = WritableUtils.ReadVInt(@in);
for (int i = 0; i < len; ++i)
{
NUnit.Framework.Assert.AreEqual("Invalid byte at " + i, fillChar, @in.ReadByte());
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void ReadFields(DataInput @in)
{
Clear();
int len = WritableUtils.ReadVInt(@in);
T[] enums = enumClass.GetEnumConstants();
for (int i = 0; i < len; ++i)
{
int ord = WritableUtils.ReadVInt(@in);
Counter counter = NewCounter(enums[ord]);
counter.SetValue(WritableUtils.ReadVLong(@in));
counters[ord] = counter;
}
}
/// <exception cref="System.IO.IOException"/>
public override void ReadFields(DataInput @in)
{
this.cache = null;
expiryDate = WritableUtils.ReadVLong(@in);
keyId = WritableUtils.ReadVInt(@in);
userId = WritableUtils.ReadString(@in);
blockPoolId = WritableUtils.ReadString(@in);
blockId = WritableUtils.ReadVLong(@in);
int length = WritableUtils.ReadVIntInRange(@in, 0, typeof(BlockTokenSecretManager.AccessMode
).GetEnumConstants().Length);
for (int i = 0; i < length; i++)
{
modes.AddItem(WritableUtils.ReadEnum <BlockTokenSecretManager.AccessMode>(@in));
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void ReadFields(DataInput @in)
{
lock (this)
{
displayName = StringInterner.WeakIntern(Text.ReadString(@in));
counters.Clear();
int size = WritableUtils.ReadVInt(@in);
for (int i = 0; i < size; i++)
{
T counter = NewCounter();
counter.ReadFields(@in);
counters[counter.GetName()] = counter;
limits.IncrCounters();
}
}
}
/// <exception cref="System.IO.IOException"/>
public override void ReadFields(BinaryReader reader)
{
byte version = @in.ReadByte();
if (version != Version)
{
throw new IOException("Unknown version of delegation token " + version);
}
owner.ReadFields(@in, Text.DefaultMaxLen);
renewer.ReadFields(@in, Text.DefaultMaxLen);
realUser.ReadFields(@in, Text.DefaultMaxLen);
issueDate = WritableUtils.ReadVLong(@in);
maxDate = WritableUtils.ReadVLong(@in);
sequenceNumber = WritableUtils.ReadVInt(@in);
masterKeyId = WritableUtils.ReadVInt(@in);
}
/// <exception cref="System.IO.IOException"/>
public virtual void ReadFields(BinaryReader reader)
{
keyId = WritableUtils.ReadVInt(@in);
expiryDate = WritableUtils.ReadVLong(@in);
int len = WritableUtils.ReadVIntInRange(@in, -1, MaxKeyLen);
if (len == -1)
{
keyBytes = null;
}
else
{
keyBytes = new byte[len];
@in.ReadFully(keyBytes);
}
}
/// <exception cref="System.IO.IOException"/>
public static JobSplit.TaskSplitMetaInfo[] ReadSplitMetaInfo(JobID jobId, FileSystem
fs, Configuration conf, Path jobSubmitDir)
{
long maxMetaInfoSize = conf.GetLong(MRJobConfig.SplitMetainfoMaxsize, MRJobConfig
.DefaultSplitMetainfoMaxsize);
Path metaSplitFile = JobSubmissionFiles.GetJobSplitMetaFile(jobSubmitDir);
string jobSplitFile = JobSubmissionFiles.GetJobSplitFile(jobSubmitDir).ToString();
FileStatus fStatus = fs.GetFileStatus(metaSplitFile);
if (maxMetaInfoSize > 0 && fStatus.GetLen() > maxMetaInfoSize)
{
throw new IOException("Split metadata size exceeded " + maxMetaInfoSize + ". Aborting job "
+ jobId);
}
FSDataInputStream @in = fs.Open(metaSplitFile);
byte[] header = new byte[JobSplit.MetaSplitFileHeader.Length];
@in.ReadFully(header);
if (!Arrays.Equals(JobSplit.MetaSplitFileHeader, header))
{
throw new IOException("Invalid header on split file");
}
int vers = WritableUtils.ReadVInt(@in);
if (vers != JobSplit.MetaSplitVersion)
{
@in.Close();
throw new IOException("Unsupported split version " + vers);
}
int numSplits = WritableUtils.ReadVInt(@in);
//TODO: check for insane values
JobSplit.TaskSplitMetaInfo[] allSplitMetaInfo = new JobSplit.TaskSplitMetaInfo[numSplits
];
for (int i = 0; i < numSplits; i++)
{
JobSplit.SplitMetaInfo splitMetaInfo = new JobSplit.SplitMetaInfo();
splitMetaInfo.ReadFields(@in);
JobSplit.TaskSplitIndex splitIndex = new JobSplit.TaskSplitIndex(jobSplitFile, splitMetaInfo
.GetStartOffset());
allSplitMetaInfo[i] = new JobSplit.TaskSplitMetaInfo(splitIndex, splitMetaInfo.GetLocations
(), splitMetaInfo.GetInputDataLength());
}
@in.Close();
return(allSplitMetaInfo);
}
/// <exception cref="System.IO.IOException"/>
public virtual void ReadFields(BinaryReader reader)
{
int len = WritableUtils.ReadVInt(@in);
if (identifier == null || identifier.Length != len)
{
identifier = new byte[len];
}
@in.ReadFully(identifier);
len = WritableUtils.ReadVInt(@in);
if (password == null || password.Length != len)
{
password = new byte[len];
}
@in.ReadFully(password);
kind.ReadFields(@in);
service.ReadFields(@in);
}
/// <summary><inheritDoc/></summary>
/// <exception cref="System.IO.IOException"/>
public virtual void ReadFields(DataInput @in)
{
// No static typeinfo on Tuples
int card = WritableUtils.ReadVInt(@in);
values = new Writable[card];
ReadBitSet(@in, card, written);
Type[] cls = new Type[card];
try
{
for (int i = 0; i < card; ++i)
{
cls[i] = Sharpen.Runtime.GetType(Org.Apache.Hadoop.IO.Text.ReadString(@in)).AsSubclass
<Writable>();
}
for (int i_1 = 0; i_1 < card; ++i_1)
{
if (cls[i_1].Equals(typeof(NullWritable)))
{
values[i_1] = NullWritable.Get();
}
else
{
values[i_1] = System.Activator.CreateInstance(cls[i_1]);
}
if (Has(i_1))
{
values[i_1].ReadFields(@in);
}
}
}
catch (TypeLoadException e)
{
throw new IOException("Failed tuple init", e);
}
catch (MemberAccessException e)
{
throw new IOException("Failed tuple init", e);
}
catch (InstantiationException e)
{
throw new IOException("Failed tuple init", e);
}
}
public virtual void BinaryProtocolStub()
{
try
{
InitSoket();
System.Console.Out.WriteLine("start OK");
// RUN_MAP.code
// should be 3
int answer = WritableUtils.ReadVInt(dataInput);
System.Console.Out.WriteLine("RunMap:" + answer);
TestPipeApplication.FakeSplit split = new TestPipeApplication.FakeSplit();
ReadObject(split, dataInput);
WritableUtils.ReadVInt(dataInput);
WritableUtils.ReadVInt(dataInput);
// end runMap
// get InputTypes
WritableUtils.ReadVInt(dataInput);
string inText = Text.ReadString(dataInput);
System.Console.Out.WriteLine("Key class:" + inText);
inText = Text.ReadString(dataInput);
System.Console.Out.WriteLine("Value class:" + inText);
int inCode = 0;
// read all data from sender and write to output
while ((inCode = WritableUtils.ReadVInt(dataInput)) == 4)
{
FloatWritable key = new FloatWritable();
NullWritable value = NullWritable.Get();
ReadObject(key, dataInput);
System.Console.Out.WriteLine("value:" + key.Get());
ReadObject(value, dataInput);
}
WritableUtils.WriteVInt(dataOut, 54);
dataOut.Flush();
dataOut.Close();
}
catch (Exception x)
{
Sharpen.Runtime.PrintStackTrace(x);
}
finally
{
CloseSoket();
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void ReadFields(BinaryReader reader, int maxLength)
{
int newLength = WritableUtils.ReadVInt(@in);
if (newLength < 0)
{
throw new IOException("tried to deserialize " + newLength + " bytes of data! newLength must be non-negative."
);
}
else
{
if (newLength >= maxLength)
{
throw new IOException("tried to deserialize " + newLength + " bytes of data, but maxLength = "
+ maxLength);
}
}
ReadWithKnownLength(@in, newLength);
}
/// <exception cref="System.IO.IOException"/>
public virtual void ReadFields(DataInput @in)
{
int numSchemes = WritableUtils.ReadVInt(@in);
// #scheme
FileSystemCounter[] enums = FileSystemCounter.Values();
for (int i = 0; i < numSchemes; ++i)
{
string scheme = WritableUtils.ReadString(@in);
// scheme
int numCounters = WritableUtils.ReadVInt(@in);
// #counter
for (int j = 0; j < numCounters; ++j)
{
FindCounter(scheme, enums[WritableUtils.ReadVInt(@in)]).SetValue(WritableUtils.ReadVLong
(@in));
}
}
}
public virtual void BinaryProtocolStub()
{
try
{
InitSoket();
//should be 5
//RUN_REDUCE boolean
WritableUtils.ReadVInt(dataInput);
WritableUtils.ReadVInt(dataInput);
int intValue = WritableUtils.ReadVInt(dataInput);
System.Console.Out.WriteLine("getIsJavaRecordWriter:" + intValue);
// reduce key
WritableUtils.ReadVInt(dataInput);
// value of reduce key
BooleanWritable value = new BooleanWritable();
ReadObject(value, dataInput);
System.Console.Out.WriteLine("reducer key :" + value);
// reduce value code:
// reduce values
while ((intValue = WritableUtils.ReadVInt(dataInput)) == 7)
{
Text txt = new Text();
// value
ReadObject(txt, dataInput);
System.Console.Out.WriteLine("reduce value :" + txt);
}
// done
WritableUtils.WriteVInt(dataOut, 54);
dataOut.Flush();
dataOut.Close();
}
catch (Exception x)
{
Sharpen.Runtime.PrintStackTrace(x);
}
finally
{
CloseSoket();
}
}
/// <exception cref="System.Exception"/>
protected internal virtual void InitSoket()
{
int port = System.Convert.ToInt32(Runtime.Getenv("mapreduce.pipes.command.port"));
IPAddress address = Sharpen.Runtime.GetLocalHost();
socket = Sharpen.Extensions.CreateSocket(address.GetHostName(), port);
InputStream input = socket.GetInputStream();
OutputStream output = socket.GetOutputStream();
// try to read
dataInput = new DataInputStream(input);
WritableUtils.ReadVInt(dataInput);
string str = Text.ReadString(dataInput);
Text.ReadString(dataInput);
dataOut = new DataOutputStream(output);
WritableUtils.WriteVInt(dataOut, 57);
string s = CreateDigest(Sharpen.Runtime.GetBytesForString("password"), str);
Text.WriteString(dataOut, s);
// start
WritableUtils.ReadVInt(dataInput);
int cuttentAnswer = WritableUtils.ReadVInt(dataInput);
System.Console.Out.WriteLine("CURRENT_PROTOCOL_VERSION:" + cuttentAnswer);
// get configuration
// should be MessageType.SET_JOB_CONF.code
WritableUtils.ReadVInt(dataInput);
// array length
int j = WritableUtils.ReadVInt(dataInput);
for (int i = 0; i < j; i++)
{
Text.ReadString(dataInput);
i++;
Text.ReadString(dataInput);
}
}
/// <exception cref="System.IO.IOException"/>
public static Block[] ReadCompactBlockArray(DataInput @in, int logVersion)
{
int num = WritableUtils.ReadVInt(@in);
if (num < 0)
{
throw new IOException("Invalid block array length: " + num);
}
Block prev = null;
Block[] ret = new Block[num];
for (int i = 0; i < num; i++)
{
long id = @in.ReadLong();
long sz = WritableUtils.ReadVLong(@in) + ((prev != null) ? prev.GetNumBytes() : 0
);
long gs = WritableUtils.ReadVLong(@in) + ((prev != null) ? prev.GetGenerationStamp
() : 0);
ret[i] = new Block(id, sz, gs);
prev = ret[i];
}
return(ret);
}
/// <summary>deserialize</summary>
/// <exception cref="System.IO.IOException"/>
public virtual void ReadFields(BinaryReader reader)
{
int newLength = WritableUtils.ReadVInt(@in);
ReadWithKnownLength(@in, newLength);
}
/// <exception cref="System.IO.IOException"/>
public virtual void ReadFields(DataInput @in)
{
bytes = WritableUtils.ReadVLong(@in);
records = WritableUtils.ReadVLong(@in);
checksum = WritableUtils.ReadVInt(@in);
}
/// <summary>Skips over one Text in the input.</summary>
/// <exception cref="System.IO.IOException"/>
public static void Skip(BinaryReader reader)
{
int length = WritableUtils.ReadVInt(@in);
WritableUtils.SkipFully(@in, length);
}
/// <summary>Reads a zero-compressed encoded integer from a stream and returns it.</summary>
/// <param name="in">input stream</param>
/// <exception cref="System.IO.IOException"/>
/// <returns>deserialized integer</returns>
public static int ReadVInt(BinaryReader reader)
{
return(WritableUtils.ReadVInt(@in));
}