/// <exception cref="System.IO.IOException"/>
public SpillRecord(Path indexFileName, JobConf job, Checksum crc, string expectedIndexOwner
)
{
FileSystem rfs = FileSystem.GetLocal(job).GetRaw();
FSDataInputStream @in = SecureIOUtils.OpenFSDataInputStream(new FilePath(indexFileName
.ToUri().GetRawPath()), expectedIndexOwner, null);
try
{
long length = rfs.GetFileStatus(indexFileName).GetLen();
int partitions = (int)length / MapTask.MapOutputIndexRecordLength;
int size = partitions * MapTask.MapOutputIndexRecordLength;
buf = ByteBuffer.Allocate(size);
if (crc != null)
{
crc.Reset();
CheckedInputStream chk = new CheckedInputStream(@in, crc);
IOUtils.ReadFully(chk, ((byte[])buf.Array()), 0, size);
if (chk.GetChecksum().GetValue() != @in.ReadLong())
{
throw new ChecksumException("Checksum error reading spill index: " + indexFileName
, -1);
}
}
else
{
IOUtils.ReadFully(@in, ((byte[])buf.Array()), 0, size);
}
entries = buf.AsLongBuffer();
}
finally
{
@in.Close();
}
}
/// <exception cref="System.IO.IOException"/>
private static TaskLog.LogFileDetail GetLogFileDetail(TaskAttemptID taskid, TaskLog.LogName
filter, bool isCleanup)
{
FilePath indexFile = GetIndexFile(taskid, isCleanup);
BufferedReader fis = new BufferedReader(new InputStreamReader(SecureIOUtils.OpenForRead
(indexFile, ObtainLogDirOwner(taskid), null), Charsets.Utf8));
//the format of the index file is
//LOG_DIR: <the dir where the task logs are really stored>
//stdout:<start-offset in the stdout file> <length>
//stderr:<start-offset in the stderr file> <length>
//syslog:<start-offset in the syslog file> <length>
TaskLog.LogFileDetail l = new TaskLog.LogFileDetail();
string str = null;
try
{
str = fis.ReadLine();
if (str == null)
{
// the file doesn't have anything
throw new IOException("Index file for the log of " + taskid + " doesn't exist.");
}
l.location = Sharpen.Runtime.Substring(str, str.IndexOf(TaskLog.LogFileDetail.Location
) + TaskLog.LogFileDetail.Location.Length);
// special cases are the debugout and profile.out files. They are
// guaranteed
// to be associated with each task attempt since jvm reuse is disabled
// when profiling/debugging is enabled
if (filter.Equals(TaskLog.LogName.Debugout) || filter.Equals(TaskLog.LogName.Profile
))
{
l.length = new FilePath(l.location, filter.ToString()).Length();
l.start = 0;
fis.Close();
return(l);
}
str = fis.ReadLine();
while (str != null)
{
// look for the exact line containing the logname
if (str.Contains(filter.ToString()))
{
str = Sharpen.Runtime.Substring(str, filter.ToString().Length + 1);
string[] startAndLen = str.Split(" ");
l.start = long.Parse(startAndLen[0]);
l.length = long.Parse(startAndLen[1]);
break;
}
str = fis.ReadLine();
}
fis.Close();
fis = null;
}
finally
{
IOUtils.Cleanup(Log, fis);
}
return(l);
}
/// <exception cref="System.IO.IOException"/>
public static FileInputStream OpenLogFileForRead(string containerIdStr, FilePath
logFile, Context context)
{
ContainerId containerId = ConverterUtils.ToContainerId(containerIdStr);
ApplicationId applicationId = containerId.GetApplicationAttemptId().GetApplicationId
();
string user = context.GetApplications()[applicationId].GetUser();
try
{
return(SecureIOUtils.OpenForRead(logFile, user, null));
}
catch (IOException e)
{
if (e.Message.Contains("did not match expected owner '" + user + "'"))
{
Log.Error("Exception reading log file " + logFile.GetAbsolutePath(), e);
throw new IOException("Exception reading log file. Application submitted by '" +
user + "' doesn't own requested log file : " + logFile.GetName(), e);
}
else
{
throw new IOException("Exception reading log file. It might be because log " + "file was aggregated : "
+ logFile.GetName(), e);
}
}
}
/// <exception cref="System.IO.IOException"/>
private static void WriteToIndexFile(string logLocation, bool isCleanup)
{
lock (typeof(TaskLog))
{
// To ensure atomicity of updates to index file, write to temporary index
// file first and then rename.
FilePath tmpIndexFile = GetTmpIndexFile(currentTaskid, isCleanup);
BufferedOutputStream bos = null;
DataOutputStream dos = null;
try
{
bos = new BufferedOutputStream(SecureIOUtils.CreateForWrite(tmpIndexFile, 0x1a4));
dos = new DataOutputStream(bos);
//the format of the index file is
//LOG_DIR: <the dir where the task logs are really stored>
//STDOUT: <start-offset in the stdout file> <length>
//STDERR: <start-offset in the stderr file> <length>
//SYSLOG: <start-offset in the syslog file> <length>
dos.WriteBytes(TaskLog.LogFileDetail.Location + logLocation + "\n" + TaskLog.LogName
.Stdout.ToString() + ":");
dos.WriteBytes(System.Convert.ToString(prevOutLength) + " ");
dos.WriteBytes(System.Convert.ToString(new FilePath(logLocation, TaskLog.LogName.
Stdout.ToString()).Length() - prevOutLength) + "\n" + TaskLog.LogName.Stderr + ":"
);
dos.WriteBytes(System.Convert.ToString(prevErrLength) + " ");
dos.WriteBytes(System.Convert.ToString(new FilePath(logLocation, TaskLog.LogName.
Stderr.ToString()).Length() - prevErrLength) + "\n" + TaskLog.LogName.Syslog.ToString
() + ":");
dos.WriteBytes(System.Convert.ToString(prevLogLength) + " ");
dos.WriteBytes(System.Convert.ToString(new FilePath(logLocation, TaskLog.LogName.
Syslog.ToString()).Length() - prevLogLength) + "\n");
dos.Close();
dos = null;
bos.Close();
bos = null;
}
finally
{
IOUtils.Cleanup(Log, dos, bos);
}
FilePath indexFile = GetIndexFile(currentTaskid, isCleanup);
Path indexFilePath = new Path(indexFile.GetAbsolutePath());
Path tmpIndexFilePath = new Path(tmpIndexFile.GetAbsolutePath());
if (localFS == null)
{
// set localFS once
localFS = FileSystem.GetLocal(new Configuration());
}
localFS.Rename(tmpIndexFilePath, indexFilePath);
}
}
/// <exception cref="System.IO.IOException"/>
protected internal virtual ChannelFuture SendMapOutput(ChannelHandlerContext ctx,
Org.Jboss.Netty.Channel.Channel ch, string user, string mapId, int reduce, ShuffleHandler.Shuffle.MapOutputInfo
mapOutputInfo)
{
IndexRecord info = mapOutputInfo.indexRecord;
ShuffleHeader header = new ShuffleHeader(mapId, info.partLength, info.rawLength,
reduce);
DataOutputBuffer dob = new DataOutputBuffer();
header.Write(dob);
ch.Write(ChannelBuffers.WrappedBuffer(dob.GetData(), 0, dob.GetLength()));
FilePath spillfile = new FilePath(mapOutputInfo.mapOutputFileName.ToString());
RandomAccessFile spill;
try
{
spill = SecureIOUtils.OpenForRandomRead(spillfile, "r", user, null);
}
catch (FileNotFoundException)
{
ShuffleHandler.Log.Info(spillfile + " not found");
return null;
}
ChannelFuture writeFuture;
if (ch.GetPipeline().Get<SslHandler>() == null)
{
FadvisedFileRegion partition = new FadvisedFileRegion(spill, info.startOffset, info
.partLength, this._enclosing.manageOsCache, this._enclosing.readaheadLength, this
._enclosing.readaheadPool, spillfile.GetAbsolutePath(), this._enclosing.shuffleBufferSize
, this._enclosing.shuffleTransferToAllowed);
writeFuture = ch.Write(partition);
writeFuture.AddListener(new _ChannelFutureListener_1135(partition));
}
else
{
// TODO error handling; distinguish IO/connection failures,
// attribute to appropriate spill output
// HTTPS cannot be done with zero copy.
FadvisedChunkedFile chunk = new FadvisedChunkedFile(spill, info.startOffset, info
.partLength, this._enclosing.sslFileBufferSize, this._enclosing.manageOsCache, this
._enclosing.readaheadLength, this._enclosing.readaheadPool, spillfile.GetAbsolutePath
());
writeFuture = ch.Write(chunk);
}
this._enclosing.metrics.shuffleConnections.Incr();
this._enclosing.metrics.shuffleOutputBytes.Incr(info.partLength);
// optimistic
return writeFuture;
}
/// <summary>Read a log file from start to end positions.</summary>
/// <remarks>
/// Read a log file from start to end positions. The offsets may be negative,
/// in which case they are relative to the end of the file. For example,
/// Reader(taskid, kind, 0, -1) is the entire file and
/// Reader(taskid, kind, -4197, -1) is the last 4196 bytes.
/// </remarks>
/// <param name="taskid">the id of the task to read the log file for</param>
/// <param name="kind">the kind of log to read</param>
/// <param name="start">the offset to read from (negative is relative to tail)</param>
/// <param name="end">the offset to read upto (negative is relative to tail)</param>
/// <param name="isCleanup">whether the attempt is cleanup attempt or not</param>
/// <exception cref="System.IO.IOException"/>
public Reader(TaskAttemptID taskid, TaskLog.LogName kind, long start, long end, bool
isCleanup)
{
// find the right log file
TaskLog.LogFileDetail fileDetail = GetLogFileDetail(taskid, kind, isCleanup);
// calculate the start and stop
long size = fileDetail.length;
if (start < 0)
{
start += size + 1;
}
if (end < 0)
{
end += size + 1;
}
start = Math.Max(0, Math.Min(start, size));
end = Math.Max(0, Math.Min(end, size));
start += fileDetail.start;
end += fileDetail.start;
bytesRemaining = end - start;
string owner = ObtainLogDirOwner(taskid);
file = SecureIOUtils.OpenForRead(new FilePath(fileDetail.location, kind.ToString(
)), owner, null);
// skip upto start
long pos = 0;
while (pos < start)
{
long result = file.Skip(start - pos);
if (result < 0)
{
bytesRemaining = 0;
break;
}
pos += result;
}
}
public virtual FileInputStream SecureOpenFile(FilePath logFile)
{
return(SecureIOUtils.OpenForRead(logFile, GetUser(), null));
}
