private static void CustomShuffleTransferCornerCases(FadvisedFileRegion fileRegion
, WritableByteChannel target, int count)
{
try
{
fileRegion.CustomShuffleTransfer(target, -1);
NUnit.Framework.Assert.Fail("Expected a IllegalArgumentException");
}
catch (ArgumentException)
{
Log.Info("Expected - illegal argument is passed.");
}
catch (Exception)
{
NUnit.Framework.Assert.Fail("Expected a IllegalArgumentException");
}
//test corner cases
try
{
fileRegion.CustomShuffleTransfer(target, count + 1);
NUnit.Framework.Assert.Fail("Expected a IllegalArgumentException");
}
catch (ArgumentException)
{
Log.Info("Expected - illegal argument is passed.");
}
catch (Exception)
{
NUnit.Framework.Assert.Fail("Expected a IllegalArgumentException");
}
}
/// <exception cref="System.IO.IOException"/>
protected internal virtual ChannelFuture SendMapOutput(ChannelHandlerContext ctx,
Org.Jboss.Netty.Channel.Channel ch, string user, string mapId, int reduce, ShuffleHandler.Shuffle.MapOutputInfo
mapOutputInfo)
{
IndexRecord info = mapOutputInfo.indexRecord;
ShuffleHeader header = new ShuffleHeader(mapId, info.partLength, info.rawLength,
reduce);
DataOutputBuffer dob = new DataOutputBuffer();
header.Write(dob);
ch.Write(ChannelBuffers.WrappedBuffer(dob.GetData(), 0, dob.GetLength()));
FilePath spillfile = new FilePath(mapOutputInfo.mapOutputFileName.ToString());
RandomAccessFile spill;
try
{
spill = SecureIOUtils.OpenForRandomRead(spillfile, "r", user, null);
}
catch (FileNotFoundException)
{
ShuffleHandler.Log.Info(spillfile + " not found");
return null;
}
ChannelFuture writeFuture;
if (ch.GetPipeline().Get<SslHandler>() == null)
{
FadvisedFileRegion partition = new FadvisedFileRegion(spill, info.startOffset, info
.partLength, this._enclosing.manageOsCache, this._enclosing.readaheadLength, this
._enclosing.readaheadPool, spillfile.GetAbsolutePath(), this._enclosing.shuffleBufferSize
, this._enclosing.shuffleTransferToAllowed);
writeFuture = ch.Write(partition);
writeFuture.AddListener(new _ChannelFutureListener_1135(partition));
}
else
{
// TODO error handling; distinguish IO/connection failures,
// attribute to appropriate spill output
// HTTPS cannot be done with zero copy.
FadvisedChunkedFile chunk = new FadvisedChunkedFile(spill, info.startOffset, info
.partLength, this._enclosing.sslFileBufferSize, this._enclosing.manageOsCache, this
._enclosing.readaheadLength, this._enclosing.readaheadPool, spillfile.GetAbsolutePath
());
writeFuture = ch.Write(chunk);
}
this._enclosing.metrics.shuffleConnections.Incr();
this._enclosing.metrics.shuffleOutputBytes.Incr(info.partLength);
// optimistic
return writeFuture;
}
public _ChannelFutureListener_1135(FadvisedFileRegion partition)
{
this.partition = partition;
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestCustomShuffleTransfer()
{
FilePath absLogDir = new FilePath("target", typeof(TestFadvisedFileRegion).Name +
"LocDir").GetAbsoluteFile();
string testDirPath = StringUtils.Join(Path.Separator, new string[] { absLogDir.GetAbsolutePath
(), "testCustomShuffleTransfer" });
FilePath testDir = new FilePath(testDirPath);
testDir.Mkdirs();
System.Console.Out.WriteLine(testDir.GetAbsolutePath());
FilePath inFile = new FilePath(testDir, "fileIn.out");
FilePath outFile = new FilePath(testDir, "fileOut.out");
//Initialize input file
byte[] initBuff = new byte[FileSize];
Random rand = new Random();
rand.NextBytes(initBuff);
FileOutputStream @out = new FileOutputStream(inFile);
try
{
@out.Write(initBuff);
}
finally
{
IOUtils.Cleanup(Log, @out);
}
//define position and count to read from a file region.
int position = 2 * 1024 * 1024;
int count = 4 * 1024 * 1024 - 1;
RandomAccessFile inputFile = null;
RandomAccessFile targetFile = null;
WritableByteChannel target = null;
FadvisedFileRegion fileRegion = null;
try
{
inputFile = new RandomAccessFile(inFile.GetAbsolutePath(), "r");
targetFile = new RandomAccessFile(outFile.GetAbsolutePath(), "rw");
target = targetFile.GetChannel();
NUnit.Framework.Assert.AreEqual(FileSize, inputFile.Length());
//create FadvisedFileRegion
fileRegion = new FadvisedFileRegion(inputFile, position, count, false, 0, null, null
, 1024, false);
//test corner cases
CustomShuffleTransferCornerCases(fileRegion, target, count);
long pos = 0;
long size;
while ((size = fileRegion.CustomShuffleTransfer(target, pos)) > 0)
{
pos += size;
}
//assert size
NUnit.Framework.Assert.AreEqual(count, (int)pos);
NUnit.Framework.Assert.AreEqual(count, targetFile.Length());
}
finally
{
if (fileRegion != null)
{
fileRegion.ReleaseExternalResources();
}
IOUtils.Cleanup(Log, target);
IOUtils.Cleanup(Log, targetFile);
IOUtils.Cleanup(Log, inputFile);
}
//Read the target file and verify that copy is done correctly
byte[] buff = new byte[FileSize];
FileInputStream @in = new FileInputStream(outFile);
try
{
int total = @in.Read(buff, 0, count);
NUnit.Framework.Assert.AreEqual(count, total);
for (int i = 0; i < count; i++)
{
NUnit.Framework.Assert.AreEqual(initBuff[position + i], buff[i]);
}
}
finally
{
IOUtils.Cleanup(Log, @in);
}
//delete files and folders
inFile.Delete();
outFile.Delete();
testDir.Delete();
absLogDir.Delete();
}
