/// <exception cref="System.IO.IOException"/>
private void VerifyConnection(Uri url, string msgToEncode, string encHash)
{
// Validate response code
int rc = connection.GetResponseCode();
if (rc != HttpURLConnection.HttpOk)
{
throw new IOException("Got invalid response code " + rc + " from " + url + ": " +
connection.GetResponseMessage());
}
// get the shuffle version
if (!ShuffleHeader.DefaultHttpHeaderName.Equals(connection.GetHeaderField(ShuffleHeader
.HttpHeaderName)) || !ShuffleHeader.DefaultHttpHeaderVersion.Equals(connection.GetHeaderField
(ShuffleHeader.HttpHeaderVersion)))
{
throw new IOException("Incompatible shuffle response version");
}
// get the replyHash which is HMac of the encHash we sent to the server
string replyHash = connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
);
if (replyHash == null)
{
throw new IOException("security validation of TT Map output failed");
}
Log.Debug("url=" + msgToEncode + ";encHash=" + encHash + ";replyHash=" + replyHash
);
// verify that replyHash is HMac of encHash
SecureShuffleUtils.VerifyReply(replyHash, encHash, shuffleSecretKey);
Log.Info("for url=" + msgToEncode + " sent hash and received reply");
}
/// <exception cref="System.Exception"/>
public virtual void TestCopyFromHostWithRetryUnreserve()
{
InMemoryMapOutput <Text, Text> immo = Org.Mockito.Mockito.Mock <InMemoryMapOutput>(
);
Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(jobWithRetry
, id, ss, mm, r, metrics, except, key, connection);
string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
(encHash), key);
Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
)).ThenReturn(replyHash);
ShuffleHeader header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1);
ByteArrayOutputStream bout = new ByteArrayOutputStream();
header.Write(new DataOutputStream(bout));
ByteArrayInputStream @in = new ByteArrayInputStream(bout.ToByteArray());
Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
.ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
)).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
// Verify that unreserve occurs if an exception happens after shuffle
// buffer is reserved.
Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong
(), Matchers.AnyInt())).ThenReturn(immo);
Org.Mockito.Mockito.DoThrow(new IOException("forced error")).When(immo).Shuffle(Matchers.Any
<MapHost>(), Matchers.Any <InputStream>(), Matchers.AnyLong(), Matchers.AnyLong()
, Matchers.Any <ShuffleClientMetrics>(), Matchers.Any <Reporter>());
underTest.CopyFromHost(host);
Org.Mockito.Mockito.Verify(immo).Abort();
}
/// <exception cref="System.Exception"/>
public virtual void TestCopyFromHostCompressFailure()
{
InMemoryMapOutput <Text, Text> immo = Org.Mockito.Mockito.Mock <InMemoryMapOutput>(
);
Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(job, id,
ss, mm, r, metrics, except, key, connection);
string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
(encHash), key);
Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
)).ThenReturn(replyHash);
ShuffleHeader header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1);
ByteArrayOutputStream bout = new ByteArrayOutputStream();
header.Write(new DataOutputStream(bout));
ByteArrayInputStream @in = new ByteArrayInputStream(bout.ToByteArray());
Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
.ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
)).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong
(), Matchers.AnyInt())).ThenReturn(immo);
Org.Mockito.Mockito.DoThrow(new InternalError()).When(immo).Shuffle(Matchers.Any <
MapHost>(), Matchers.Any <InputStream>(), Matchers.AnyLong(), Matchers.AnyLong(),
Matchers.Any <ShuffleClientMetrics>(), Matchers.Any <Reporter>());
underTest.CopyFromHost(host);
Org.Mockito.Mockito.Verify(connection).AddRequestProperty(SecureShuffleUtils.HttpHeaderUrlHash
, encHash);
Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Times(1)).CopyFailed(map1ID, host
, true, false);
}
public virtual void TestCopyFromHostIncompatibleShuffleVersionWithRetry()
{
string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
(encHash), key);
Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
.ThenReturn("mapreduce").ThenReturn("other").ThenReturn("other");
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
)).ThenReturn("1.0.1").ThenReturn("1.0.0").ThenReturn("1.0.1");
Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
)).ThenReturn(replyHash);
ByteArrayInputStream @in = new ByteArrayInputStream(new byte[0]);
Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
for (int i = 0; i < 3; ++i)
{
Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(jobWithRetry
, id, ss, mm, r, metrics, except, key, connection);
underTest.CopyFromHost(host);
}
Org.Mockito.Mockito.Verify(connection, Org.Mockito.Mockito.Times(3)).AddRequestProperty
(SecureShuffleUtils.HttpHeaderUrlHash, encHash);
Org.Mockito.Mockito.Verify(allErrs, Org.Mockito.Mockito.Times(3)).Increment(1);
Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Times(3)).CopyFailed(map1ID, host
, false, false);
Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Times(3)).CopyFailed(map2ID, host
, false, false);
Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Times(3)).PutBackKnownMapOutput
(Matchers.Any <MapHost>(), Matchers.Eq(map1ID));
Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Times(3)).PutBackKnownMapOutput
(Matchers.Any <MapHost>(), Matchers.Eq(map2ID));
}
public virtual void TestCopyFromHostBogusHeader()
{
Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(job, id,
ss, mm, r, metrics, except, key, connection);
string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
(encHash), key);
Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
.ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
)).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
)).ThenReturn(replyHash);
ByteArrayInputStream @in = new ByteArrayInputStream(Sharpen.Runtime.GetBytesForString
("\u00010 BOGUS DATA\nBOGUS DATA\nBOGUS DATA\n"));
Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
underTest.CopyFromHost(host);
Org.Mockito.Mockito.Verify(connection).AddRequestProperty(SecureShuffleUtils.HttpHeaderUrlHash
, encHash);
Org.Mockito.Mockito.Verify(allErrs).Increment(1);
Org.Mockito.Mockito.Verify(ss).CopyFailed(map1ID, host, true, false);
Org.Mockito.Mockito.Verify(ss).CopyFailed(map2ID, host, true, false);
Org.Mockito.Mockito.Verify(ss).PutBackKnownMapOutput(Matchers.Any <MapHost>(), Matchers.Eq
(map1ID));
Org.Mockito.Mockito.Verify(ss).PutBackKnownMapOutput(Matchers.Any <MapHost>(), Matchers.Eq
(map2ID));
}
public virtual void TestReduceOutOfDiskSpace()
{
Log.Info("testReduceOutOfDiskSpace");
Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(job, id,
ss, mm, r, metrics, except, key, connection);
string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
(encHash), key);
ShuffleHeader header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1);
ByteArrayOutputStream bout = new ByteArrayOutputStream();
header.Write(new DataOutputStream(bout));
ByteArrayInputStream @in = new ByteArrayInputStream(bout.ToByteArray());
Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
.ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
)).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
)).ThenReturn(replyHash);
Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong
(), Matchers.AnyInt())).ThenThrow(new DiskChecker.DiskErrorException("No disk space available"
));
underTest.CopyFromHost(host);
Org.Mockito.Mockito.Verify(ss).ReportLocalError(Matchers.Any <IOException>());
}
public virtual void TestCopyFromHostExtraBytes()
{
Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(job, id,
ss, mm, r, metrics, except, key, connection);
string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
(encHash), key);
Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
.ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
)).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
)).ThenReturn(replyHash);
ShuffleHeader header = new ShuffleHeader(map1ID.ToString(), 14, 10, 1);
ByteArrayOutputStream bout = new ByteArrayOutputStream();
DataOutputStream dos = new DataOutputStream(bout);
IFileOutputStream ios = new IFileOutputStream(dos);
header.Write(dos);
ios.Write(Sharpen.Runtime.GetBytesForString("MAPDATA123"));
ios.Finish();
ShuffleHeader header2 = new ShuffleHeader(map2ID.ToString(), 14, 10, 1);
IFileOutputStream ios2 = new IFileOutputStream(dos);
header2.Write(dos);
ios2.Write(Sharpen.Runtime.GetBytesForString("MAPDATA456"));
ios2.Finish();
ByteArrayInputStream @in = new ByteArrayInputStream(bout.ToByteArray());
Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
// 8 < 10 therefore there appear to be extra bytes in the IFileInputStream
InMemoryMapOutput <Text, Text> mapOut = new InMemoryMapOutput <Text, Text>(job, map1ID
, mm, 8, null, true);
InMemoryMapOutput <Text, Text> mapOut2 = new InMemoryMapOutput <Text, Text>(job, map2ID
, mm, 10, null, true);
Org.Mockito.Mockito.When(mm.Reserve(Matchers.Eq(map1ID), Matchers.AnyLong(), Matchers.AnyInt
())).ThenReturn(mapOut);
Org.Mockito.Mockito.When(mm.Reserve(Matchers.Eq(map2ID), Matchers.AnyLong(), Matchers.AnyInt
())).ThenReturn(mapOut2);
underTest.CopyFromHost(host);
Org.Mockito.Mockito.Verify(allErrs).Increment(1);
Org.Mockito.Mockito.Verify(ss).CopyFailed(map1ID, host, true, false);
Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Never()).CopyFailed(map2ID, host
, true, false);
Org.Mockito.Mockito.Verify(ss).PutBackKnownMapOutput(Matchers.Any <MapHost>(), Matchers.Eq
(map1ID));
Org.Mockito.Mockito.Verify(ss).PutBackKnownMapOutput(Matchers.Any <MapHost>(), Matchers.Eq
(map2ID));
}
/// <exception cref="System.Exception"/>
public virtual void TestInterruptOnDisk()
{
int Fetcher = 7;
Path p = new Path("file:///tmp/foo");
Path pTmp = OnDiskMapOutput.GetTempPath(p, Fetcher);
FileSystem mFs = Org.Mockito.Mockito.Mock <FileSystem>(ReturnsDeepStubs);
MapOutputFile mof = Org.Mockito.Mockito.Mock <MapOutputFile>();
Org.Mockito.Mockito.When(mof.GetInputFileForWrite(Matchers.Any <TaskID>(), Matchers.AnyLong
())).ThenReturn(p);
OnDiskMapOutput <Text, Text> odmo = Org.Mockito.Mockito.Spy(new OnDiskMapOutput <Text
, Text>(map1ID, id, mm, 100L, job, mof, Fetcher, true, mFs, p));
Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong
(), Matchers.AnyInt())).ThenReturn(odmo);
Org.Mockito.Mockito.DoNothing().When(mm).WaitForResource();
Org.Mockito.Mockito.When(ss.GetHost()).ThenReturn(host);
string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
(encHash), key);
Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
)).ThenReturn(replyHash);
ShuffleHeader header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1);
ByteArrayOutputStream bout = new ByteArrayOutputStream();
header.Write(new DataOutputStream(bout));
TestFetcher.StuckInputStream @in = new TestFetcher.StuckInputStream(new ByteArrayInputStream
(bout.ToByteArray()));
Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
.ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
)).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
Org.Mockito.Mockito.DoAnswer(new _Answer_610(@in)).When(connection).Disconnect();
Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(job, id,
ss, mm, r, metrics, except, key, connection, Fetcher);
underTest.Start();
// wait for read in inputstream
@in.WaitForFetcher();
underTest.ShutDown();
underTest.Join();
// rely on test timeout to kill if stuck
NUnit.Framework.Assert.IsTrue(@in.WasClosedProperly());
Org.Mockito.Mockito.Verify(mFs).Create(Matchers.Eq(pTmp));
Org.Mockito.Mockito.Verify(mFs).Delete(Matchers.Eq(pTmp), Matchers.Eq(false));
Org.Mockito.Mockito.Verify(odmo).Abort();
}
/// <exception cref="System.IO.IOException"/>
protected internal virtual void VerifyRequest(string appid, ChannelHandlerContext
ctx, HttpRequest request, HttpResponse response, Uri requestUri)
{
SecretKey tokenSecret = this._enclosing.secretManager.RetrieveTokenSecret(appid);
if (null == tokenSecret)
{
ShuffleHandler.Log.Info("Request for unknown token " + appid);
throw new IOException("could not find jobid");
}
// string to encrypt
string enc_str = SecureShuffleUtils.BuildMsgFrom(requestUri);
// hash from the fetcher
string urlHashStr = request.GetHeader(SecureShuffleUtils.HttpHeaderUrlHash);
if (urlHashStr == null)
{
ShuffleHandler.Log.Info("Missing header hash for " + appid);
throw new IOException("fetcher cannot be authenticated");
}
if (ShuffleHandler.Log.IsDebugEnabled())
{
int len = urlHashStr.Length;
ShuffleHandler.Log.Debug("verifying request. enc_str=" + enc_str + "; hash=..." +
Sharpen.Runtime.Substring(urlHashStr, len - len / 2, len - 1));
}
// verify - throws exception
SecureShuffleUtils.VerifyReply(urlHashStr, enc_str, tokenSecret);
// verification passed - encode the reply
string reply = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString(
urlHashStr, Charsets.Utf8), tokenSecret);
response.SetHeader(SecureShuffleUtils.HttpHeaderReplyUrlHash, reply);
// Put shuffle version into http header
response.SetHeader(ShuffleHeader.HttpHeaderName, ShuffleHeader.DefaultHttpHeaderName
);
response.SetHeader(ShuffleHeader.HttpHeaderVersion, ShuffleHeader.DefaultHttpHeaderVersion
);
if (ShuffleHandler.Log.IsDebugEnabled())
{
int len = reply.Length;
ShuffleHandler.Log.Debug("Fetcher request verfied. enc_str=" + enc_str + ";reply="
+ Sharpen.Runtime.Substring(reply, len - len / 2, len - 1));
}
}
/// <exception cref="System.IO.IOException"/>
private void SetupConnectionsWithRetry(MapHost host, ICollection <TaskAttemptID> remaining
, Uri url)
{
OpenConnectionWithRetry(host, remaining, url);
if (stopped)
{
return;
}
// generate hash of the url
string msgToEncode = SecureShuffleUtils.BuildMsgFrom(url);
string encHash = SecureShuffleUtils.HashFromString(msgToEncode, shuffleSecretKey);
SetupShuffleConnection(encHash);
Connect(connection, connectionTimeout);
// verify that the thread wasn't stopped during calls to connect
if (stopped)
{
return;
}
VerifyConnection(url, msgToEncode, encHash);
}
/// <exception cref="System.Exception"/>
public virtual void TestInterruptInMemory()
{
int Fetcher = 2;
InMemoryMapOutput <Text, Text> immo = Org.Mockito.Mockito.Spy(new InMemoryMapOutput
<Text, Text>(job, id, mm, 100, null, true));
Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong
(), Matchers.AnyInt())).ThenReturn(immo);
Org.Mockito.Mockito.DoNothing().When(mm).WaitForResource();
Org.Mockito.Mockito.When(ss.GetHost()).ThenReturn(host);
string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
(encHash), key);
Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
.ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
)).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
)).ThenReturn(replyHash);
ShuffleHeader header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1);
ByteArrayOutputStream bout = new ByteArrayOutputStream();
header.Write(new DataOutputStream(bout));
TestFetcher.StuckInputStream @in = new TestFetcher.StuckInputStream(new ByteArrayInputStream
(bout.ToByteArray()));
Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
Org.Mockito.Mockito.DoAnswer(new _Answer_562(@in)).When(connection).Disconnect();
Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(job, id,
ss, mm, r, metrics, except, key, connection, Fetcher);
underTest.Start();
// wait for read in inputstream
@in.WaitForFetcher();
underTest.ShutDown();
underTest.Join();
// rely on test timeout to kill if stuck
NUnit.Framework.Assert.IsTrue(@in.WasClosedProperly());
Org.Mockito.Mockito.Verify(immo).Abort();
}
/// <exception cref="System.Exception"/>
public virtual void TestCopyFromHostWithRetry()
{
InMemoryMapOutput <Text, Text> immo = Org.Mockito.Mockito.Mock <InMemoryMapOutput>(
);
ss = Org.Mockito.Mockito.Mock <ShuffleSchedulerImpl>();
Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(jobWithRetry
, id, ss, mm, r, metrics, except, key, connection, true);
string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
(encHash), key);
Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
)).ThenReturn(replyHash);
ShuffleHeader header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1);
ByteArrayOutputStream bout = new ByteArrayOutputStream();
header.Write(new DataOutputStream(bout));
ByteArrayInputStream @in = new ByteArrayInputStream(bout.ToByteArray());
Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
.ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
)).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong
(), Matchers.AnyInt())).ThenReturn(immo);
long retryTime = Time.MonotonicNow();
Org.Mockito.Mockito.DoAnswer(new _Answer_375(retryTime)).When(immo).Shuffle(Matchers.Any
<MapHost>(), Matchers.Any <InputStream>(), Matchers.AnyLong(), Matchers.AnyLong()
, Matchers.Any <ShuffleClientMetrics>(), Matchers.Any <Reporter>());
// Emulate host down for 3 seconds.
underTest.CopyFromHost(host);
Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Never()).CopyFailed(Matchers.Any
<TaskAttemptID>(), Matchers.Any <MapHost>(), Matchers.AnyBoolean(), Matchers.AnyBoolean
());
}
public virtual void TestCopyFromHostWait()
{
Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(job, id,
ss, mm, r, metrics, except, key, connection);
string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
(encHash), key);
Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
)).ThenReturn(replyHash);
ShuffleHeader header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1);
ByteArrayOutputStream bout = new ByteArrayOutputStream();
header.Write(new DataOutputStream(bout));
ByteArrayInputStream @in = new ByteArrayInputStream(bout.ToByteArray());
Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
.ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
)).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
//Defaults to null, which is what we want to test
Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong
(), Matchers.AnyInt())).ThenReturn(null);
underTest.CopyFromHost(host);
Org.Mockito.Mockito.Verify(connection).AddRequestProperty(SecureShuffleUtils.HttpHeaderUrlHash
, encHash);
Org.Mockito.Mockito.Verify(allErrs, Org.Mockito.Mockito.Never()).Increment(1);
Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Never()).CopyFailed(map1ID, host
, true, false);
Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Never()).CopyFailed(map2ID, host
, true, false);
Org.Mockito.Mockito.Verify(ss).PutBackKnownMapOutput(Matchers.Any <MapHost>(), Matchers.Eq
(map1ID));
Org.Mockito.Mockito.Verify(ss).PutBackKnownMapOutput(Matchers.Any <MapHost>(), Matchers.Eq
(map2ID));
}
/// <exception cref="System.IO.IOException"/>
public static string CreateDigest(byte[] password, string data)
{
SecretKey key = JobTokenSecretManager.CreateSecretKey(password);
return(SecureShuffleUtils.HashFromString(data, key));
}
/// <exception cref="System.IO.IOException"/>
protected internal virtual string CreateDigest(byte[] password, string data)
{
SecretKey key = JobTokenSecretManager.CreateSecretKey(password);
return(SecureShuffleUtils.HashFromString(data, key));
}