/// <exception cref="System.IO.IOException"/>
private void WriteSessionScript(Path launchDst, Path pidFile)
{
DataOutputStream @out = null;
TextWriter pout = null;
try
{
@out = this._enclosing.lfs.Create(this.sessionScriptPath, EnumSet.Of(CreateFlag.Create
, CreateFlag.Overwrite));
pout = new TextWriter(@out, false, "UTF-8");
// We need to do a move as writing to a file is not atomic
// Process reading a file being written to may get garbled data
// hence write pid to tmp file first followed by a mv
pout.WriteLine("#!/usr/bin/env bash");
pout.WriteLine();
pout.WriteLine("echo " + this.dockerPidScript + " > " + pidFile.ToString() + ".tmp"
);
pout.WriteLine("/bin/mv -f " + pidFile.ToString() + ".tmp " + pidFile);
pout.WriteLine(this.dockerCommand + " bash \"" + launchDst.ToUri().GetPath().ToString
() + "\"");
}
finally
{
IOUtils.Cleanup(DockerContainerExecutor.Log, pout, @out);
}
this._enclosing.lfs.SetPermission(this.sessionScriptPath, ContainerExecutor.TaskLaunchScriptPermission
);
}
/// <summary>Disallow the scanner from scanning the given block pool.</summary>
/// <param name="bpid">The block pool id.</param>
public virtual void DisableBlockPoolId(string bpid)
{
lock (this)
{
IEnumerator <FsVolumeSpi.BlockIterator> i = blockIters.GetEnumerator();
while (i.HasNext())
{
FsVolumeSpi.BlockIterator iter = i.Next();
if (iter.GetBlockPoolId().Equals(bpid))
{
Log.Trace("{}: disabling scanning on block pool {}", this, bpid);
i.Remove();
IOUtils.Cleanup(null, iter);
if (curBlockIter == iter)
{
curBlockIter = null;
}
Sharpen.Runtime.Notify(this);
return;
}
}
Log.Warn("{}: can't remove block pool {}, because it was never " + "added.", this
, bpid);
}
}
public virtual void TestOneJNMissingSegments()
{
QJMTestUtil.WriteSegment(cluster, qjm, 1, 3, true);
WaitForAllPendingCalls(qjm.GetLoggerSetForTests());
cluster.GetJournalNode(0).StopAndJoin(0);
QJMTestUtil.WriteSegment(cluster, qjm, 4, 3, true);
WaitForAllPendingCalls(qjm.GetLoggerSetForTests());
cluster.RestartJournalNode(0);
QJMTestUtil.WriteSegment(cluster, qjm, 7, 3, true);
WaitForAllPendingCalls(qjm.GetLoggerSetForTests());
cluster.GetJournalNode(1).StopAndJoin(0);
QuorumJournalManager readerQjm = CreateSpyingQJM();
IList <EditLogInputStream> streams = Lists.NewArrayList();
try
{
readerQjm.SelectInputStreams(streams, 1, false);
QJMTestUtil.VerifyEdits(streams, 1, 9);
}
finally
{
IOUtils.Cleanup(Log, Sharpen.Collections.ToArray(streams, new IDisposable[0]));
readerQjm.Close();
}
}
/// <summary>
/// test
/// <see>BloomMapFile.Reader.get()</see>
/// method
/// </summary>
public virtual void TestGetBloomMapFile()
{
int Size = 10;
BloomMapFile.Reader reader = null;
BloomMapFile.Writer writer = null;
try
{
writer = new BloomMapFile.Writer(conf, TestFile, MapFile.Writer.KeyClass(typeof(IntWritable
)), MapFile.Writer.ValueClass(typeof(Text)));
for (int i = 0; i < Size; i++)
{
writer.Append(new IntWritable(i), new Text());
}
writer.Close();
reader = new BloomMapFile.Reader(TestFile, conf, MapFile.Reader.Comparator(new WritableComparator
(typeof(IntWritable))));
for (int i_1 = 0; i_1 < Size; i_1++)
{
NUnit.Framework.Assert.IsNotNull("testGetBloomMapFile error !!!", reader.Get(new
IntWritable(i_1), new Text()));
}
NUnit.Framework.Assert.IsNull("testGetBloomMapFile error !!!", reader.Get(new IntWritable
(Size + 5), new Text()));
}
catch (Exception)
{
Fail("unexpect ex in testGetBloomMapFile !!!");
}
finally
{
IOUtils.Cleanup(null, writer, reader);
}
}
/// <exception cref="System.IO.IOException"/>
private static TaskLog.LogFileDetail GetLogFileDetail(TaskAttemptID taskid, TaskLog.LogName
filter, bool isCleanup)
{
FilePath indexFile = GetIndexFile(taskid, isCleanup);
BufferedReader fis = new BufferedReader(new InputStreamReader(SecureIOUtils.OpenForRead
(indexFile, ObtainLogDirOwner(taskid), null), Charsets.Utf8));
//the format of the index file is
//LOG_DIR: <the dir where the task logs are really stored>
//stdout:<start-offset in the stdout file> <length>
//stderr:<start-offset in the stderr file> <length>
//syslog:<start-offset in the syslog file> <length>
TaskLog.LogFileDetail l = new TaskLog.LogFileDetail();
string str = null;
try
{
str = fis.ReadLine();
if (str == null)
{
// the file doesn't have anything
throw new IOException("Index file for the log of " + taskid + " doesn't exist.");
}
l.location = Sharpen.Runtime.Substring(str, str.IndexOf(TaskLog.LogFileDetail.Location
) + TaskLog.LogFileDetail.Location.Length);
// special cases are the debugout and profile.out files. They are
// guaranteed
// to be associated with each task attempt since jvm reuse is disabled
// when profiling/debugging is enabled
if (filter.Equals(TaskLog.LogName.Debugout) || filter.Equals(TaskLog.LogName.Profile
))
{
l.length = new FilePath(l.location, filter.ToString()).Length();
l.start = 0;
fis.Close();
return(l);
}
str = fis.ReadLine();
while (str != null)
{
// look for the exact line containing the logname
if (str.Contains(filter.ToString()))
{
str = Sharpen.Runtime.Substring(str, filter.ToString().Length + 1);
string[] startAndLen = str.Split(" ");
l.start = long.Parse(startAndLen[0]);
l.length = long.Parse(startAndLen[1]);
break;
}
str = fis.ReadLine();
}
fis.Close();
fis = null;
}
finally
{
IOUtils.Cleanup(Log, fis);
}
return(l);
}
public virtual void TestPathExplosionWriterCreation()
{
Path path = new Path(TestDir, "testPathExplosionWriterCreation.mapfile");
string TestErrorMessage = "Mkdirs failed to create directory " + path.GetName();
MapFile.Writer writer = null;
try
{
FileSystem fsSpy = Org.Mockito.Mockito.Spy(FileSystem.Get(conf));
Path pathSpy = Org.Mockito.Mockito.Spy(path);
Org.Mockito.Mockito.When(fsSpy.Mkdirs(path)).ThenThrow(new IOException(TestErrorMessage
));
Org.Mockito.Mockito.When(pathSpy.GetFileSystem(conf)).ThenReturn(fsSpy);
writer = new MapFile.Writer(conf, pathSpy, MapFile.Writer.KeyClass(typeof(IntWritable
)), MapFile.Writer.ValueClass(typeof(IntWritable)));
NUnit.Framework.Assert.Fail("fail in testPathExplosionWriterCreation !!!");
}
catch (IOException ex)
{
Assert.Equal("testPathExplosionWriterCreation ex message error !!!"
, ex.Message, TestErrorMessage);
}
catch (Exception)
{
NUnit.Framework.Assert.Fail("fail in testPathExplosionWriterCreation. Other ex !!!"
);
}
finally
{
IOUtils.Cleanup(null, writer);
}
}
public virtual void TestOpenManyFilesViaTcp()
{
int NumOpens = 500;
Configuration conf = new Configuration();
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, false);
MiniDFSCluster cluster = null;
FSDataInputStream[] streams = new FSDataInputStream[NumOpens];
try
{
cluster = new MiniDFSCluster.Builder(conf).Build();
DistributedFileSystem dfs = cluster.GetFileSystem();
Path TestPath = new Path("/testFile");
DFSTestUtil.CreateFile(dfs, TestPath, 131072, (short)1, 1);
for (int i = 0; i < NumOpens; i++)
{
streams[i] = dfs.Open(TestPath);
Log.Info("opening file " + i + "...");
NUnit.Framework.Assert.IsTrue(-1 != streams[i].Read());
streams[i].Unbuffer();
}
}
finally
{
foreach (FSDataInputStream stream in streams)
{
IOUtils.Cleanup(null, stream);
}
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void Close()
{
lock (this)
{
IOUtils.Cleanup(FileSystemApplicationHistoryStore.Log, this.writer, this.fsdos);
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void WriteHistoryData(FileSystemApplicationHistoryStore.HistoryDataKey
key, byte[] value)
{
lock (this)
{
DataOutputStream dos = null;
try
{
dos = this.writer.PrepareAppendKey(-1);
key.Write(dos);
}
finally
{
IOUtils.Cleanup(FileSystemApplicationHistoryStore.Log, dos);
}
try
{
dos = this.writer.PrepareAppendValue(value.Length);
dos.Write(value);
}
finally
{
IOUtils.Cleanup(FileSystemApplicationHistoryStore.Log, dos);
}
}
}
/// <exception cref="System.IO.IOException"/>
private void WriteFile(Path file, byte[] data)
{
int WriteBufferSize = 4096;
FSDataOutputStream @out = fs.Create(file, FilePermissions, true, WriteBufferSize,
fs.GetDefaultReplication(file), fs.GetDefaultBlockSize(file), null);
try
{
try
{
@out.Write(data);
@out.Close();
@out = null;
}
finally
{
IOUtils.Cleanup(Log, @out);
}
}
catch (IOException e)
{
fs.Delete(file, false);
throw;
}
}
private static string GetUniqueLocalTraceFileName()
{
string tmp = Runtime.GetProperty("java.io.tmpdir", "/tmp");
string nonce = null;
BufferedReader reader = null;
try
{
// On Linux we can get a unique local file name by reading the process id
// out of /proc/self/stat. (There isn't any portable way to get the
// process ID from Java.)
reader = new BufferedReader(new InputStreamReader(new FileInputStream("/proc/self/stat"
), Charsets.Utf8));
string line = reader.ReadLine();
if (line == null)
{
throw new EOFException();
}
nonce = line.Split(" ")[0];
}
catch (IOException)
{
}
finally
{
IOUtils.Cleanup(Log, reader);
}
if (nonce == null)
{
// If we can't use the process ID, use a random nonce.
nonce = UUID.RandomUUID().ToString();
}
return(new FilePath(tmp, nonce).GetAbsolutePath());
}
/// <exception cref="System.IO.IOException"/>
public override void StoreTokenMasterKey(DelegationKey key)
{
if (Log.IsDebugEnabled())
{
Log.Debug("Storing master key " + key.GetKeyId());
}
Path keyPath = new Path(tokenKeysStatePath, TokenMasterKeyFilePrefix + key.GetKeyId
());
if (fs.Exists(keyPath))
{
throw new IOException(keyPath + " already exists");
}
ByteArrayOutputStream memStream = new ByteArrayOutputStream();
DataOutputStream dataStream = new DataOutputStream(memStream);
try
{
key.Write(dataStream);
dataStream.Close();
dataStream = null;
}
finally
{
IOUtils.Cleanup(Log, dataStream);
}
CreateNewFile(keyPath, memStream.ToByteArray());
}
public virtual void TearDown()
{
Runtime.SetOut(oldStdout);
Runtime.SetErr(oldStderr);
IOUtils.Cleanup(Log, printStdout, printStderr);
Assert.True(FileUtil.FullyDelete(TestDir));
}
/// <summary>
/// Read the md5 file stored alongside the given data file
/// and match the md5 file content.
/// </summary>
/// <param name="dataFile">the file containing data</param>
/// <returns>
/// a matcher with two matched groups
/// where group(1) is the md5 string and group(2) is the data file path.
/// </returns>
/// <exception cref="System.IO.IOException"/>
private static Matcher ReadStoredMd5(FilePath md5File)
{
BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream
(md5File), Charsets.Utf8));
string md5Line;
try
{
md5Line = reader.ReadLine();
if (md5Line == null)
{
md5Line = string.Empty;
}
md5Line = md5Line.Trim();
}
catch (IOException ioe)
{
throw new IOException("Error reading md5 file at " + md5File, ioe);
}
finally
{
IOUtils.Cleanup(Log, reader);
}
Matcher matcher = LineRegex.Matcher(md5Line);
if (!matcher.Matches())
{
throw new IOException("Invalid MD5 file " + md5File + ": the content \"" + md5Line
+ "\" does not match the expected pattern.");
}
return(matcher);
}
public virtual void TestOnFinalKey()
{
string TestMethodKey = "testOnFinalKey.mapfile";
int Size = 10;
MapFile.Writer writer = null;
MapFile.Reader reader = null;
try
{
writer = CreateWriter(TestMethodKey, typeof(IntWritable), typeof(IntWritable));
for (int i = 0; i < Size; i++)
{
writer.Append(new IntWritable(i), new IntWritable(i));
}
writer.Close();
reader = CreateReader(TestMethodKey, typeof(IntWritable));
IntWritable expectedKey = new IntWritable(0);
reader.FinalKey(expectedKey);
Assert.Equal("testOnFinalKey not same !!!", expectedKey, new IntWritable
(9));
}
catch (IOException)
{
NUnit.Framework.Assert.Fail("testOnFinalKey error !!!");
}
finally
{
IOUtils.Cleanup(null, writer, reader);
}
}
/// <exception cref="System.IO.IOException"/>
private void OnGetFileChecksum(ChannelHandlerContext ctx)
{
MD5MD5CRC32FileChecksum checksum = null;
string nnId = @params.NamenodeId();
DFSClient dfsclient = NewDfsClient(nnId, conf);
try
{
checksum = dfsclient.GetFileChecksum(path, long.MaxValue);
dfsclient.Close();
dfsclient = null;
}
finally
{
IOUtils.Cleanup(Log, dfsclient);
}
byte[] js = Sharpen.Runtime.GetBytesForString(JsonUtil.ToJsonString(checksum), Charsets
.Utf8);
DefaultFullHttpResponse resp = new DefaultFullHttpResponse(HttpVersion.Http11, HttpResponseStatus
.Ok, Unpooled.WrappedBuffer(js));
resp.Headers().Set(HttpHeaders.Names.ContentType, ApplicationJsonUtf8);
resp.Headers().Set(HttpHeaders.Names.ContentLength, js.Length);
resp.Headers().Set(HttpHeaders.Names.Connection, HttpHeaders.Values.Close);
ctx.WriteAndFlush(resp).AddListener(ChannelFutureListener.Close);
}
public virtual void TestReaderGetClosest()
{
string TestMethodKey = "testReaderWithWrongKeyClass.mapfile";
MapFile.Writer writer = null;
MapFile.Reader reader = null;
try
{
writer = CreateWriter(TestMethodKey, typeof(IntWritable), typeof(Text));
for (int i = 0; i < 10; i++)
{
writer.Append(new IntWritable(i), new Text("value" + i));
}
writer.Close();
reader = CreateReader(TestMethodKey, typeof(Text));
reader.GetClosest(new Text("2"), new Text(string.Empty));
NUnit.Framework.Assert.Fail("no excepted exception in testReaderWithWrongKeyClass !!!"
);
}
catch (IOException)
{
}
finally
{
/* Should be thrown to pass the test */
IOUtils.Cleanup(null, writer, reader);
}
}
/// <summary>Create a file with a length of <code>fileSize</code>.</summary>
/// <remarks>
/// Create a file with a length of <code>fileSize</code>.
/// The file is filled with 'a'.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
private void GenFile(Path file, long fileSize)
{
long startTime = Time.Now();
FSDataOutputStream @out = null;
try
{
@out = Org.Apache.Hadoop.FS.LoadGenerator.LoadGenerator.fc.Create(file, EnumSet.Of
(CreateFlag.Create, CreateFlag.Overwrite), Options.CreateOpts.CreateParent(), Options.CreateOpts
.BufferSize(4096), Options.CreateOpts.RepFac((short)3));
this.executionTime[Org.Apache.Hadoop.FS.LoadGenerator.LoadGenerator.Create] += (Time
.Now() - startTime);
Org.Apache.Hadoop.FS.LoadGenerator.LoadGenerator.numOfOps[Org.Apache.Hadoop.FS.LoadGenerator.LoadGenerator
.Create]++;
long i = fileSize;
while (i > 0)
{
long s = Math.Min(fileSize, this._enclosing.WriteContents.Length);
@out.Write(this._enclosing.WriteContents, 0, (int)s);
i -= s;
}
startTime = Time.Now();
this.executionTime[Org.Apache.Hadoop.FS.LoadGenerator.LoadGenerator.WriteClose] +=
(Time.Now() - startTime);
Org.Apache.Hadoop.FS.LoadGenerator.LoadGenerator.numOfOps[Org.Apache.Hadoop.FS.LoadGenerator.LoadGenerator
.WriteClose]++;
}
finally
{
IOUtils.Cleanup(Org.Apache.Hadoop.FS.LoadGenerator.LoadGenerator.Log, @out);
}
}
public virtual void TestDescOrderWithThrowExceptionWriterAppend()
{
MapFile.Writer writer = null;
try
{
writer = CreateWriter(".mapfile", typeof(IntWritable), typeof(Text));
writer.Append(new IntWritable(2), new Text("value: " + 1));
writer.Append(new IntWritable(2), new Text("value: " + 2));
writer.Append(new IntWritable(2), new Text("value: " + 4));
writer.Append(new IntWritable(1), new Text("value: " + 3));
NUnit.Framework.Assert.Fail("testDescOrderWithThrowExceptionWriterAppend not expected exception error !!!"
);
}
catch (IOException)
{
}
catch (Exception)
{
NUnit.Framework.Assert.Fail("testDescOrderWithThrowExceptionWriterAppend other ex throw !!!"
);
}
finally
{
IOUtils.Cleanup(null, writer);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestWriteConf()
{
Configuration conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsBlockSizeKey, 4096);
System.Console.Out.WriteLine("Setting conf in: " + Runtime.IdentityHashCode(conf)
);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
FileSystem fs = null;
OutputStream os = null;
try
{
fs = cluster.GetFileSystem();
Path filePath = new Path("/testWriteConf.xml");
os = fs.Create(filePath);
StringBuilder longString = new StringBuilder();
for (int i = 0; i < 100000; i++)
{
longString.Append("hello");
}
// 500KB
conf.Set("foobar", longString.ToString());
conf.WriteXml(os);
os.Close();
os = null;
fs.Close();
fs = null;
}
finally
{
IOUtils.Cleanup(null, os, fs);
cluster.Shutdown();
}
}
/// <summary>
/// test
/// <see cref="Reader"/>
/// constructor with
/// IOException
/// </summary>
public virtual void TestIOExceptionInWriterConstructor()
{
Path dirNameSpy = Org.Mockito.Mockito.Spy(TestFile);
BloomMapFile.Reader reader = null;
BloomMapFile.Writer writer = null;
try
{
writer = new BloomMapFile.Writer(conf, TestFile, MapFile.Writer.KeyClass(typeof(IntWritable
)), MapFile.Writer.ValueClass(typeof(Text)));
writer.Append(new IntWritable(1), new Text("123124142"));
writer.Close();
Org.Mockito.Mockito.When(dirNameSpy.GetFileSystem(conf)).ThenThrow(new IOException
());
reader = new BloomMapFile.Reader(dirNameSpy, conf, MapFile.Reader.Comparator(new
WritableComparator(typeof(IntWritable))));
NUnit.Framework.Assert.IsNull("testIOExceptionInWriterConstructor error !!!", reader
.GetBloomFilter());
}
catch (Exception)
{
Fail("unexpect ex in testIOExceptionInWriterConstructor !!!");
}
finally
{
IOUtils.Cleanup(null, writer, reader);
}
}
/// <summary>Close the replica.</summary>
/// <remarks>
/// Close the replica.
/// Must be called after there are no more references to the replica in the
/// cache or elsewhere.
/// </remarks>
internal virtual void Close()
{
string suffix = string.Empty;
Preconditions.CheckState(refCount == 0, "tried to close replica with refCount %d: %s"
, refCount, this);
refCount = -1;
Preconditions.CheckState(purged, "tried to close unpurged replica %s", this);
if (HasMmap())
{
Munmap();
if (Log.IsTraceEnabled())
{
suffix += " munmapped.";
}
}
IOUtils.Cleanup(Log, dataStream, metaStream);
if (slot != null)
{
cache.ScheduleSlotReleaser(slot);
if (Log.IsTraceEnabled())
{
suffix += " scheduling " + slot + " for later release.";
}
}
if (Log.IsTraceEnabled())
{
Log.Trace("closed " + this + suffix);
}
}
/// <exception cref="System.Exception"/>
private void CheckMembershipVaryingSizedKeys(string name, IList <Text> keys)
{
FileSystem fs = FileSystem.GetLocal(conf);
Path qualifiedDirName = fs.MakeQualified(TestDir);
BloomMapFile.Writer writer = null;
BloomMapFile.Reader reader = null;
try
{
writer = new BloomMapFile.Writer(conf, fs, qualifiedDirName.ToString(), typeof(Text
), typeof(NullWritable));
foreach (Text key in keys)
{
writer.Append(key, NullWritable.Get());
}
writer.Close();
// will check for membership in opposite order of how keys were inserted
reader = new BloomMapFile.Reader(fs, qualifiedDirName.ToString(), conf);
Collections.Reverse(keys);
foreach (Text key_1 in keys)
{
Assert.True("False negative for existing key " + key_1, reader.
ProbablyHasKey(key_1));
}
reader.Close();
fs.Delete(qualifiedDirName, true);
}
finally
{
IOUtils.Cleanup(null, writer, reader);
}
}
public virtual void TestMidKeyOnCurrentApi()
{
// Write a mapfile of simple data: keys are
string TestPrefix = "testMidKeyOnCurrentApi.mapfile";
MapFile.Writer writer = null;
MapFile.Reader reader = null;
try
{
writer = CreateWriter(TestPrefix, typeof(IntWritable), typeof(IntWritable));
// 0,1,....9
int Size = 10;
for (int i = 0; i < Size; i++)
{
writer.Append(new IntWritable(i), new IntWritable(i));
}
writer.Close();
reader = CreateReader(TestPrefix, typeof(IntWritable));
Assert.Equal(new IntWritable((Size - 1) / 2), reader.MidKey());
}
finally
{
IOUtils.Cleanup(null, writer, reader);
}
}
/*
* In order to make this write atomic as a part of write we will first write
* data to .tmp file and then rename it. Here we are assuming that rename is
* atomic for underlying file system.
*/
/// <exception cref="System.Exception"/>
protected internal virtual void WriteFile(Path outputPath, byte[] data, bool makeUnradableByAdmin
)
{
Path tempPath = new Path(outputPath.GetParent(), outputPath.GetName() + ".tmp");
FSDataOutputStream fsOut = null;
// This file will be overwritten when app/attempt finishes for saving the
// final status.
try
{
fsOut = fs.Create(tempPath, true);
if (makeUnradableByAdmin)
{
SetUnreadableBySuperuserXattrib(tempPath);
}
fsOut.Write(data);
fsOut.Close();
fsOut = null;
fs.Rename(tempPath, outputPath);
}
finally
{
IOUtils.Cleanup(Log, fsOut);
}
}
public virtual void TestRenameWithFalse()
{
string ErrorMessage = "Could not rename";
string NewFileName = "test-new.mapfile";
string OldFileName = "test-old.mapfile";
MapFile.Writer writer = null;
try
{
FileSystem fs = FileSystem.GetLocal(conf);
FileSystem spyFs = Org.Mockito.Mockito.Spy(fs);
writer = CreateWriter(OldFileName, typeof(IntWritable), typeof(IntWritable));
writer.Close();
Path oldDir = new Path(TestDir, OldFileName);
Path newDir = new Path(TestDir, NewFileName);
Org.Mockito.Mockito.When(spyFs.Rename(oldDir, newDir)).ThenReturn(false);
MapFile.Rename(spyFs, oldDir.ToString(), newDir.ToString());
NUnit.Framework.Assert.Fail("testRenameWithException no exception error !!!");
}
catch (IOException ex)
{
Assert.True("testRenameWithFalse invalid IOExceptionMessage error !!!"
, ex.Message.StartsWith(ErrorMessage));
}
finally
{
IOUtils.Cleanup(null, writer);
}
}
public virtual void TestReaderWhileAnotherWrites()
{
QuorumJournalManager readerQjm = CloseLater(CreateSpyingQJM());
IList <EditLogInputStream> streams = Lists.NewArrayList();
readerQjm.SelectInputStreams(streams, 0, false);
NUnit.Framework.Assert.AreEqual(0, streams.Count);
QJMTestUtil.WriteSegment(cluster, qjm, 1, 3, true);
readerQjm.SelectInputStreams(streams, 0, false);
try
{
NUnit.Framework.Assert.AreEqual(1, streams.Count);
// Validate the actual stream contents.
EditLogInputStream stream = streams[0];
NUnit.Framework.Assert.AreEqual(1, stream.GetFirstTxId());
NUnit.Framework.Assert.AreEqual(3, stream.GetLastTxId());
QJMTestUtil.VerifyEdits(streams, 1, 3);
NUnit.Framework.Assert.IsNull(stream.ReadOp());
}
finally
{
IOUtils.Cleanup(Log, Sharpen.Collections.ToArray(streams, new IDisposable[0]));
streams.Clear();
}
// Ensure correct results when there is a stream in-progress, but we don't
// ask for in-progress.
QJMTestUtil.WriteSegment(cluster, qjm, 4, 3, false);
readerQjm.SelectInputStreams(streams, 0, false);
try
{
NUnit.Framework.Assert.AreEqual(1, streams.Count);
EditLogInputStream stream = streams[0];
NUnit.Framework.Assert.AreEqual(1, stream.GetFirstTxId());
NUnit.Framework.Assert.AreEqual(3, stream.GetLastTxId());
QJMTestUtil.VerifyEdits(streams, 1, 3);
}
finally
{
IOUtils.Cleanup(Log, Sharpen.Collections.ToArray(streams, new IDisposable[0]));
streams.Clear();
}
// TODO: check results for selectInputStreams with inProgressOK = true.
// This doesn't currently work, due to a bug where RedundantEditInputStream
// throws an exception if there are any unvalidated in-progress edits in the list!
// But, it shouldn't be necessary for current use cases.
qjm.FinalizeLogSegment(4, 6);
readerQjm.SelectInputStreams(streams, 0, false);
try
{
NUnit.Framework.Assert.AreEqual(2, streams.Count);
NUnit.Framework.Assert.AreEqual(4, streams[1].GetFirstTxId());
NUnit.Framework.Assert.AreEqual(6, streams[1].GetLastTxId());
QJMTestUtil.VerifyEdits(streams, 1, 6);
}
finally
{
IOUtils.Cleanup(Log, Sharpen.Collections.ToArray(streams, new IDisposable[0]));
streams.Clear();
}
}
public virtual void TestWriteWithFailDirCreation()
{
string ErrorMessage = "Mkdirs failed to create directory";
Path dirName = new Path(TestDir, "fail.mapfile");
MapFile.Writer writer = null;
try
{
FileSystem fs = FileSystem.GetLocal(conf);
FileSystem spyFs = Org.Mockito.Mockito.Spy(fs);
Path pathSpy = Org.Mockito.Mockito.Spy(dirName);
Org.Mockito.Mockito.When(pathSpy.GetFileSystem(conf)).ThenReturn(spyFs);
Org.Mockito.Mockito.When(spyFs.Mkdirs(dirName)).ThenReturn(false);
writer = new MapFile.Writer(conf, pathSpy, MapFile.Writer.KeyClass(typeof(IntWritable
)), MapFile.Writer.ValueClass(typeof(Text)));
NUnit.Framework.Assert.Fail("testWriteWithFailDirCreation error !!!");
}
catch (IOException ex)
{
Assert.True("testWriteWithFailDirCreation ex error !!!", ex.Message
.StartsWith(ErrorMessage));
}
finally
{
IOUtils.Cleanup(null, writer);
}
}
/// <summary>Read the cut points from the given IFile.</summary>
/// <param name="fs">The file system</param>
/// <param name="p">The path to read</param>
/// <param name="keyClass">The map output key class</param>
/// <param name="job">The job config</param>
/// <exception cref="System.IO.IOException"/>
private K[] ReadPartitions(FileSystem fs, Path p, Type keyClass, Configuration conf
)
{
// matching key types enforced by passing in
// map output key class
SequenceFile.Reader reader = new SequenceFile.Reader(fs, p, conf);
AList <K> parts = new AList <K>();
K key = ReflectionUtils.NewInstance(keyClass, conf);
NullWritable value = NullWritable.Get();
try
{
while (reader.Next(key, value))
{
parts.AddItem(key);
key = ReflectionUtils.NewInstance(keyClass, conf);
}
reader.Close();
reader = null;
}
finally
{
IOUtils.Cleanup(Log, reader);
}
return(Sharpen.Collections.ToArray(parts, (K[])System.Array.CreateInstance(keyClass
, parts.Count)));
}
public virtual void TestHedgedReadLoopTooManyTimes()
{
Configuration conf = new Configuration();
int numHedgedReadPoolThreads = 5;
int hedgedReadTimeoutMillis = 50;
conf.SetInt(DFSConfigKeys.DfsDfsclientHedgedReadThreadpoolSize, numHedgedReadPoolThreads
);
conf.SetLong(DFSConfigKeys.DfsDfsclientHedgedReadThresholdMillis, hedgedReadTimeoutMillis
);
conf.SetInt(DFSConfigKeys.DfsClientRetryWindowBase, 0);
// Set up the InjectionHandler
DFSClientFaultInjector.instance = Org.Mockito.Mockito.Mock <DFSClientFaultInjector
>();
DFSClientFaultInjector injector = DFSClientFaultInjector.instance;
int sleepMs = 100;
Org.Mockito.Mockito.DoAnswer(new _Answer_296(hedgedReadTimeoutMillis, sleepMs)).When
(injector).FetchFromDatanodeException();
Org.Mockito.Mockito.DoAnswer(new _Answer_309(sleepMs)).When(injector).ReadFromDatanodeDelay
();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Format(
true).Build();
DistributedFileSystem fileSys = cluster.GetFileSystem();
DFSClient dfsClient = fileSys.GetClient();
FSDataOutputStream output = null;
DFSInputStream input = null;
string filename = "/hedgedReadMaxOut.dat";
try
{
Path file = new Path(filename);
output = fileSys.Create(file, (short)2);
byte[] data = new byte[64 * 1024];
output.Write(data);
output.Flush();
output.Write(data);
output.Flush();
output.Write(data);
output.Flush();
output.Close();
byte[] buffer = new byte[64 * 1024];
input = dfsClient.Open(filename);
input.Read(0, buffer, 0, 1024);
input.Close();
NUnit.Framework.Assert.AreEqual(3, input.GetHedgedReadOpsLoopNumForTesting());
}
catch (BlockMissingException)
{
NUnit.Framework.Assert.IsTrue(false);
}
finally
{
Org.Mockito.Mockito.Reset(injector);
IOUtils.Cleanup(null, input);
IOUtils.Cleanup(null, output);
fileSys.Close();
cluster.Shutdown();
}
}
