/// <summary>Convert from a JSON file</summary>
/// <param name="resource">input file</param>
/// <returns>the parsed JSON</returns>
/// <exception cref="System.IO.IOException">IO problems</exception>
/// <exception cref="Org.Codehaus.Jackson.Map.JsonMappingException">failure to map from the JSON to this class
/// </exception>
/// <exception cref="Org.Codehaus.Jackson.JsonParseException"/>
public virtual T FromResource(string resource)
{
lock (this)
{
InputStream resStream = null;
try
{
resStream = this.GetType().GetResourceAsStream(resource);
if (resStream == null)
{
throw new FileNotFoundException(resource);
}
return(mapper.ReadValue(resStream, classType));
}
catch (IOException e)
{
Log.Error("Exception while parsing json resource {}: {}", resource, e);
throw;
}
finally
{
IOUtils.CloseStream(resStream);
}
}
}
/// <summary>
/// Test which randomly alternates between appending with
/// CRC32 and with CRC32C, crossing several block boundaries.
/// </summary>
/// <remarks>
/// Test which randomly alternates between appending with
/// CRC32 and with CRC32C, crossing several block boundaries.
/// Then, checks that all of the data can be read back correct.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
public virtual void TestAlgoSwitchRandomized()
{
FileSystem fsWithCrc32 = CreateFsWithChecksum("CRC32", 512);
FileSystem fsWithCrc32C = CreateFsWithChecksum("CRC32C", 512);
Path p = new Path("/testAlgoSwitchRandomized");
long seed = Time.Now();
System.Console.Out.WriteLine("seed: " + seed);
Random r = new Random(seed);
// Create empty to start
IOUtils.CloseStream(fsWithCrc32.Create(p));
long st = Time.Now();
int len = 0;
while (Time.Now() - st < RandomTestRuntime)
{
int thisLen = r.Next(500);
FileSystem fs = (r.NextBoolean() ? fsWithCrc32 : fsWithCrc32C);
FSDataOutputStream stm = fs.Append(p);
try
{
AppendTestUtil.Write(stm, len, thisLen);
}
finally
{
stm.Close();
}
len += thisLen;
}
AppendTestUtil.Check(fsWithCrc32, p, len);
AppendTestUtil.Check(fsWithCrc32C, p, len);
}
/// <exception cref="System.IO.IOException"/>
private void TestRbwReplicas(MiniDFSCluster cluster, bool isCorrupt)
{
FSDataOutputStream @out = null;
FileSystem fs = cluster.GetFileSystem();
Path src = new Path("/test.txt");
try
{
int fileLen = 515;
// create some rbw replicas on disk
byte[] writeBuf = new byte[fileLen];
new Random().NextBytes(writeBuf);
@out = fs.Create(src);
@out.Write(writeBuf);
@out.Hflush();
DataNode dn = cluster.GetDataNodes()[0];
foreach (FsVolumeSpi v in Dataset(dn).GetVolumes())
{
FsVolumeImpl volume = (FsVolumeImpl)v;
FilePath currentDir = volume.GetCurrentDir().GetParentFile().GetParentFile();
FilePath rbwDir = new FilePath(currentDir, "rbw");
foreach (FilePath file in rbwDir.ListFiles())
{
if (isCorrupt && Block.IsBlockFilename(file))
{
new RandomAccessFile(file, "rw").SetLength(fileLen - 1);
}
}
}
// corrupt
cluster.RestartDataNodes();
cluster.WaitActive();
dn = cluster.GetDataNodes()[0];
// check volumeMap: one rwr replica
string bpid = cluster.GetNamesystem().GetBlockPoolId();
ReplicaMap replicas = Dataset(dn).volumeMap;
NUnit.Framework.Assert.AreEqual(1, replicas.Size(bpid));
ReplicaInfo replica = replicas.Replicas(bpid).GetEnumerator().Next();
NUnit.Framework.Assert.AreEqual(HdfsServerConstants.ReplicaState.Rwr, replica.GetState
());
if (isCorrupt)
{
NUnit.Framework.Assert.AreEqual((fileLen - 1) / 512 * 512, replica.GetNumBytes());
}
else
{
NUnit.Framework.Assert.AreEqual(fileLen, replica.GetNumBytes());
}
Dataset(dn).Invalidate(bpid, new Block[] { replica });
}
finally
{
IOUtils.CloseStream(@out);
if (fs.Exists(src))
{
fs.Delete(src, false);
}
fs.Close();
}
}
/// <summary>Read checksum into given buffer</summary>
/// <param name="buf">buffer to read the checksum into</param>
/// <param name="checksumOffset">offset at which to write the checksum into buf</param>
/// <param name="checksumLen">length of checksum to write</param>
/// <exception cref="System.IO.IOException">on error</exception>
private void ReadChecksum(byte[] buf, int checksumOffset, int checksumLen)
{
if (checksumSize <= 0 && checksumIn == null)
{
return;
}
try
{
checksumIn.ReadFully(buf, checksumOffset, checksumLen);
}
catch (IOException e)
{
Log.Warn(" Could not read or failed to veirfy checksum for data" + " at offset "
+ offset + " for block " + block, e);
IOUtils.CloseStream(checksumIn);
checksumIn = null;
if (corruptChecksumOk)
{
if (checksumOffset < checksumLen)
{
// Just fill the array with zeros.
Arrays.Fill(buf, checksumOffset, checksumLen, unchecked ((byte)0));
}
}
else
{
throw;
}
}
}
/// <summary>
/// The idea for making sure that there is no more than one instance
/// running in an HDFS is to create a file in the HDFS, writes the hostname
/// of the machine on which the instance is running to the file, but did not
/// close the file until it exits.
/// </summary>
/// <remarks>
/// The idea for making sure that there is no more than one instance
/// running in an HDFS is to create a file in the HDFS, writes the hostname
/// of the machine on which the instance is running to the file, but did not
/// close the file until it exits.
/// This prevents the second instance from running because it can not
/// creates the file while the first one is running.
/// This method checks if there is any running instance. If no, mark yes.
/// Note that this is an atomic operation.
/// </remarks>
/// <returns>
/// null if there is a running instance;
/// otherwise, the output stream to the newly created file.
/// </returns>
/// <exception cref="System.IO.IOException"/>
private OutputStream CheckAndMarkRunning()
{
try
{
if (fs.Exists(idPath))
{
// try appending to it so that it will fail fast if another balancer is
// running.
IOUtils.CloseStream(fs.Append(idPath));
fs.Delete(idPath, true);
}
FSDataOutputStream fsout = fs.Create(idPath, false);
// mark balancer idPath to be deleted during filesystem closure
fs.DeleteOnExit(idPath);
if (write2IdFile)
{
fsout.WriteBytes(Sharpen.Runtime.GetLocalHost().GetHostName());
fsout.Hflush();
}
return(fsout);
}
catch (RemoteException e)
{
if (typeof(AlreadyBeingCreatedException).FullName.Equals(e.GetClassName()))
{
return(null);
}
else
{
throw;
}
}
}
/// <exception cref="System.Exception"/>
public virtual void TestRmWithNonexistentGlob()
{
Configuration conf = new Configuration();
FsShell shell = new FsShell();
shell.SetConf(conf);
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
TextWriter err = new TextWriter(bytes);
TextWriter oldErr = System.Console.Error;
Runtime.SetErr(err);
string results;
try
{
int exit = shell.Run(new string[] { "-rm", "nomatch*" });
Assert.Equal(1, exit);
results = bytes.ToString();
Assert.True(results.Contains("rm: `nomatch*': No such file or directory"
));
}
finally
{
IOUtils.CloseStream(err);
Runtime.SetErr(oldErr);
}
}
private static Credentials ConvertCredentialsFromByteBuffer(ByteBuffer appAttemptTokens
)
{
DataInputByteBuffer dibb = new DataInputByteBuffer();
try
{
Credentials credentials = null;
if (appAttemptTokens != null)
{
credentials = new Credentials();
appAttemptTokens.Rewind();
dibb.Reset(appAttemptTokens);
credentials.ReadTokenStorageStream(dibb);
}
return(credentials);
}
catch (IOException)
{
Log.Error("Failed to convert Credentials from ByteBuffer.");
System.Diagnostics.Debug.Assert(false);
return(null);
}
finally
{
IOUtils.CloseStream(dibb);
}
}
public virtual void TestOpenFileTwice()
{
Describe("verify that two opened file streams are independent");
Path path = Path("testopenfiletwice.txt");
byte[] block = ContractTestUtils.Dataset(TestFileLen, 0, 255);
//this file now has a simple rule: offset => value
ContractTestUtils.CreateFile(GetFileSystem(), path, false, block);
//open first
FSDataInputStream instream1 = GetFileSystem().Open(path);
int c = instream1.Read();
Assert.Equal(0, c);
FSDataInputStream instream2 = null;
try
{
instream2 = GetFileSystem().Open(path);
Assert.Equal("first read of instream 2", 0, instream2.Read());
Assert.Equal("second read of instream 1", 1, instream1.Read());
instream1.Close();
Assert.Equal("second read of instream 2", 1, instream2.Read());
//close instream1 again
instream1.Close();
}
finally
{
IOUtils.CloseStream(instream1);
IOUtils.CloseStream(instream2);
}
}
protected internal VersionInfo(string component)
{
info = new Properties();
string versionInfoFile = component + "-version-info.properties";
InputStream @is = null;
try
{
@is = Thread.CurrentThread().GetContextClassLoader().GetResourceAsStream(
versionInfoFile);
if (@is == null)
{
throw new IOException("Resource not found");
}
info.Load(@is);
}
catch (IOException ex)
{
LogFactory.GetLog(GetType()).Warn("Could not read '" + versionInfoFile + "', " +
ex.ToString(), ex);
}
finally
{
IOUtils.CloseStream(@is);
}
}
public virtual void TestAppendWithPipelineRecovery()
{
Configuration conf = new Configuration();
MiniDFSCluster cluster = null;
FSDataOutputStream @out = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).ManageDataDfsDirs(true).ManageNameDfsDirs
(true).NumDataNodes(4).Racks(new string[] { "/rack1", "/rack1", "/rack2", "/rack2" }).Build();
cluster.WaitActive();
DistributedFileSystem fs = cluster.GetFileSystem();
Path path = new Path("/test1");
@out = fs.Create(path, true, BlockSize, (short)3, BlockSize);
AppendTestUtil.Write(@out, 0, 1024);
@out.Close();
cluster.StopDataNode(3);
@out = fs.Append(path);
AppendTestUtil.Write(@out, 1024, 1024);
@out.Close();
cluster.RestartNameNode(true);
AppendTestUtil.Check(fs, path, 2048);
}
finally
{
IOUtils.CloseStream(@out);
if (null != cluster)
{
cluster.Shutdown();
}
}
}
/// <summary>
/// Used on Windows to determine if the specified file is a symlink that
/// targets a directory.
/// </summary>
/// <remarks>
/// Used on Windows to determine if the specified file is a symlink that
/// targets a directory. On most platforms, these checks can be done using
/// commons-io. On Windows, the commons-io implementation is unreliable and
/// always returns false. Instead, this method checks the output of the dir
/// command. After migrating to Java 7, this method can be removed in favor
/// of the new method java.nio.file.Files.isSymbolicLink, which is expected to
/// work cross-platform.
/// </remarks>
/// <param name="file">File to check</param>
/// <returns>boolean true if the file is a symlink that targets a directory</returns>
/// <exception cref="System.IO.IOException">thrown for any I/O error</exception>
private static bool IsWindowsSymlinkedDirectory(FilePath file)
{
string dirOut = Shell.ExecCommand("cmd", "/c", "dir", file.GetAbsoluteFile().GetParent
());
StringReader sr = new StringReader(dirOut);
BufferedReader br = new BufferedReader(sr);
try
{
string line = br.ReadLine();
while (line != null)
{
line = br.ReadLine();
if (line.Contains(file.GetName()) && line.Contains("<SYMLINKD>"))
{
return(true);
}
}
return(false);
}
finally
{
IOUtils.CloseStream(br);
IOUtils.CloseStream(sr);
}
}
public virtual void TestBlockReportsWhileFileBeingWritten()
{
FSDataOutputStream @out = fs.Create(TestFilePath);
try
{
AppendTestUtil.Write(@out, 0, 10);
@out.Hflush();
// Block report will include the RBW replica, but will be
// queued on the StandbyNode.
cluster.TriggerBlockReports();
}
finally
{
IOUtils.CloseStream(@out);
}
cluster.TransitionToStandby(0);
cluster.TransitionToActive(1);
// Verify that no replicas are marked corrupt, and that the
// file is readable from the failed-over standby.
BlockManagerTestUtil.UpdateState(nn1.GetNamesystem().GetBlockManager());
BlockManagerTestUtil.UpdateState(nn2.GetNamesystem().GetBlockManager());
NUnit.Framework.Assert.AreEqual(0, nn1.GetNamesystem().GetCorruptReplicaBlocks());
NUnit.Framework.Assert.AreEqual(0, nn2.GetNamesystem().GetCorruptReplicaBlocks());
DFSTestUtil.ReadFile(fs, TestFilePath);
}
/// <exception cref="System.IO.IOException"/>
public static int WriteCompressedByteArray(BinaryWriter writer, byte[] bytes)
{
if (bytes != null)
{
ByteArrayOutputStream bos = new ByteArrayOutputStream();
GZIPOutputStream gzout = new GZIPOutputStream(bos);
try
{
gzout.Write(bytes, 0, bytes.Length);
gzout.Close();
gzout = null;
}
finally
{
IOUtils.CloseStream(gzout);
}
byte[] buffer = bos.ToByteArray();
int len = buffer.Length;
@out.WriteInt(len);
@out.Write(buffer, 0, len);
/* debug only! Once we have confidence, can lose this. */
return((bytes.Length != 0) ? (100 * buffer.Length) / bytes.Length : 0);
}
else
{
@out.WriteInt(-1);
return(-1);
}
}
/// <summary>
/// Test for the case where the client beings to read a long block, but doesn't
/// read bytes off the stream quickly.
/// </summary>
/// <remarks>
/// Test for the case where the client beings to read a long block, but doesn't
/// read bytes off the stream quickly. The datanode should time out sending the
/// chunks and the transceiver should die, even if it has a long keepalive.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestSlowReader()
{
// Set a client socket cache expiry time much longer than
// the datanode-side expiration time.
long ClientExpiryMs = 600000L;
Configuration clientConf = new Configuration(conf);
clientConf.SetLong(DFSConfigKeys.DfsClientSocketCacheExpiryMsecKey, ClientExpiryMs
);
clientConf.Set(DFSConfigKeys.DfsClientContext, "testSlowReader");
DistributedFileSystem fs = (DistributedFileSystem)FileSystem.Get(cluster.GetURI()
, clientConf);
// Restart the DN with a shorter write timeout.
MiniDFSCluster.DataNodeProperties props = cluster.StopDataNode(0);
props.conf.SetInt(DFSConfigKeys.DfsDatanodeSocketWriteTimeoutKey, WriteTimeout);
props.conf.SetInt(DFSConfigKeys.DfsDatanodeSocketReuseKeepaliveKey, 120000);
NUnit.Framework.Assert.IsTrue(cluster.RestartDataNode(props, true));
dn = cluster.GetDataNodes()[0];
// Wait for heartbeats to avoid a startup race where we
// try to write the block while the DN is still starting.
cluster.TriggerHeartbeats();
DFSTestUtil.CreateFile(fs, TestFile, 1024 * 1024 * 8L, (short)1, 0L);
FSDataInputStream stm = fs.Open(TestFile);
stm.Read();
AssertXceiverCount(1);
GenericTestUtils.WaitFor(new _Supplier_193(this), 500, 50000);
// DN should time out in sendChunks, and this should force
// the xceiver to exit.
IOUtils.CloseStream(stm);
}
// Set up fault injection mock.
/// <summary>
/// Run through the creation of a log without any faults injected,
/// and count how many RPCs are made to each node.
/// </summary>
/// <remarks>
/// Run through the creation of a log without any faults injected,
/// and count how many RPCs are made to each node. This sets the
/// bounds for the other test cases, so they can exhaustively explore
/// the space of potential failures.
/// </remarks>
/// <exception cref="System.Exception"/>
private static long DetermineMaxIpcNumber()
{
Configuration conf = new Configuration();
MiniJournalCluster cluster = new MiniJournalCluster.Builder(conf).Build();
QuorumJournalManager qjm = null;
long ret;
try
{
qjm = CreateInjectableQJM(cluster);
qjm.Format(QJMTestUtil.FakeNsinfo);
DoWorkload(cluster, qjm);
ICollection <int> ipcCounts = Sets.NewTreeSet();
foreach (AsyncLogger l in qjm.GetLoggerSetForTests().GetLoggersForTests())
{
TestQJMWithFaults.InvocationCountingChannel ch = (TestQJMWithFaults.InvocationCountingChannel
)l;
ch.WaitForAllPendingCalls();
ipcCounts.AddItem(ch.GetRpcCount());
}
// All of the loggers should have sent the same number of RPCs, since there
// were no failures.
NUnit.Framework.Assert.AreEqual(1, ipcCounts.Count);
ret = ipcCounts.First();
Log.Info("Max IPC count = " + ret);
}
finally
{
IOUtils.CloseStream(qjm);
cluster.Shutdown();
}
return(ret);
}
internal QueueConfigurationParser(string confFile, bool areAclsEnabled)
{
//Default root.
//xml tags for mapred-queues.xml
// The value read from queues config file for this tag is not used at all.
// To enable queue acls and job acls, mapreduce.cluster.acls.enabled is
// to be set in mapred-site.xml
aclsEnabled = areAclsEnabled;
FilePath file = new FilePath(confFile).GetAbsoluteFile();
if (!file.Exists())
{
throw new RuntimeException("Configuration file not found at " + confFile);
}
InputStream @in = null;
try
{
@in = new BufferedInputStream(new FileInputStream(file));
LoadFrom(@in);
}
catch (IOException ioe)
{
throw new RuntimeException(ioe);
}
finally
{
IOUtils.CloseStream(@in);
}
}
/// <summary>
/// Corrupt the given VERSION file by replacing a given
/// key with a new value and re-writing the file.
/// </summary>
/// <param name="versionFile">the VERSION file to corrupt</param>
/// <param name="key">the key to replace</param>
/// <param name="value">the new value for this key</param>
/// <exception cref="System.IO.IOException"/>
public static void CorruptVersionFile(FilePath versionFile, string key, string value
)
{
Properties props = new Properties();
FileInputStream fis = new FileInputStream(versionFile);
FileOutputStream @out = null;
try
{
props.Load(fis);
IOUtils.CloseStream(fis);
if (value == null || value.IsEmpty())
{
props.Remove(key);
}
else
{
props.SetProperty(key, value);
}
@out = new FileOutputStream(versionFile);
props.Store(@out, null);
}
finally
{
IOUtils.Cleanup(null, fis, @out);
}
}
private static ByteBuffer ConvertCredentialsToByteBuffer(Credentials credentials)
{
ByteBuffer appAttemptTokens = null;
DataOutputBuffer dob = new DataOutputBuffer();
try
{
if (credentials != null)
{
credentials.WriteTokenStorageToStream(dob);
appAttemptTokens = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength());
}
return(appAttemptTokens);
}
catch (IOException)
{
Log.Error("Failed to convert Credentials to ByteBuffer.");
System.Diagnostics.Debug.Assert(false);
return(null);
}
finally
{
IOUtils.CloseStream(dob);
}
}
/// <summary>
/// Calculate the md5sum of an image after zeroing out the transaction ID
/// field in the header.
/// </summary>
/// <remarks>
/// Calculate the md5sum of an image after zeroing out the transaction ID
/// field in the header. This is useful for tests that want to verify
/// that two checkpoints have identical namespaces.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
public static string GetImageFileMD5IgnoringTxId(FilePath imageFile)
{
FilePath tmpFile = FilePath.CreateTempFile("hadoop_imagefile_tmp", "fsimage");
tmpFile.DeleteOnExit();
try
{
Files.Copy(imageFile, tmpFile);
RandomAccessFile raf = new RandomAccessFile(tmpFile, "rw");
try
{
raf.Seek(ImageTxidPos);
raf.WriteLong(0);
}
finally
{
IOUtils.CloseStream(raf);
}
return(GetFileMD5(tmpFile));
}
finally
{
tmpFile.Delete();
}
}
/// <summary>Assert that a set of properties files all contain the same data.</summary>
/// <remarks>
/// Assert that a set of properties files all contain the same data.
/// We cannot simply check the md5sums here, since Properties files
/// contain timestamps -- thus, two properties files from the same
/// saveNamespace operation may actually differ in md5sum.
/// </remarks>
/// <param name="propFiles">the files to compare</param>
/// <exception cref="System.IO.IOException">if the files cannot be opened or read</exception>
/// <exception cref="System.Exception">if the files differ</exception>
public static void AssertPropertiesFilesSame(FilePath[] propFiles)
{
ICollection <KeyValuePair <object, object> > prevProps = null;
foreach (FilePath f in propFiles)
{
Properties props;
FileInputStream @is = new FileInputStream(f);
try
{
props = new Properties();
props.Load(@is);
}
finally
{
IOUtils.CloseStream(@is);
}
if (prevProps == null)
{
prevProps = props;
}
else
{
ICollection <KeyValuePair <object, object> > diff = Sets.SymmetricDifference(prevProps
, props);
if (!diff.IsEmpty())
{
NUnit.Framework.Assert.Fail("Properties file " + f + " differs from " + propFiles
[0]);
}
}
}
}
public override void Run()
{
FSDataOutputStream @out = null;
int i = 0;
try
{
@out = fs.Create(filepath);
for (; running; i++)
{
System.Console.Out.WriteLine(GetName() + " writes " + i);
@out.Write(i);
@out.Hflush();
Sleep(100);
}
}
catch (Exception e)
{
System.Console.Out.WriteLine(GetName() + " dies: e=" + e);
}
finally
{
System.Console.Out.WriteLine(GetName() + ": i=" + i);
IOUtils.CloseStream(@out);
}
}
/// <exception cref="Javax.Servlet.ServletException"/>
/// <exception cref="System.IO.IOException"/>
protected override void DoGet(HttpServletRequest request, HttpServletResponse response
)
{
FileInputStream editFileIn = null;
try
{
ServletContext context = GetServletContext();
Configuration conf = (Configuration)GetServletContext().GetAttribute(JspHelper.CurrentConf
);
string journalId = request.GetParameter(JournalIdParam);
QuorumJournalManager.CheckJournalId(journalId);
JNStorage storage = JournalNodeHttpServer.GetJournalFromContext(context, journalId
).GetStorage();
// Check security
if (!CheckRequestorOrSendError(conf, request, response))
{
return;
}
// Check that the namespace info is correct
if (!CheckStorageInfoOrSendError(storage, request, response))
{
return;
}
long segmentTxId = ServletUtil.ParseLongParam(request, SegmentTxidParam);
FileJournalManager fjm = storage.GetJournalManager();
FilePath editFile;
lock (fjm)
{
// Synchronize on the FJM so that the file doesn't get finalized
// out from underneath us while we're in the process of opening
// it up.
FileJournalManager.EditLogFile elf = fjm.GetLogFile(segmentTxId);
if (elf == null)
{
response.SendError(HttpServletResponse.ScNotFound, "No edit log found starting at txid "
+ segmentTxId);
return;
}
editFile = elf.GetFile();
ImageServlet.SetVerificationHeadersForGet(response, editFile);
ImageServlet.SetFileNameHeaders(response, editFile);
editFileIn = new FileInputStream(editFile);
}
DataTransferThrottler throttler = ImageServlet.GetThrottler(conf);
// send edits
TransferFsImage.CopyFileToStream(response.GetOutputStream(), editFile, editFileIn
, throttler);
}
catch (Exception t)
{
string errMsg = "getedit failed. " + StringUtils.StringifyException(t);
response.SendError(HttpServletResponse.ScInternalServerError, errMsg);
throw new IOException(errMsg);
}
finally
{
IOUtils.CloseStream(editFileIn);
}
}
public virtual void TestCreatedFileIsImmediatelyVisible()
{
Describe("verify that a newly created file exists as soon as open returns");
Path path = Path("testCreatedFileIsImmediatelyVisible");
FSDataOutputStream @out = null;
try
{
@out = GetFileSystem().Create(path, false, 4096, (short)1, 1024);
if (!GetFileSystem().Exists(path))
{
if (IsSupported(IsBlobstore))
{
// object store: downgrade to a skip so that the failure is visible
// in test results
ContractTestUtils.Skip("Filesystem is an object store and newly created files are not immediately visible"
);
}
AssertPathExists("expected path to be visible before anything written", path);
}
}
finally
{
IOUtils.CloseStream(@out);
}
}
public virtual void TestGetSet()
{
BestEffortLongFile f = new BestEffortLongFile(File, 12345L);
try
{
// Before the file exists, should return default.
NUnit.Framework.Assert.AreEqual(12345L, f.Get());
// And first access should open it.
NUnit.Framework.Assert.IsTrue(File.Exists());
Random r = new Random();
for (int i = 0; i < 100; i++)
{
long newVal = r.NextLong();
// Changing the value should be reflected in the next get() call.
f.Set(newVal);
NUnit.Framework.Assert.AreEqual(newVal, f.Get());
// And should be reflected in a new instance (ie it actually got
// written to the file)
BestEffortLongFile f2 = new BestEffortLongFile(File, 999L);
try
{
NUnit.Framework.Assert.AreEqual(newVal, f2.Get());
}
finally
{
IOUtils.CloseStream(f2);
}
}
}
finally
{
IOUtils.CloseStream(f);
}
}
/// <exception cref="System.IO.IOException"/>
private void WriteToHostsFile(params string[] hosts)
{
if (!hostFile.Exists())
{
TempDir.Mkdirs();
hostFile.CreateNewFile();
}
FileOutputStream fStream = null;
try
{
fStream = new FileOutputStream(hostFile);
for (int i = 0; i < hosts.Length; i++)
{
fStream.Write(Sharpen.Runtime.GetBytesForString(hosts[i]));
fStream.Write(Sharpen.Runtime.GetBytesForString("\n"));
}
}
finally
{
if (fStream != null)
{
IOUtils.CloseStream(fStream);
fStream = null;
}
}
}
public static void TearDown()
{
IOUtils.CloseStream(dfs);
if (cluster != null)
{
cluster.Shutdown();
}
}
public virtual void TestRestartDfs()
{
Configuration conf = new HdfsConfiguration();
// Turn off persistent IPC, so that the DFSClient can survive NN restart
conf.SetInt(CommonConfigurationKeysPublic.IpcClientConnectionMaxidletimeKey, 0);
MiniDFSCluster cluster = null;
long len = 0;
FSDataOutputStream stream;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build();
FileSystem fs = cluster.GetFileSystem();
// Creating a file with 4096 blockSize to write multiple blocks
stream = fs.Create(FilePath, true, BlockSize, (short)1, BlockSize);
stream.Write(DataBeforeRestart);
stream.Hflush();
// Wait for at least a few blocks to get through
while (len <= BlockSize)
{
FileStatus status = fs.GetFileStatus(FilePath);
len = status.GetLen();
Sharpen.Thread.Sleep(100);
}
// explicitly do NOT close the file.
cluster.RestartNameNode();
// Check that the file has no less bytes than before the restart
// This would mean that blocks were successfully persisted to the log
FileStatus status_1 = fs.GetFileStatus(FilePath);
NUnit.Framework.Assert.IsTrue("Length too short: " + status_1.GetLen(), status_1.
GetLen() >= len);
// And keep writing (ensures that leases are also persisted correctly)
stream.Write(DataAfterRestart);
stream.Close();
// Verify that the data showed up, both from before and after the restart.
FSDataInputStream readStream = fs.Open(FilePath);
try
{
byte[] verifyBuf = new byte[DataBeforeRestart.Length];
IOUtils.ReadFully(readStream, verifyBuf, 0, verifyBuf.Length);
Assert.AssertArrayEquals(DataBeforeRestart, verifyBuf);
IOUtils.ReadFully(readStream, verifyBuf, 0, verifyBuf.Length);
Assert.AssertArrayEquals(DataAfterRestart, verifyBuf);
}
finally
{
IOUtils.CloseStream(readStream);
}
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <exception cref="System.Exception"/>
private void DoWriteOverFailoverTest(TestPipelinesFailover.TestScenario scenario,
TestPipelinesFailover.MethodToTestIdempotence methodToTest)
{
Configuration conf = new Configuration();
conf.SetInt(DFSConfigKeys.DfsBlockSizeKey, BlockSize);
// Don't check replication periodically.
conf.SetInt(DFSConfigKeys.DfsNamenodeReplicationIntervalKey, 1000);
FSDataOutputStream stm = null;
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology
.SimpleHATopology()).NumDataNodes(3).Build();
try
{
int sizeWritten = 0;
cluster.WaitActive();
cluster.TransitionToActive(0);
Sharpen.Thread.Sleep(500);
Log.Info("Starting with NN 0 active");
FileSystem fs = HATestUtil.ConfigureFailoverFs(cluster, conf);
stm = fs.Create(TestPath);
// write a block and a half
AppendTestUtil.Write(stm, 0, BlockAndAHalf);
sizeWritten += BlockAndAHalf;
// Make sure all of the blocks are written out before failover.
stm.Hflush();
Log.Info("Failing over to NN 1");
scenario.Run(cluster);
// NOTE: explicitly do *not* make any further metadata calls
// to the NN here. The next IPC call should be to allocate the next
// block. Any other call would notice the failover and not test
// idempotence of the operation (HDFS-3031)
FSNamesystem ns1 = cluster.GetNameNode(1).GetNamesystem();
BlockManagerTestUtil.UpdateState(ns1.GetBlockManager());
NUnit.Framework.Assert.AreEqual(0, ns1.GetPendingReplicationBlocks());
NUnit.Framework.Assert.AreEqual(0, ns1.GetCorruptReplicaBlocks());
NUnit.Framework.Assert.AreEqual(0, ns1.GetMissingBlocksCount());
// If we're testing allocateBlock()'s idempotence, write another
// block and a half, so we have to allocate a new block.
// Otherise, don't write anything, so our next RPC will be
// completeFile() if we're testing idempotence of that operation.
if (methodToTest == TestPipelinesFailover.MethodToTestIdempotence.AllocateBlock)
{
// write another block and a half
AppendTestUtil.Write(stm, sizeWritten, BlockAndAHalf);
sizeWritten += BlockAndAHalf;
}
stm.Close();
stm = null;
AppendTestUtil.Check(fs, TestPath, sizeWritten);
}
finally
{
IOUtils.CloseStream(stm);
cluster.Shutdown();
}
}
/// <summary>Retrieves the number of links to the specified file.</summary>
/// <exception cref="System.IO.IOException"/>
public static int GetLinkCount(FilePath fileName)
{
if (fileName == null)
{
throw new IOException("invalid argument to getLinkCount: file name is null");
}
if (!fileName.Exists())
{
throw new FileNotFoundException(fileName + " not found.");
}
// construct and execute shell command
string[] cmd = getHardLinkCommand.LinkCount(fileName);
string inpMsg = null;
string errMsg = null;
int exitValue = -1;
BufferedReader @in = null;
Shell.ShellCommandExecutor shexec = new Shell.ShellCommandExecutor(cmd);
try
{
shexec.Execute();
@in = new BufferedReader(new StringReader(shexec.GetOutput()));
inpMsg = @in.ReadLine();
exitValue = shexec.GetExitCode();
if (inpMsg == null || exitValue != 0)
{
throw CreateIOException(fileName, inpMsg, errMsg, exitValue, null);
}
if (Shell.Solaris)
{
string[] result = inpMsg.Split("\\s+");
return(System.Convert.ToInt32(result[1]));
}
else
{
return(System.Convert.ToInt32(inpMsg));
}
}
catch (Shell.ExitCodeException e)
{
inpMsg = shexec.GetOutput();
errMsg = e.Message;
exitValue = e.GetExitCode();
throw CreateIOException(fileName, inpMsg, errMsg, exitValue, e);
}
catch (FormatException e)
{
throw CreateIOException(fileName, inpMsg, errMsg, exitValue, e);
}
finally
{
IOUtils.CloseStream(@in);
}
}
/// <exception cref="System.IO.IOException"/>
internal static FSEditLogLoader.EditLogValidation ScanEditLog(FilePath file)
{
Org.Apache.Hadoop.Hdfs.Server.Namenode.EditLogFileInputStream @in;
try
{
@in = new Org.Apache.Hadoop.Hdfs.Server.Namenode.EditLogFileInputStream(file);
// read the header, initialize the inputstream, but do not check the
// layoutversion
@in.GetVersion(false);
}
catch (EditLogFileInputStream.LogHeaderCorruptException e)
{
Log.Warn("Log file " + file + " has no valid header", e);
return(new FSEditLogLoader.EditLogValidation(0, HdfsConstants.InvalidTxid, true));
}
long lastPos = 0;
long lastTxId = HdfsConstants.InvalidTxid;
long numValid = 0;
try
{
while (true)
{
long txid = HdfsConstants.InvalidTxid;
lastPos = @in.GetPosition();
try
{
if ((txid = @in.ScanNextOp()) == HdfsConstants.InvalidTxid)
{
break;
}
}
catch (Exception t)
{
FSImage.Log.Warn("Caught exception after scanning through " + numValid + " ops from "
+ @in + " while determining its valid length. Position was " + lastPos, t);
@in.Resync();
FSImage.Log.Warn("After resync, position is " + @in.GetPosition());
continue;
}
if (lastTxId == HdfsConstants.InvalidTxid || txid > lastTxId)
{
lastTxId = txid;
}
numValid++;
}
return(new FSEditLogLoader.EditLogValidation(lastPos, lastTxId, false));
}
finally
{
IOUtils.CloseStream(@in);
}
}