// expected
/// <exception cref="System.Exception"/>
public virtual void TestSyncFileRange()
{
FileOutputStream fos = new FileOutputStream(new FilePath(TestDir, "testSyncFileRange"
));
try
{
fos.Write(Runtime.GetBytesForString("foo"));
NativeIO.POSIX.Sync_file_range(fos.GetFD(), 0, 1024, NativeIO.POSIX.SyncFileRangeWrite
);
}
catch (NotSupportedException)
{
// no way to verify that this actually has synced,
// but if it doesn't throw, we can assume it worked
// we should just skip the unit test on machines where we don't
// have fadvise support
Assume.AssumeTrue(false);
}
finally
{
fos.Close();
}
try
{
NativeIO.POSIX.Sync_file_range(fos.GetFD(), 0, 1024, NativeIO.POSIX.SyncFileRangeWrite
);
NUnit.Framework.Assert.Fail("Did not throw on bad file");
}
catch (NativeIOException nioe)
{
Assert.Equal(Errno.Ebadf, nioe.GetErrno());
}
}
// Expect NoSuchPaddingException
/// <exception cref="System.Exception"/>
public virtual void TestUpdateArguments()
{
Assume.AssumeTrue(OpensslCipher.GetLoadingFailureReason() == null);
OpensslCipher cipher = OpensslCipher.GetInstance("AES/CTR/NoPadding");
Assert.True(cipher != null);
cipher.Init(OpensslCipher.EncryptMode, key, iv);
// Require direct buffers
ByteBuffer input = ByteBuffer.Allocate(1024);
ByteBuffer output = ByteBuffer.Allocate(1024);
try
{
cipher.Update(input, output);
NUnit.Framework.Assert.Fail("Input and output buffer should be direct buffer.");
}
catch (ArgumentException e)
{
GenericTestUtils.AssertExceptionContains("Direct buffers are required", e);
}
// Output buffer length should be sufficient to store output data
input = ByteBuffer.AllocateDirect(1024);
output = ByteBuffer.AllocateDirect(1000);
try
{
cipher.Update(input, output);
NUnit.Framework.Assert.Fail("Output buffer length should be sufficient " + "to store output data"
);
}
catch (ShortBufferException e)
{
GenericTestUtils.AssertExceptionContains("Output buffer is not sufficient", e);
}
}
public virtual void TestShortCircuitTraceHooks()
{
Assume.AssumeTrue(NativeCodeLoader.IsNativeCodeLoaded() && !Path.Windows);
conf = new Configuration();
conf.Set(DFSConfigKeys.DfsClientHtracePrefix + SpanReceiverHost.SpanReceiversConfSuffix
, typeof(TestTracing.SetSpanReceiver).FullName);
conf.SetLong("dfs.blocksize", 100 * 1024);
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, true);
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, false);
conf.Set(DFSConfigKeys.DfsDomainSocketPathKey, "testShortCircuitTraceHooks._PORT"
);
conf.Set(DFSConfigKeys.DfsChecksumTypeKey, "CRC32C");
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
dfs = cluster.GetFileSystem();
try
{
DFSTestUtil.CreateFile(dfs, TestPath, TestLength, (short)1, 5678L);
TraceScope ts = Trace.StartSpan("testShortCircuitTraceHooks", Sampler.Always);
FSDataInputStream stream = dfs.Open(TestPath);
byte[] buf = new byte[TestLength];
IOUtils.ReadFully(stream, buf, 0, TestLength);
stream.Close();
ts.Close();
string[] expectedSpanNames = new string[] { "OpRequestShortCircuitAccessProto", "ShortCircuitShmRequestProto" };
TestTracing.AssertSpanNamesFound(expectedSpanNames);
}
finally
{
dfs.Close();
cluster.Shutdown();
}
}
public virtual void TestFailToRename()
{
Assume.AssumeTrue(Shell.Windows);
OutputStream fos = null;
try
{
fos = new AtomicFileOutputStream(DstFile);
fos.Write(Sharpen.Runtime.GetBytesForString(TestString));
FileUtil.SetWritable(TestDir, false);
exception.Expect(typeof(IOException));
exception.ExpectMessage("failure in native rename");
try
{
fos.Close();
}
finally
{
fos = null;
}
}
finally
{
IOUtils.Cleanup(null, fos);
FileUtil.SetWritable(TestDir, true);
}
}
public virtual void TestBothOldAndNewShortCircuitConfigured()
{
short ReplFactor = 1;
int FileLength = 512;
Assume.AssumeTrue(null == DomainSocket.GetLoadingFailureReason());
TemporarySocketDirectory socketDir = new TemporarySocketDirectory();
HdfsConfiguration conf = GetConfiguration(socketDir);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
socketDir.Close();
FileSystem fs = cluster.GetFileSystem();
Path path = new Path("/foo");
byte[] orig = new byte[FileLength];
for (int i = 0; i < orig.Length; i++)
{
orig[i] = unchecked ((byte)(i % 10));
}
FSDataOutputStream fos = fs.Create(path, (short)1);
fos.Write(orig);
fos.Close();
DFSTestUtil.WaitReplication(fs, path, ReplFactor);
FSDataInputStream fis = cluster.GetFileSystem().Open(path);
byte[] buf = new byte[FileLength];
IOUtils.ReadFully(fis, buf, 0, FileLength);
fis.Close();
Assert.AssertArrayEquals(orig, buf);
Arrays.Equals(orig, buf);
cluster.Shutdown();
}
/// <exception cref="System.Exception"/>
public virtual void TestJournalLocking()
{
Assume.AssumeTrue(journal.GetStorage().GetStorageDir(0).IsLockSupported());
Storage.StorageDirectory sd = journal.GetStorage().GetStorageDir(0);
FilePath lockFile = new FilePath(sd.GetRoot(), Storage.StorageFileLock);
// Journal should be locked, since the format() call locks it.
GenericTestUtils.AssertExists(lockFile);
journal.NewEpoch(FakeNsinfo, 1);
try
{
new Journal(conf, TestLogDir, Jid, HdfsServerConstants.StartupOption.Regular, mockErrorReporter
);
NUnit.Framework.Assert.Fail("Did not fail to create another journal in same dir");
}
catch (IOException ioe)
{
GenericTestUtils.AssertExceptionContains("Cannot lock storage", ioe);
}
journal.Close();
// Journal should no longer be locked after the close() call.
// Hence, should be able to create a new Journal in the same dir.
Journal journal2 = new Journal(conf, TestLogDir, Jid, HdfsServerConstants.StartupOption
.Regular, mockErrorReporter);
journal2.NewEpoch(FakeNsinfo, 2);
journal2.Close();
}
public virtual void TestDuplicates()
{
Assume.AssumeTrue(!Shell.Windows);
string GetAllUsersCmd = "echo \"root:x:0:0:root:/root:/bin/bash\n" + "hdfs:x:11501:10787:Grid Distributed File System:/home/hdfs:/bin/bash\n"
+ "hdfs:x:11502:10788:Grid Distributed File System:/home/hdfs:/bin/bash\n" + "hdfs1:x:11501:10787:Grid Distributed File System:/home/hdfs:/bin/bash\n"
+ "hdfs2:x:11502:10787:Grid Distributed File System:/home/hdfs:/bin/bash\n" + "bin:x:2:2:bin:/bin:/bin/sh\n"
+ "bin:x:1:1:bin:/bin:/sbin/nologin\n" + "daemon:x:1:1:daemon:/usr/sbin:/bin/sh\n"
+ "daemon:x:2:2:daemon:/sbin:/sbin/nologin\"" + " | cut -d: -f1,3";
string GetAllGroupsCmd = "echo \"hdfs:*:11501:hrt_hdfs\n" + "mapred:x:497\n" + "mapred2:x:497\n"
+ "mapred:x:498\n" + "mapred3:x:498\"" + " | cut -d: -f1,3";
// Maps for id to name map
BiMap <int, string> uMap = HashBiMap.Create();
BiMap <int, string> gMap = HashBiMap.Create();
ShellBasedIdMapping.UpdateMapInternal(uMap, "user", GetAllUsersCmd, ":", EmptyPassThroughMap
);
Assert.Equal(5, uMap.Count);
Assert.Equal("root", uMap[0]);
Assert.Equal("hdfs", uMap[11501]);
Assert.Equal("hdfs2", uMap[11502]);
Assert.Equal("bin", uMap[2]);
Assert.Equal("daemon", uMap[1]);
ShellBasedIdMapping.UpdateMapInternal(gMap, "group", GetAllGroupsCmd, ":", EmptyPassThroughMap
);
Assert.True(gMap.Count == 3);
Assert.Equal("hdfs", gMap[11501]);
Assert.Equal("mapred", gMap[497]);
Assert.Equal("mapred3", gMap[498]);
}
public virtual void TestStaticMapping()
{
Assume.AssumeTrue(!Shell.Windows);
IDictionary <int, int> uidStaticMap = new ShellBasedIdMapping.PassThroughMap <int>(
);
IDictionary <int, int> gidStaticMap = new ShellBasedIdMapping.PassThroughMap <int>(
);
uidStaticMap[11501] = 10;
gidStaticMap[497] = 200;
// Maps for id to name map
BiMap <int, string> uMap = HashBiMap.Create();
BiMap <int, string> gMap = HashBiMap.Create();
string GetAllUsersCmd = "echo \"atm:x:1000:1000:Aaron T. Myers,,,:/home/atm:/bin/bash\n"
+ "hdfs:x:11501:10787:Grid Distributed File System:/home/hdfs:/bin/bash\"" + " | cut -d: -f1,3";
string GetAllGroupsCmd = "echo \"hdfs:*:11501:hrt_hdfs\n" + "mapred:x:497\n" + "mapred2:x:498\""
+ " | cut -d: -f1,3";
ShellBasedIdMapping.UpdateMapInternal(uMap, "user", GetAllUsersCmd, ":", uidStaticMap
);
ShellBasedIdMapping.UpdateMapInternal(gMap, "group", GetAllGroupsCmd, ":", gidStaticMap
);
Assert.Equal("hdfs", uMap[10]);
Assert.Equal(10, (int)uMap.Inverse()["hdfs"]);
Assert.Equal("atm", uMap[1000]);
Assert.Equal(1000, (int)uMap.Inverse()["atm"]);
Assert.Equal("hdfs", gMap[11501]);
Assert.Equal(11501, (int)gMap.Inverse()["hdfs"]);
Assert.Equal("mapred", gMap[200]);
Assert.Equal(200, (int)gMap.Inverse()["mapred"]);
Assert.Equal("mapred2", gMap[498]);
Assert.Equal(498, (int)gMap.Inverse()["mapred2"]);
}
public virtual void TestElementwiseProduct(ConcatVectorTest.DenseTestVector d1, ConcatVectorTest.DenseTestVector d2)
{
for (int i = 0; i < d1.values.Length; i++)
{
for (int j = 0; j < d1.values[i].Length; j++)
{
Assume.AssumeTrue(d1.values[i][j] == d1.vector.GetValueAt(i, j));
}
}
for (int i_1 = 0; i_1 < d2.values.Length; i_1++)
{
for (int j = 0; j < d2.values[i_1].Length; j++)
{
Assume.AssumeTrue(d2.values[i_1][j] == d2.vector.GetValueAt(i_1, j));
}
}
ConcatVector clone = d1.vector.DeepClone();
clone.ElementwiseProductInPlace(d2.vector);
for (int i_2 = 0; i_2 < d1.values.Length; i_2++)
{
for (int j = 0; j < d1.values[i_2].Length; j++)
{
double val = 0.0f;
if (i_2 < d2.values.Length)
{
if (j < d2.values[i_2].Length)
{
val = d1.values[i_2][j] * d2.values[i_2][j];
}
}
NUnit.Framework.Assert.AreEqual(clone.GetValueAt(i_2, j), 5.0e-4, val);
}
}
}
public virtual void TestUnderReplicationAfterVolFailure()
{
// This test relies on denying access to data volumes to simulate data volume
// failure. This doesn't work on Windows, because an owner of an object
// always has the ability to read and change permissions on the object.
Assume.AssumeTrue(!Path.Windows);
// Bring up one more datanode
cluster.StartDataNodes(conf, 1, true, null, null);
cluster.WaitActive();
BlockManager bm = cluster.GetNamesystem().GetBlockManager();
Path file1 = new Path("/test1");
DFSTestUtil.CreateFile(fs, file1, 1024, (short)3, 1L);
DFSTestUtil.WaitReplication(fs, file1, (short)3);
// Fail the first volume on both datanodes
FilePath dn1Vol1 = new FilePath(dataDir, "data" + (2 * 0 + 1));
FilePath dn2Vol1 = new FilePath(dataDir, "data" + (2 * 1 + 1));
DataNodeTestUtils.InjectDataDirFailure(dn1Vol1, dn2Vol1);
Path file2 = new Path("/test2");
DFSTestUtil.CreateFile(fs, file2, 1024, (short)3, 1L);
DFSTestUtil.WaitReplication(fs, file2, (short)3);
// underReplicatedBlocks are due to failed volumes
int underReplicatedBlocks = BlockManagerTestUtil.CheckHeartbeatAndGetUnderReplicatedBlocksCount
(cluster.GetNamesystem(), bm);
NUnit.Framework.Assert.IsTrue("There is no under replicated block after volume failure"
, underReplicatedBlocks > 0);
}
public virtual void TestGetMaxedMarginals(TableFactor factor, int marginalizeTo)
{
if (!Arrays.Stream(factor.neighborIndices).Boxed().Collect(Collectors.ToSet()).Contains(marginalizeTo))
{
return;
}
int indexOfVariable = -1;
for (int i = 0; i < factor.neighborIndices.Length; i++)
{
if (factor.neighborIndices[i] == marginalizeTo)
{
indexOfVariable = i;
break;
}
}
Assume.AssumeTrue(indexOfVariable > -1);
double[] gold = new double[factor.GetDimensions()[indexOfVariable]];
for (int i_1 = 0; i_1 < gold.Length; i_1++)
{
gold[i_1] = double.NegativeInfinity;
}
foreach (int[] assignment in factor)
{
gold[assignment[indexOfVariable]] = Math.Max(gold[assignment[indexOfVariable]], factor.GetAssignmentValue(assignment));
}
Normalize(gold);
Assert.AssertArrayEquals(factor.GetMaxedMarginals()[indexOfVariable], 1.0e-5, gold);
}
/// <summary>Tests for a given volumes to be tolerated and volumes failed.</summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
private void TestVolumeConfig(int volumesTolerated, int volumesFailed, bool expectedBPServiceState
, bool manageDfsDirs)
{
Assume.AssumeTrue(!Runtime.GetProperty("os.name").StartsWith("Windows"));
int dnIndex = 0;
// Fail the current directory since invalid storage directory perms
// get fixed up automatically on datanode startup.
FilePath[] dirs = new FilePath[] { new FilePath(cluster.GetInstanceStorageDir(dnIndex
, 0), "current"), new FilePath(cluster.GetInstanceStorageDir(dnIndex, 1), "current"
) };
try
{
for (int i = 0; i < volumesFailed; i++)
{
PrepareDirToFail(dirs[i]);
}
RestartDatanodes(volumesTolerated, manageDfsDirs);
NUnit.Framework.Assert.AreEqual(expectedBPServiceState, cluster.GetDataNodes()[0]
.IsBPServiceAlive(cluster.GetNamesystem().GetBlockPoolId()));
}
finally
{
foreach (FilePath dir in dirs)
{
FileUtil.Chmod(dir.ToString(), "755");
}
}
}
public virtual void TestConfigureMinValidVolumes()
{
Assume.AssumeTrue(!Runtime.GetProperty("os.name").StartsWith("Windows"));
// Bring up two additional datanodes that need both of their volumes
// functioning in order to stay up.
conf.SetInt(DFSConfigKeys.DfsDatanodeFailedVolumesToleratedKey, 0);
cluster.StartDataNodes(conf, 2, true, null, null);
cluster.WaitActive();
DatanodeManager dm = cluster.GetNamesystem().GetBlockManager().GetDatanodeManager
();
long origCapacity = DFSTestUtil.GetLiveDatanodeCapacity(dm);
long dnCapacity = DFSTestUtil.GetDatanodeCapacity(dm, 0);
// Fail a volume on the 2nd DN
FilePath dn2Vol1 = new FilePath(dataDir, "data" + (2 * 1 + 1));
DataNodeTestUtils.InjectDataDirFailure(dn2Vol1);
// Should only get two replicas (the first DN and the 3rd)
Path file1 = new Path("/test1");
DFSTestUtil.CreateFile(fs, file1, 1024, (short)3, 1L);
DFSTestUtil.WaitReplication(fs, file1, (short)2);
// Check that this single failure caused a DN to die.
DFSTestUtil.WaitForDatanodeStatus(dm, 2, 1, 0, origCapacity - (1 * dnCapacity), WaitForHeartbeats
);
// If we restore the volume we should still only be able to get
// two replicas since the DN is still considered dead.
DataNodeTestUtils.RestoreDataDirFromFailure(dn2Vol1);
Path file2 = new Path("/test2");
DFSTestUtil.CreateFile(fs, file2, 1024, (short)3, 1L);
DFSTestUtil.WaitReplication(fs, file2, (short)2);
}
/// <exception cref="System.Exception"/>
public virtual void TestMlock()
{
Assume.AssumeTrue(NativeIO.IsAvailable());
FilePath TestFile = new FilePath(new FilePath(Runtime.GetProperty("test.build.data"
, "build/test/data")), "testMlockFile");
int BufLen = 12289;
byte[] buf = new byte[BufLen];
int bufSum = 0;
for (int i = 0; i < buf.Length; i++)
{
buf[i] = unchecked ((byte)(i % 60));
bufSum += buf[i];
}
FileOutputStream fos = new FileOutputStream(TestFile);
try
{
fos.Write(buf);
fos.GetChannel().Force(true);
}
finally
{
fos.Close();
}
FileInputStream fis = null;
FileChannel channel = null;
try
{
// Map file into memory
fis = new FileInputStream(TestFile);
channel = fis.GetChannel();
long fileSize = channel.Size();
MappedByteBuffer mapbuf = channel.Map(FileChannel.MapMode.ReadOnly, 0, fileSize);
// mlock the buffer
NativeIO.POSIX.Mlock(mapbuf, fileSize);
// Read the buffer
int sum = 0;
for (int i_1 = 0; i_1 < fileSize; i_1++)
{
sum += mapbuf.Get(i_1);
}
Assert.Equal("Expected sums to be equal", bufSum, sum);
// munmap the buffer, which also implicitly unlocks it
NativeIO.POSIX.Munmap(mapbuf);
}
finally
{
if (channel != null)
{
channel.Close();
}
if (fis != null)
{
fis.Close();
}
}
}
public virtual void TestBlockTokenRpcLeak()
{
Configuration conf = new Configuration();
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
Assume.AssumeTrue(FdDir.Exists());
BlockTokenSecretManager sm = new BlockTokenSecretManager(blockKeyUpdateInterval,
blockTokenLifetime, 0, "fake-pool", null);
Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token = sm.GenerateToken
(block3, EnumSet.AllOf <BlockTokenSecretManager.AccessMode>());
Server server = CreateMockDatanode(sm, token, conf);
server.Start();
IPEndPoint addr = NetUtils.GetConnectAddress(server);
DatanodeID fakeDnId = DFSTestUtil.GetLocalDatanodeID(addr.Port);
ExtendedBlock b = new ExtendedBlock("fake-pool", new Org.Apache.Hadoop.Hdfs.Protocol.Block
(12345L));
LocatedBlock fakeBlock = new LocatedBlock(b, new DatanodeInfo[0]);
fakeBlock.SetBlockToken(token);
// Create another RPC proxy with the same configuration - this will never
// attempt to connect anywhere -- but it causes the refcount on the
// RPC "Client" object to stay above 0 such that RPC.stopProxy doesn't
// actually close the TCP connections to the real target DN.
ClientDatanodeProtocol proxyToNoWhere = RPC.GetProxy <ClientDatanodeProtocol>(ClientDatanodeProtocol
.versionID, new IPEndPoint("1.1.1.1", 1), UserGroupInformation.CreateRemoteUser(
"junk"), conf, NetUtils.GetDefaultSocketFactory(conf));
ClientDatanodeProtocol proxy = null;
int fdsAtStart = CountOpenFileDescriptors();
try
{
long endTime = Time.Now() + 3000;
while (Time.Now() < endTime)
{
proxy = DFSUtil.CreateClientDatanodeProtocolProxy(fakeDnId, conf, 1000, false, fakeBlock
);
NUnit.Framework.Assert.AreEqual(block3.GetBlockId(), proxy.GetReplicaVisibleLength
(block3));
if (proxy != null)
{
RPC.StopProxy(proxy);
}
Log.Info("Num open fds:" + CountOpenFileDescriptors());
}
int fdsAtEnd = CountOpenFileDescriptors();
if (fdsAtEnd - fdsAtStart > 50)
{
NUnit.Framework.Assert.Fail("Leaked " + (fdsAtEnd - fdsAtStart) + " fds!");
}
}
finally
{
server.Stop();
}
RPC.StopProxy(proxyToNoWhere);
}
/// <exception cref="System.IO.IOException"/>
private static OsSecureRandom GetOsSecureRandom()
{
Assume.AssumeTrue(SystemUtils.IsOsLinux);
OsSecureRandom random = new OsSecureRandom();
random.SetConf(new Configuration());
return(random);
}
/// <exception cref="System.Exception"/>
public virtual void TestRenameTo()
{
FilePath TestDir = new FilePath(new FilePath(Runtime.GetProperty("test.build.data"
, "build/test/data")), "renameTest");
Assume.AssumeTrue(TestDir.Mkdirs());
FilePath nonExistentFile = new FilePath(TestDir, "nonexistent");
FilePath targetFile = new FilePath(TestDir, "target");
// Test attempting to rename a nonexistent file.
try
{
NativeIO.RenameTo(nonExistentFile, targetFile);
NUnit.Framework.Assert.Fail();
}
catch (NativeIOException e)
{
if (Path.Windows)
{
Assert.Equal(string.Format("The system cannot find the file specified.%n"
), e.Message);
}
else
{
Assert.Equal(Errno.Enoent, e.GetErrno());
}
}
// Test renaming a file to itself. It should succeed and do nothing.
FilePath sourceFile = new FilePath(TestDir, "source");
Assert.True(sourceFile.CreateNewFile());
NativeIO.RenameTo(sourceFile, sourceFile);
// Test renaming a source to a destination.
NativeIO.RenameTo(sourceFile, targetFile);
// Test renaming a source to a path which uses a file as a directory.
sourceFile = new FilePath(TestDir, "source");
Assert.True(sourceFile.CreateNewFile());
FilePath badTarget = new FilePath(targetFile, "subdir");
try
{
NativeIO.RenameTo(sourceFile, badTarget);
NUnit.Framework.Assert.Fail();
}
catch (NativeIOException e)
{
if (Path.Windows)
{
Assert.Equal(string.Format("The parameter is incorrect.%n"), e
.Message);
}
else
{
Assert.Equal(Errno.Enotdir, e.GetErrno());
}
}
FileUtils.DeleteQuietly(TestDir);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Sharpen.URISyntaxException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="Sharpen.ExecutionException"/>
public virtual void TestDownloadPublicWithStatCache()
{
Configuration conf = new Configuration();
FileContext files = FileContext.GetLocalFSFileContext(conf);
Path basedir = files.MakeQualified(new Path("target", typeof(TestFSDownload).Name
));
// if test directory doesn't have ancestor permission, skip this test
FileSystem f = basedir.GetFileSystem(conf);
Assume.AssumeTrue(FSDownload.AncestorsHaveExecutePermissions(f, basedir, null));
files.Mkdir(basedir, null, true);
conf.SetStrings(typeof(TestFSDownload).FullName, basedir.ToString());
int size = 512;
ConcurrentMap <Path, AtomicInteger> counts = new ConcurrentHashMap <Path, AtomicInteger
>();
CacheLoader <Path, Future <FileStatus> > loader = FSDownload.CreateStatusCacheLoader
(conf);
LoadingCache <Path, Future <FileStatus> > statCache = CacheBuilder.NewBuilder().Build
(new _CacheLoader_328(counts, loader));
// increment the count
// use the default loader
// test FSDownload.isPublic() concurrently
int fileCount = 3;
IList <Callable <bool> > tasks = new AList <Callable <bool> >();
for (int i = 0; i < fileCount; i++)
{
Random rand = new Random();
long sharedSeed = rand.NextLong();
rand.SetSeed(sharedSeed);
System.Console.Out.WriteLine("SEED: " + sharedSeed);
Path path = new Path(basedir, "test-file-" + i);
CreateFile(files, path, size, rand);
FileSystem fs = path.GetFileSystem(conf);
FileStatus sStat = fs.GetFileStatus(path);
tasks.AddItem(new _Callable_358(fs, path, sStat, statCache));
}
ExecutorService exec = Executors.NewFixedThreadPool(fileCount);
try
{
IList <Future <bool> > futures = exec.InvokeAll(tasks);
// files should be public
foreach (Future <bool> future in futures)
{
NUnit.Framework.Assert.IsTrue(future.Get());
}
// for each path exactly one file status call should be made
foreach (AtomicInteger count in counts.Values)
{
NUnit.Framework.Assert.AreSame(count.Get(), 1);
}
}
finally
{
exec.Shutdown();
}
}
/// <summary>
/// Test that
/// <see cref="LocalDirAllocator.GetAllLocalPathsToRead(string, Org.Apache.Hadoop.Conf.Configuration)
/// "/>
///
/// returns correct filenames and "file" schema.
/// </summary>
/// <exception cref="System.IO.IOException"/>
public virtual void TestGetAllLocalPathsToRead()
{
Assume.AssumeTrue(!isWindows);
string dir0 = BuildBufferDir(Root, 0);
string dir1 = BuildBufferDir(Root, 1);
try
{
conf.Set(Context, dir0 + "," + dir1);
Assert.True(localFs.Mkdirs(new Path(dir0)));
Assert.True(localFs.Mkdirs(new Path(dir1)));
localFs.Create(new Path(dir0 + Path.Separator + Filename));
localFs.Create(new Path(dir1 + Path.Separator + Filename));
// check both the paths are returned as paths to read:
IEnumerable <Path> pathIterable = dirAllocator.GetAllLocalPathsToRead(Filename, conf
);
int count = 0;
foreach (Path p in pathIterable)
{
count++;
Assert.Equal(Filename, p.GetName());
Assert.Equal("file", p.GetFileSystem(conf).GetUri().GetScheme(
));
}
Assert.Equal(2, count);
// test #next() while no element to iterate any more:
try
{
Path p_1 = pathIterable.GetEnumerator().Next();
NUnit.Framework.Assert.IsFalse("NoSuchElementException must be thrown, but returned ["
+ p_1 + "] instead.", true);
}
catch (NoSuchElementException)
{
}
// exception expected
// okay
// test modification not allowed:
IEnumerable <Path> pathIterable2 = dirAllocator.GetAllLocalPathsToRead(Filename, conf
);
IEnumerator <Path> it = pathIterable2.GetEnumerator();
try
{
it.Remove();
NUnit.Framework.Assert.IsFalse(true);
}
catch (NotSupportedException)
{
}
}
finally
{
// exception expected
// okay
Shell.ExecCommand(new string[] { "chmod", "u+w", BufferDirRoot });
RmBufferDirs();
}
}
public virtual void SetUp()
{
// These tests simulate volume failures by denying execute permission on the
// volume's path. On Windows, the owner of an object is always allowed
// access, so we can't run these tests on Windows.
Assume.AssumeTrue(!Path.Windows);
// Allow a single volume failure (there are two volumes)
InitCluster(1, 2, 1);
}
public virtual void TestGetWindowsLocalPath()
{
Assume.AssumeTrue(Path.Windows);
string winDstFile = (new FilePath(dstPath.ToUri().GetPath().ToString())).GetAbsolutePath
();
ShellRun(0, "-get", srcPath.ToString(), winDstFile);
CheckPath(dstPath, false);
}
public virtual void Setup()
{
Assume.AssumeTrue(NativeCrc32.IsAvailable());
Assert.Equal("These tests assume they can write a checksum value as a 4-byte int."
, 4, checksumType.size);
Configuration conf = new Configuration();
bytesPerChecksum = conf.GetInt(IoBytesPerChecksumKey, IoBytesPerChecksumDefault);
fileName = this.GetType().Name;
checksum = DataChecksum.NewDataChecksum(checksumType, bytesPerChecksum);
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestListStatusWithColons()
{
Assume.AssumeTrue(!Shell.Windows);
FilePath colonFile = new FilePath(TestRootDir, "foo:bar");
colonFile.Mkdirs();
FileStatus[] stats = fileSys.ListStatus(new Path(TestRootDir));
Assert.Equal("Unexpected number of stats", 1, stats.Length);
Assert.Equal("Bad path from stat", colonFile.GetAbsolutePath()
, stats[0].GetPath().ToUri().GetPath());
}
/// <exception cref="Org.Apache.Hadoop.HA.BadFencingConfigurationException"/>
public virtual void TestFence()
{
Assume.AssumeTrue(IsConfigured());
Configuration conf = new Configuration();
conf.Set(SshFenceByTcpPort.ConfIdentitiesKey, TestKeyfile);
SshFenceByTcpPort fence = new SshFenceByTcpPort();
fence.SetConf(conf);
Assert.True(fence.TryFence(TestTarget, null));
}
/// <exception cref="System.Exception"/>
public virtual void TestStatFileNotFound()
{
Assume.AssumeTrue(Stat.IsAvailable());
try
{
stat.GetFileStatus();
NUnit.Framework.Assert.Fail("Expected FileNotFoundException");
}
catch (FileNotFoundException)
{
}
}
// Expected
/// <exception cref="System.IO.IOException"/>
public virtual void TestDanglingLink()
{
Assume.AssumeTrue(!Path.Windows);
Path fileAbs = new Path(TestBaseDir1() + "/file");
Path fileQual = new Path(TestURI().ToString(), fileAbs);
Path link = new Path(TestBaseDir1() + "/linkToFile");
Path linkQual = new Path(TestURI().ToString(), link.ToString());
wrapper.CreateSymlink(fileAbs, link, false);
// Deleting the link using FileContext currently fails because
// resolve looks up LocalFs rather than RawLocalFs for the path
// so we call ChecksumFs delete (which doesn't delete dangling
// links) instead of delegating to delete in RawLocalFileSystem
// which deletes via fullyDelete. testDeleteLink above works
// because the link is not dangling.
//assertTrue(fc.delete(link, false));
FileUtil.FullyDelete(new FilePath(link.ToUri().GetPath()));
wrapper.CreateSymlink(fileAbs, link, false);
try
{
wrapper.GetFileStatus(link);
NUnit.Framework.Assert.Fail("Got FileStatus for dangling link");
}
catch (FileNotFoundException)
{
}
// Expected. File's exists method returns false for dangling links
// We can stat a dangling link
UserGroupInformation user = UserGroupInformation.GetCurrentUser();
FileStatus fsd = wrapper.GetFileLinkStatus(link);
Assert.Equal(fileQual, fsd.GetSymlink());
Assert.True(fsd.IsSymlink());
NUnit.Framework.Assert.IsFalse(fsd.IsDirectory());
Assert.Equal(user.GetUserName(), fsd.GetOwner());
// Compare against user's primary group
Assert.Equal(user.GetGroupNames()[0], fsd.GetGroup());
Assert.Equal(linkQual, fsd.GetPath());
// Accessing the link
try
{
ReadFile(link);
NUnit.Framework.Assert.Fail("Got FileStatus for dangling link");
}
catch (FileNotFoundException)
{
}
// Ditto.
// Creating the file makes the link work
CreateAndWriteFile(fileAbs);
wrapper.GetFileStatus(link);
}
public virtual void TestRunCommandWithNoResources()
{
// Windows only test
Assume.AssumeTrue(Shell.Windows);
Configuration conf = new Configuration();
string[] command = containerExecutor.GetRunCommand("echo", "group1", null, null,
conf, Resource.NewInstance(1024, 1));
// Assert the cpu and memory limits are set correctly in the command
string[] expected = new string[] { Shell.Winutils, "task", "create", "-m", "-1",
"-c", "-1", "group1", "cmd /c " + "echo" };
NUnit.Framework.Assert.IsTrue(Arrays.Equals(expected, command));
}
/// <exception cref="System.Exception"/>
public virtual void TestInstall()
{
Assume.AssumeTrue(SystemUtils.IsOsUnix);
SignalLogger.Instance.Register(Log);
try
{
SignalLogger.Instance.Register(Log);
NUnit.Framework.Assert.Fail("expected IllegalStateException from double registration"
);
}
catch (InvalidOperationException)
{
}
}
public virtual void TestCopyFileFromWindowsLocalPath()
{
Assume.AssumeTrue(Path.Windows);
string windowsTestRootPath = (new FilePath(testRootDir.ToUri().GetPath().ToString
())).GetAbsolutePath();
Path testRoot = new Path(windowsTestRootPath, "testPutFile");
lfs.Delete(testRoot, true);
lfs.Mkdirs(testRoot);
Path targetDir = new Path(testRoot, "target");
Path filePath = new Path(testRoot, new Path("srcFile"));
lfs.Create(filePath).Close();
CheckPut(filePath, targetDir, true);
}
public virtual void TestAddDenseToSparse(double[] dense1, int sparseIndex, double v)
{
Assume.AssumeTrue(sparseIndex >= 0);
Assume.AssumeTrue(sparseIndex <= 100);
ConcatVector v1 = new ConcatVector(1);
v1.SetDenseComponent(0, dense1);
ConcatVector v2 = new ConcatVector(1);
v2.SetSparseComponent(0, (int)sparseIndex, v);
double expected = v1.DotProduct(v2) + 0.7f * (v1.DotProduct(v1));
v2.AddVectorInPlace(v1, 0.7f);
NUnit.Framework.Assert.AreEqual(v2.DotProduct(v1), 5.0e-4, expected);
}