// expected /// <exception cref="System.Exception"/> public virtual void TestSyncFileRange() { FileOutputStream fos = new FileOutputStream(new FilePath(TestDir, "testSyncFileRange" )); try { fos.Write(Runtime.GetBytesForString("foo")); NativeIO.POSIX.Sync_file_range(fos.GetFD(), 0, 1024, NativeIO.POSIX.SyncFileRangeWrite ); } catch (NotSupportedException) { // no way to verify that this actually has synced, // but if it doesn't throw, we can assume it worked // we should just skip the unit test on machines where we don't // have fadvise support Assume.AssumeTrue(false); } finally { fos.Close(); } try { NativeIO.POSIX.Sync_file_range(fos.GetFD(), 0, 1024, NativeIO.POSIX.SyncFileRangeWrite ); NUnit.Framework.Assert.Fail("Did not throw on bad file"); } catch (NativeIOException nioe) { Assert.Equal(Errno.Ebadf, nioe.GetErrno()); } }
// Expect NoSuchPaddingException /// <exception cref="System.Exception"/> public virtual void TestUpdateArguments() { Assume.AssumeTrue(OpensslCipher.GetLoadingFailureReason() == null); OpensslCipher cipher = OpensslCipher.GetInstance("AES/CTR/NoPadding"); Assert.True(cipher != null); cipher.Init(OpensslCipher.EncryptMode, key, iv); // Require direct buffers ByteBuffer input = ByteBuffer.Allocate(1024); ByteBuffer output = ByteBuffer.Allocate(1024); try { cipher.Update(input, output); NUnit.Framework.Assert.Fail("Input and output buffer should be direct buffer."); } catch (ArgumentException e) { GenericTestUtils.AssertExceptionContains("Direct buffers are required", e); } // Output buffer length should be sufficient to store output data input = ByteBuffer.AllocateDirect(1024); output = ByteBuffer.AllocateDirect(1000); try { cipher.Update(input, output); NUnit.Framework.Assert.Fail("Output buffer length should be sufficient " + "to store output data" ); } catch (ShortBufferException e) { GenericTestUtils.AssertExceptionContains("Output buffer is not sufficient", e); } }
public virtual void TestShortCircuitTraceHooks() { Assume.AssumeTrue(NativeCodeLoader.IsNativeCodeLoaded() && !Path.Windows); conf = new Configuration(); conf.Set(DFSConfigKeys.DfsClientHtracePrefix + SpanReceiverHost.SpanReceiversConfSuffix , typeof(TestTracing.SetSpanReceiver).FullName); conf.SetLong("dfs.blocksize", 100 * 1024); conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, true); conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, false); conf.Set(DFSConfigKeys.DfsDomainSocketPathKey, "testShortCircuitTraceHooks._PORT" ); conf.Set(DFSConfigKeys.DfsChecksumTypeKey, "CRC32C"); cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); dfs = cluster.GetFileSystem(); try { DFSTestUtil.CreateFile(dfs, TestPath, TestLength, (short)1, 5678L); TraceScope ts = Trace.StartSpan("testShortCircuitTraceHooks", Sampler.Always); FSDataInputStream stream = dfs.Open(TestPath); byte[] buf = new byte[TestLength]; IOUtils.ReadFully(stream, buf, 0, TestLength); stream.Close(); ts.Close(); string[] expectedSpanNames = new string[] { "OpRequestShortCircuitAccessProto", "ShortCircuitShmRequestProto" }; TestTracing.AssertSpanNamesFound(expectedSpanNames); } finally { dfs.Close(); cluster.Shutdown(); } }
public virtual void TestFailToRename() { Assume.AssumeTrue(Shell.Windows); OutputStream fos = null; try { fos = new AtomicFileOutputStream(DstFile); fos.Write(Sharpen.Runtime.GetBytesForString(TestString)); FileUtil.SetWritable(TestDir, false); exception.Expect(typeof(IOException)); exception.ExpectMessage("failure in native rename"); try { fos.Close(); } finally { fos = null; } } finally { IOUtils.Cleanup(null, fos); FileUtil.SetWritable(TestDir, true); } }
public virtual void TestBothOldAndNewShortCircuitConfigured() { short ReplFactor = 1; int FileLength = 512; Assume.AssumeTrue(null == DomainSocket.GetLoadingFailureReason()); TemporarySocketDirectory socketDir = new TemporarySocketDirectory(); HdfsConfiguration conf = GetConfiguration(socketDir); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); cluster.WaitActive(); socketDir.Close(); FileSystem fs = cluster.GetFileSystem(); Path path = new Path("/foo"); byte[] orig = new byte[FileLength]; for (int i = 0; i < orig.Length; i++) { orig[i] = unchecked ((byte)(i % 10)); } FSDataOutputStream fos = fs.Create(path, (short)1); fos.Write(orig); fos.Close(); DFSTestUtil.WaitReplication(fs, path, ReplFactor); FSDataInputStream fis = cluster.GetFileSystem().Open(path); byte[] buf = new byte[FileLength]; IOUtils.ReadFully(fis, buf, 0, FileLength); fis.Close(); Assert.AssertArrayEquals(orig, buf); Arrays.Equals(orig, buf); cluster.Shutdown(); }
/// <exception cref="System.Exception"/> public virtual void TestJournalLocking() { Assume.AssumeTrue(journal.GetStorage().GetStorageDir(0).IsLockSupported()); Storage.StorageDirectory sd = journal.GetStorage().GetStorageDir(0); FilePath lockFile = new FilePath(sd.GetRoot(), Storage.StorageFileLock); // Journal should be locked, since the format() call locks it. GenericTestUtils.AssertExists(lockFile); journal.NewEpoch(FakeNsinfo, 1); try { new Journal(conf, TestLogDir, Jid, HdfsServerConstants.StartupOption.Regular, mockErrorReporter ); NUnit.Framework.Assert.Fail("Did not fail to create another journal in same dir"); } catch (IOException ioe) { GenericTestUtils.AssertExceptionContains("Cannot lock storage", ioe); } journal.Close(); // Journal should no longer be locked after the close() call. // Hence, should be able to create a new Journal in the same dir. Journal journal2 = new Journal(conf, TestLogDir, Jid, HdfsServerConstants.StartupOption .Regular, mockErrorReporter); journal2.NewEpoch(FakeNsinfo, 2); journal2.Close(); }
public virtual void TestDuplicates() { Assume.AssumeTrue(!Shell.Windows); string GetAllUsersCmd = "echo \"root:x:0:0:root:/root:/bin/bash\n" + "hdfs:x:11501:10787:Grid Distributed File System:/home/hdfs:/bin/bash\n" + "hdfs:x:11502:10788:Grid Distributed File System:/home/hdfs:/bin/bash\n" + "hdfs1:x:11501:10787:Grid Distributed File System:/home/hdfs:/bin/bash\n" + "hdfs2:x:11502:10787:Grid Distributed File System:/home/hdfs:/bin/bash\n" + "bin:x:2:2:bin:/bin:/bin/sh\n" + "bin:x:1:1:bin:/bin:/sbin/nologin\n" + "daemon:x:1:1:daemon:/usr/sbin:/bin/sh\n" + "daemon:x:2:2:daemon:/sbin:/sbin/nologin\"" + " | cut -d: -f1,3"; string GetAllGroupsCmd = "echo \"hdfs:*:11501:hrt_hdfs\n" + "mapred:x:497\n" + "mapred2:x:497\n" + "mapred:x:498\n" + "mapred3:x:498\"" + " | cut -d: -f1,3"; // Maps for id to name map BiMap <int, string> uMap = HashBiMap.Create(); BiMap <int, string> gMap = HashBiMap.Create(); ShellBasedIdMapping.UpdateMapInternal(uMap, "user", GetAllUsersCmd, ":", EmptyPassThroughMap ); Assert.Equal(5, uMap.Count); Assert.Equal("root", uMap[0]); Assert.Equal("hdfs", uMap[11501]); Assert.Equal("hdfs2", uMap[11502]); Assert.Equal("bin", uMap[2]); Assert.Equal("daemon", uMap[1]); ShellBasedIdMapping.UpdateMapInternal(gMap, "group", GetAllGroupsCmd, ":", EmptyPassThroughMap ); Assert.True(gMap.Count == 3); Assert.Equal("hdfs", gMap[11501]); Assert.Equal("mapred", gMap[497]); Assert.Equal("mapred3", gMap[498]); }
public virtual void TestStaticMapping() { Assume.AssumeTrue(!Shell.Windows); IDictionary <int, int> uidStaticMap = new ShellBasedIdMapping.PassThroughMap <int>( ); IDictionary <int, int> gidStaticMap = new ShellBasedIdMapping.PassThroughMap <int>( ); uidStaticMap[11501] = 10; gidStaticMap[497] = 200; // Maps for id to name map BiMap <int, string> uMap = HashBiMap.Create(); BiMap <int, string> gMap = HashBiMap.Create(); string GetAllUsersCmd = "echo \"atm:x:1000:1000:Aaron T. Myers,,,:/home/atm:/bin/bash\n" + "hdfs:x:11501:10787:Grid Distributed File System:/home/hdfs:/bin/bash\"" + " | cut -d: -f1,3"; string GetAllGroupsCmd = "echo \"hdfs:*:11501:hrt_hdfs\n" + "mapred:x:497\n" + "mapred2:x:498\"" + " | cut -d: -f1,3"; ShellBasedIdMapping.UpdateMapInternal(uMap, "user", GetAllUsersCmd, ":", uidStaticMap ); ShellBasedIdMapping.UpdateMapInternal(gMap, "group", GetAllGroupsCmd, ":", gidStaticMap ); Assert.Equal("hdfs", uMap[10]); Assert.Equal(10, (int)uMap.Inverse()["hdfs"]); Assert.Equal("atm", uMap[1000]); Assert.Equal(1000, (int)uMap.Inverse()["atm"]); Assert.Equal("hdfs", gMap[11501]); Assert.Equal(11501, (int)gMap.Inverse()["hdfs"]); Assert.Equal("mapred", gMap[200]); Assert.Equal(200, (int)gMap.Inverse()["mapred"]); Assert.Equal("mapred2", gMap[498]); Assert.Equal(498, (int)gMap.Inverse()["mapred2"]); }
public virtual void TestElementwiseProduct(ConcatVectorTest.DenseTestVector d1, ConcatVectorTest.DenseTestVector d2) { for (int i = 0; i < d1.values.Length; i++) { for (int j = 0; j < d1.values[i].Length; j++) { Assume.AssumeTrue(d1.values[i][j] == d1.vector.GetValueAt(i, j)); } } for (int i_1 = 0; i_1 < d2.values.Length; i_1++) { for (int j = 0; j < d2.values[i_1].Length; j++) { Assume.AssumeTrue(d2.values[i_1][j] == d2.vector.GetValueAt(i_1, j)); } } ConcatVector clone = d1.vector.DeepClone(); clone.ElementwiseProductInPlace(d2.vector); for (int i_2 = 0; i_2 < d1.values.Length; i_2++) { for (int j = 0; j < d1.values[i_2].Length; j++) { double val = 0.0f; if (i_2 < d2.values.Length) { if (j < d2.values[i_2].Length) { val = d1.values[i_2][j] * d2.values[i_2][j]; } } NUnit.Framework.Assert.AreEqual(clone.GetValueAt(i_2, j), 5.0e-4, val); } } }
public virtual void TestUnderReplicationAfterVolFailure() { // This test relies on denying access to data volumes to simulate data volume // failure. This doesn't work on Windows, because an owner of an object // always has the ability to read and change permissions on the object. Assume.AssumeTrue(!Path.Windows); // Bring up one more datanode cluster.StartDataNodes(conf, 1, true, null, null); cluster.WaitActive(); BlockManager bm = cluster.GetNamesystem().GetBlockManager(); Path file1 = new Path("/test1"); DFSTestUtil.CreateFile(fs, file1, 1024, (short)3, 1L); DFSTestUtil.WaitReplication(fs, file1, (short)3); // Fail the first volume on both datanodes FilePath dn1Vol1 = new FilePath(dataDir, "data" + (2 * 0 + 1)); FilePath dn2Vol1 = new FilePath(dataDir, "data" + (2 * 1 + 1)); DataNodeTestUtils.InjectDataDirFailure(dn1Vol1, dn2Vol1); Path file2 = new Path("/test2"); DFSTestUtil.CreateFile(fs, file2, 1024, (short)3, 1L); DFSTestUtil.WaitReplication(fs, file2, (short)3); // underReplicatedBlocks are due to failed volumes int underReplicatedBlocks = BlockManagerTestUtil.CheckHeartbeatAndGetUnderReplicatedBlocksCount (cluster.GetNamesystem(), bm); NUnit.Framework.Assert.IsTrue("There is no under replicated block after volume failure" , underReplicatedBlocks > 0); }
public virtual void TestGetMaxedMarginals(TableFactor factor, int marginalizeTo) { if (!Arrays.Stream(factor.neighborIndices).Boxed().Collect(Collectors.ToSet()).Contains(marginalizeTo)) { return; } int indexOfVariable = -1; for (int i = 0; i < factor.neighborIndices.Length; i++) { if (factor.neighborIndices[i] == marginalizeTo) { indexOfVariable = i; break; } } Assume.AssumeTrue(indexOfVariable > -1); double[] gold = new double[factor.GetDimensions()[indexOfVariable]]; for (int i_1 = 0; i_1 < gold.Length; i_1++) { gold[i_1] = double.NegativeInfinity; } foreach (int[] assignment in factor) { gold[assignment[indexOfVariable]] = Math.Max(gold[assignment[indexOfVariable]], factor.GetAssignmentValue(assignment)); } Normalize(gold); Assert.AssertArrayEquals(factor.GetMaxedMarginals()[indexOfVariable], 1.0e-5, gold); }
/// <summary>Tests for a given volumes to be tolerated and volumes failed.</summary> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> private void TestVolumeConfig(int volumesTolerated, int volumesFailed, bool expectedBPServiceState , bool manageDfsDirs) { Assume.AssumeTrue(!Runtime.GetProperty("os.name").StartsWith("Windows")); int dnIndex = 0; // Fail the current directory since invalid storage directory perms // get fixed up automatically on datanode startup. FilePath[] dirs = new FilePath[] { new FilePath(cluster.GetInstanceStorageDir(dnIndex , 0), "current"), new FilePath(cluster.GetInstanceStorageDir(dnIndex, 1), "current" ) }; try { for (int i = 0; i < volumesFailed; i++) { PrepareDirToFail(dirs[i]); } RestartDatanodes(volumesTolerated, manageDfsDirs); NUnit.Framework.Assert.AreEqual(expectedBPServiceState, cluster.GetDataNodes()[0] .IsBPServiceAlive(cluster.GetNamesystem().GetBlockPoolId())); } finally { foreach (FilePath dir in dirs) { FileUtil.Chmod(dir.ToString(), "755"); } } }
public virtual void TestConfigureMinValidVolumes() { Assume.AssumeTrue(!Runtime.GetProperty("os.name").StartsWith("Windows")); // Bring up two additional datanodes that need both of their volumes // functioning in order to stay up. conf.SetInt(DFSConfigKeys.DfsDatanodeFailedVolumesToleratedKey, 0); cluster.StartDataNodes(conf, 2, true, null, null); cluster.WaitActive(); DatanodeManager dm = cluster.GetNamesystem().GetBlockManager().GetDatanodeManager (); long origCapacity = DFSTestUtil.GetLiveDatanodeCapacity(dm); long dnCapacity = DFSTestUtil.GetDatanodeCapacity(dm, 0); // Fail a volume on the 2nd DN FilePath dn2Vol1 = new FilePath(dataDir, "data" + (2 * 1 + 1)); DataNodeTestUtils.InjectDataDirFailure(dn2Vol1); // Should only get two replicas (the first DN and the 3rd) Path file1 = new Path("/test1"); DFSTestUtil.CreateFile(fs, file1, 1024, (short)3, 1L); DFSTestUtil.WaitReplication(fs, file1, (short)2); // Check that this single failure caused a DN to die. DFSTestUtil.WaitForDatanodeStatus(dm, 2, 1, 0, origCapacity - (1 * dnCapacity), WaitForHeartbeats ); // If we restore the volume we should still only be able to get // two replicas since the DN is still considered dead. DataNodeTestUtils.RestoreDataDirFromFailure(dn2Vol1); Path file2 = new Path("/test2"); DFSTestUtil.CreateFile(fs, file2, 1024, (short)3, 1L); DFSTestUtil.WaitReplication(fs, file2, (short)2); }
/// <exception cref="System.Exception"/> public virtual void TestMlock() { Assume.AssumeTrue(NativeIO.IsAvailable()); FilePath TestFile = new FilePath(new FilePath(Runtime.GetProperty("test.build.data" , "build/test/data")), "testMlockFile"); int BufLen = 12289; byte[] buf = new byte[BufLen]; int bufSum = 0; for (int i = 0; i < buf.Length; i++) { buf[i] = unchecked ((byte)(i % 60)); bufSum += buf[i]; } FileOutputStream fos = new FileOutputStream(TestFile); try { fos.Write(buf); fos.GetChannel().Force(true); } finally { fos.Close(); } FileInputStream fis = null; FileChannel channel = null; try { // Map file into memory fis = new FileInputStream(TestFile); channel = fis.GetChannel(); long fileSize = channel.Size(); MappedByteBuffer mapbuf = channel.Map(FileChannel.MapMode.ReadOnly, 0, fileSize); // mlock the buffer NativeIO.POSIX.Mlock(mapbuf, fileSize); // Read the buffer int sum = 0; for (int i_1 = 0; i_1 < fileSize; i_1++) { sum += mapbuf.Get(i_1); } Assert.Equal("Expected sums to be equal", bufSum, sum); // munmap the buffer, which also implicitly unlocks it NativeIO.POSIX.Munmap(mapbuf); } finally { if (channel != null) { channel.Close(); } if (fis != null) { fis.Close(); } } }
public virtual void TestBlockTokenRpcLeak() { Configuration conf = new Configuration(); conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos"); UserGroupInformation.SetConfiguration(conf); Assume.AssumeTrue(FdDir.Exists()); BlockTokenSecretManager sm = new BlockTokenSecretManager(blockKeyUpdateInterval, blockTokenLifetime, 0, "fake-pool", null); Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token = sm.GenerateToken (block3, EnumSet.AllOf <BlockTokenSecretManager.AccessMode>()); Server server = CreateMockDatanode(sm, token, conf); server.Start(); IPEndPoint addr = NetUtils.GetConnectAddress(server); DatanodeID fakeDnId = DFSTestUtil.GetLocalDatanodeID(addr.Port); ExtendedBlock b = new ExtendedBlock("fake-pool", new Org.Apache.Hadoop.Hdfs.Protocol.Block (12345L)); LocatedBlock fakeBlock = new LocatedBlock(b, new DatanodeInfo[0]); fakeBlock.SetBlockToken(token); // Create another RPC proxy with the same configuration - this will never // attempt to connect anywhere -- but it causes the refcount on the // RPC "Client" object to stay above 0 such that RPC.stopProxy doesn't // actually close the TCP connections to the real target DN. ClientDatanodeProtocol proxyToNoWhere = RPC.GetProxy <ClientDatanodeProtocol>(ClientDatanodeProtocol .versionID, new IPEndPoint("1.1.1.1", 1), UserGroupInformation.CreateRemoteUser( "junk"), conf, NetUtils.GetDefaultSocketFactory(conf)); ClientDatanodeProtocol proxy = null; int fdsAtStart = CountOpenFileDescriptors(); try { long endTime = Time.Now() + 3000; while (Time.Now() < endTime) { proxy = DFSUtil.CreateClientDatanodeProtocolProxy(fakeDnId, conf, 1000, false, fakeBlock ); NUnit.Framework.Assert.AreEqual(block3.GetBlockId(), proxy.GetReplicaVisibleLength (block3)); if (proxy != null) { RPC.StopProxy(proxy); } Log.Info("Num open fds:" + CountOpenFileDescriptors()); } int fdsAtEnd = CountOpenFileDescriptors(); if (fdsAtEnd - fdsAtStart > 50) { NUnit.Framework.Assert.Fail("Leaked " + (fdsAtEnd - fdsAtStart) + " fds!"); } } finally { server.Stop(); } RPC.StopProxy(proxyToNoWhere); }
/// <exception cref="System.IO.IOException"/> private static OsSecureRandom GetOsSecureRandom() { Assume.AssumeTrue(SystemUtils.IsOsLinux); OsSecureRandom random = new OsSecureRandom(); random.SetConf(new Configuration()); return(random); }
/// <exception cref="System.Exception"/> public virtual void TestRenameTo() { FilePath TestDir = new FilePath(new FilePath(Runtime.GetProperty("test.build.data" , "build/test/data")), "renameTest"); Assume.AssumeTrue(TestDir.Mkdirs()); FilePath nonExistentFile = new FilePath(TestDir, "nonexistent"); FilePath targetFile = new FilePath(TestDir, "target"); // Test attempting to rename a nonexistent file. try { NativeIO.RenameTo(nonExistentFile, targetFile); NUnit.Framework.Assert.Fail(); } catch (NativeIOException e) { if (Path.Windows) { Assert.Equal(string.Format("The system cannot find the file specified.%n" ), e.Message); } else { Assert.Equal(Errno.Enoent, e.GetErrno()); } } // Test renaming a file to itself. It should succeed and do nothing. FilePath sourceFile = new FilePath(TestDir, "source"); Assert.True(sourceFile.CreateNewFile()); NativeIO.RenameTo(sourceFile, sourceFile); // Test renaming a source to a destination. NativeIO.RenameTo(sourceFile, targetFile); // Test renaming a source to a path which uses a file as a directory. sourceFile = new FilePath(TestDir, "source"); Assert.True(sourceFile.CreateNewFile()); FilePath badTarget = new FilePath(targetFile, "subdir"); try { NativeIO.RenameTo(sourceFile, badTarget); NUnit.Framework.Assert.Fail(); } catch (NativeIOException e) { if (Path.Windows) { Assert.Equal(string.Format("The parameter is incorrect.%n"), e .Message); } else { Assert.Equal(Errno.Enotdir, e.GetErrno()); } } FileUtils.DeleteQuietly(TestDir); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Sharpen.URISyntaxException"/> /// <exception cref="System.Exception"/> /// <exception cref="Sharpen.ExecutionException"/> public virtual void TestDownloadPublicWithStatCache() { Configuration conf = new Configuration(); FileContext files = FileContext.GetLocalFSFileContext(conf); Path basedir = files.MakeQualified(new Path("target", typeof(TestFSDownload).Name )); // if test directory doesn't have ancestor permission, skip this test FileSystem f = basedir.GetFileSystem(conf); Assume.AssumeTrue(FSDownload.AncestorsHaveExecutePermissions(f, basedir, null)); files.Mkdir(basedir, null, true); conf.SetStrings(typeof(TestFSDownload).FullName, basedir.ToString()); int size = 512; ConcurrentMap <Path, AtomicInteger> counts = new ConcurrentHashMap <Path, AtomicInteger >(); CacheLoader <Path, Future <FileStatus> > loader = FSDownload.CreateStatusCacheLoader (conf); LoadingCache <Path, Future <FileStatus> > statCache = CacheBuilder.NewBuilder().Build (new _CacheLoader_328(counts, loader)); // increment the count // use the default loader // test FSDownload.isPublic() concurrently int fileCount = 3; IList <Callable <bool> > tasks = new AList <Callable <bool> >(); for (int i = 0; i < fileCount; i++) { Random rand = new Random(); long sharedSeed = rand.NextLong(); rand.SetSeed(sharedSeed); System.Console.Out.WriteLine("SEED: " + sharedSeed); Path path = new Path(basedir, "test-file-" + i); CreateFile(files, path, size, rand); FileSystem fs = path.GetFileSystem(conf); FileStatus sStat = fs.GetFileStatus(path); tasks.AddItem(new _Callable_358(fs, path, sStat, statCache)); } ExecutorService exec = Executors.NewFixedThreadPool(fileCount); try { IList <Future <bool> > futures = exec.InvokeAll(tasks); // files should be public foreach (Future <bool> future in futures) { NUnit.Framework.Assert.IsTrue(future.Get()); } // for each path exactly one file status call should be made foreach (AtomicInteger count in counts.Values) { NUnit.Framework.Assert.AreSame(count.Get(), 1); } } finally { exec.Shutdown(); } }
/// <summary> /// Test that /// <see cref="LocalDirAllocator.GetAllLocalPathsToRead(string, Org.Apache.Hadoop.Conf.Configuration) /// "/> /// /// returns correct filenames and "file" schema. /// </summary> /// <exception cref="System.IO.IOException"/> public virtual void TestGetAllLocalPathsToRead() { Assume.AssumeTrue(!isWindows); string dir0 = BuildBufferDir(Root, 0); string dir1 = BuildBufferDir(Root, 1); try { conf.Set(Context, dir0 + "," + dir1); Assert.True(localFs.Mkdirs(new Path(dir0))); Assert.True(localFs.Mkdirs(new Path(dir1))); localFs.Create(new Path(dir0 + Path.Separator + Filename)); localFs.Create(new Path(dir1 + Path.Separator + Filename)); // check both the paths are returned as paths to read: IEnumerable <Path> pathIterable = dirAllocator.GetAllLocalPathsToRead(Filename, conf ); int count = 0; foreach (Path p in pathIterable) { count++; Assert.Equal(Filename, p.GetName()); Assert.Equal("file", p.GetFileSystem(conf).GetUri().GetScheme( )); } Assert.Equal(2, count); // test #next() while no element to iterate any more: try { Path p_1 = pathIterable.GetEnumerator().Next(); NUnit.Framework.Assert.IsFalse("NoSuchElementException must be thrown, but returned [" + p_1 + "] instead.", true); } catch (NoSuchElementException) { } // exception expected // okay // test modification not allowed: IEnumerable <Path> pathIterable2 = dirAllocator.GetAllLocalPathsToRead(Filename, conf ); IEnumerator <Path> it = pathIterable2.GetEnumerator(); try { it.Remove(); NUnit.Framework.Assert.IsFalse(true); } catch (NotSupportedException) { } } finally { // exception expected // okay Shell.ExecCommand(new string[] { "chmod", "u+w", BufferDirRoot }); RmBufferDirs(); } }
public virtual void SetUp() { // These tests simulate volume failures by denying execute permission on the // volume's path. On Windows, the owner of an object is always allowed // access, so we can't run these tests on Windows. Assume.AssumeTrue(!Path.Windows); // Allow a single volume failure (there are two volumes) InitCluster(1, 2, 1); }
public virtual void TestGetWindowsLocalPath() { Assume.AssumeTrue(Path.Windows); string winDstFile = (new FilePath(dstPath.ToUri().GetPath().ToString())).GetAbsolutePath (); ShellRun(0, "-get", srcPath.ToString(), winDstFile); CheckPath(dstPath, false); }
public virtual void Setup() { Assume.AssumeTrue(NativeCrc32.IsAvailable()); Assert.Equal("These tests assume they can write a checksum value as a 4-byte int." , 4, checksumType.size); Configuration conf = new Configuration(); bytesPerChecksum = conf.GetInt(IoBytesPerChecksumKey, IoBytesPerChecksumDefault); fileName = this.GetType().Name; checksum = DataChecksum.NewDataChecksum(checksumType, bytesPerChecksum); }
/// <exception cref="System.IO.IOException"/> public virtual void TestListStatusWithColons() { Assume.AssumeTrue(!Shell.Windows); FilePath colonFile = new FilePath(TestRootDir, "foo:bar"); colonFile.Mkdirs(); FileStatus[] stats = fileSys.ListStatus(new Path(TestRootDir)); Assert.Equal("Unexpected number of stats", 1, stats.Length); Assert.Equal("Bad path from stat", colonFile.GetAbsolutePath() , stats[0].GetPath().ToUri().GetPath()); }
/// <exception cref="Org.Apache.Hadoop.HA.BadFencingConfigurationException"/> public virtual void TestFence() { Assume.AssumeTrue(IsConfigured()); Configuration conf = new Configuration(); conf.Set(SshFenceByTcpPort.ConfIdentitiesKey, TestKeyfile); SshFenceByTcpPort fence = new SshFenceByTcpPort(); fence.SetConf(conf); Assert.True(fence.TryFence(TestTarget, null)); }
/// <exception cref="System.Exception"/> public virtual void TestStatFileNotFound() { Assume.AssumeTrue(Stat.IsAvailable()); try { stat.GetFileStatus(); NUnit.Framework.Assert.Fail("Expected FileNotFoundException"); } catch (FileNotFoundException) { } }
// Expected /// <exception cref="System.IO.IOException"/> public virtual void TestDanglingLink() { Assume.AssumeTrue(!Path.Windows); Path fileAbs = new Path(TestBaseDir1() + "/file"); Path fileQual = new Path(TestURI().ToString(), fileAbs); Path link = new Path(TestBaseDir1() + "/linkToFile"); Path linkQual = new Path(TestURI().ToString(), link.ToString()); wrapper.CreateSymlink(fileAbs, link, false); // Deleting the link using FileContext currently fails because // resolve looks up LocalFs rather than RawLocalFs for the path // so we call ChecksumFs delete (which doesn't delete dangling // links) instead of delegating to delete in RawLocalFileSystem // which deletes via fullyDelete. testDeleteLink above works // because the link is not dangling. //assertTrue(fc.delete(link, false)); FileUtil.FullyDelete(new FilePath(link.ToUri().GetPath())); wrapper.CreateSymlink(fileAbs, link, false); try { wrapper.GetFileStatus(link); NUnit.Framework.Assert.Fail("Got FileStatus for dangling link"); } catch (FileNotFoundException) { } // Expected. File's exists method returns false for dangling links // We can stat a dangling link UserGroupInformation user = UserGroupInformation.GetCurrentUser(); FileStatus fsd = wrapper.GetFileLinkStatus(link); Assert.Equal(fileQual, fsd.GetSymlink()); Assert.True(fsd.IsSymlink()); NUnit.Framework.Assert.IsFalse(fsd.IsDirectory()); Assert.Equal(user.GetUserName(), fsd.GetOwner()); // Compare against user's primary group Assert.Equal(user.GetGroupNames()[0], fsd.GetGroup()); Assert.Equal(linkQual, fsd.GetPath()); // Accessing the link try { ReadFile(link); NUnit.Framework.Assert.Fail("Got FileStatus for dangling link"); } catch (FileNotFoundException) { } // Ditto. // Creating the file makes the link work CreateAndWriteFile(fileAbs); wrapper.GetFileStatus(link); }
public virtual void TestRunCommandWithNoResources() { // Windows only test Assume.AssumeTrue(Shell.Windows); Configuration conf = new Configuration(); string[] command = containerExecutor.GetRunCommand("echo", "group1", null, null, conf, Resource.NewInstance(1024, 1)); // Assert the cpu and memory limits are set correctly in the command string[] expected = new string[] { Shell.Winutils, "task", "create", "-m", "-1", "-c", "-1", "group1", "cmd /c " + "echo" }; NUnit.Framework.Assert.IsTrue(Arrays.Equals(expected, command)); }
/// <exception cref="System.Exception"/> public virtual void TestInstall() { Assume.AssumeTrue(SystemUtils.IsOsUnix); SignalLogger.Instance.Register(Log); try { SignalLogger.Instance.Register(Log); NUnit.Framework.Assert.Fail("expected IllegalStateException from double registration" ); } catch (InvalidOperationException) { } }
public virtual void TestCopyFileFromWindowsLocalPath() { Assume.AssumeTrue(Path.Windows); string windowsTestRootPath = (new FilePath(testRootDir.ToUri().GetPath().ToString ())).GetAbsolutePath(); Path testRoot = new Path(windowsTestRootPath, "testPutFile"); lfs.Delete(testRoot, true); lfs.Mkdirs(testRoot); Path targetDir = new Path(testRoot, "target"); Path filePath = new Path(testRoot, new Path("srcFile")); lfs.Create(filePath).Close(); CheckPut(filePath, targetDir, true); }
public virtual void TestAddDenseToSparse(double[] dense1, int sparseIndex, double v) { Assume.AssumeTrue(sparseIndex >= 0); Assume.AssumeTrue(sparseIndex <= 100); ConcatVector v1 = new ConcatVector(1); v1.SetDenseComponent(0, dense1); ConcatVector v2 = new ConcatVector(1); v2.SetSparseComponent(0, (int)sparseIndex, v); double expected = v1.DotProduct(v2) + 0.7f * (v1.DotProduct(v1)); v2.AddVectorInPlace(v1, 0.7f); NUnit.Framework.Assert.AreEqual(v2.DotProduct(v1), 5.0e-4, expected); }