public virtual void TestCheckSequential() { DFSClient dfsClient = Org.Mockito.Mockito.Mock <DFSClient>(); Nfs3FileAttributes attr = new Nfs3FileAttributes(); HdfsDataOutputStream fos = Org.Mockito.Mockito.Mock <HdfsDataOutputStream>(); Org.Mockito.Mockito.When(fos.GetPos()).ThenReturn((long)0); NfsConfiguration config = new NfsConfiguration(); config.SetBoolean(NfsConfigKeys.LargeFileUpload, false); OpenFileCtx ctx = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, new ShellBasedIdMapping (config), false, config); ctx.GetPendingWritesForTest()[new OffsetRange(5, 10)] = new WriteCtx(null, 0, 0, 0, null, null, null, 0, false, null); ctx.GetPendingWritesForTest()[new OffsetRange(10, 15)] = new WriteCtx(null, 0, 0, 0, null, null, null, 0, false, null); ctx.GetPendingWritesForTest()[new OffsetRange(20, 25)] = new WriteCtx(null, 0, 0, 0, null, null, null, 0, false, null); NUnit.Framework.Assert.IsTrue(!ctx.CheckSequential(5, 4)); NUnit.Framework.Assert.IsTrue(ctx.CheckSequential(9, 5)); NUnit.Framework.Assert.IsTrue(ctx.CheckSequential(10, 5)); NUnit.Framework.Assert.IsTrue(ctx.CheckSequential(14, 5)); NUnit.Framework.Assert.IsTrue(!ctx.CheckSequential(15, 5)); NUnit.Framework.Assert.IsTrue(!ctx.CheckSequential(20, 5)); NUnit.Framework.Assert.IsTrue(!ctx.CheckSequential(25, 5)); NUnit.Framework.Assert.IsTrue(!ctx.CheckSequential(999, 5)); }
public virtual void TestCheckCommitAixCompatMode() { DFSClient dfsClient = Org.Mockito.Mockito.Mock <DFSClient>(); Nfs3FileAttributes attr = new Nfs3FileAttributes(); HdfsDataOutputStream fos = Org.Mockito.Mockito.Mock <HdfsDataOutputStream>(); NfsConfiguration conf = new NfsConfiguration(); conf.SetBoolean(NfsConfigKeys.LargeFileUpload, false); // Enable AIX compatibility mode. OpenFileCtx ctx = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, new ShellBasedIdMapping (new NfsConfiguration()), true, conf); // Test fall-through to pendingWrites check in the event that commitOffset // is greater than the number of bytes we've so far flushed. Org.Mockito.Mockito.When(fos.GetPos()).ThenReturn((long)2); OpenFileCtx.COMMIT_STATUS status = ctx.CheckCommitInternal(5, null, 1, attr, false ); NUnit.Framework.Assert.IsTrue(status == OpenFileCtx.COMMIT_STATUS.CommitFinished); // Test the case when we actually have received more bytes than we're trying // to commit. ctx.GetPendingWritesForTest()[new OffsetRange(0, 10)] = new WriteCtx(null, 0, 0, 0, null, null, null, 0, false, null); Org.Mockito.Mockito.When(fos.GetPos()).ThenReturn((long)10); ctx.SetNextOffsetForTest((long)10); status = ctx.CheckCommitInternal(5, null, 1, attr, false); NUnit.Framework.Assert.IsTrue(status == OpenFileCtx.COMMIT_STATUS.CommitDoSync); }
public virtual void TestDeprecatedKeys() { NfsConfiguration conf = new NfsConfiguration(); conf.SetInt("nfs3.server.port", 998); NUnit.Framework.Assert.IsTrue(conf.GetInt(NfsConfigKeys.DfsNfsServerPortKey, 0) == 998); conf.SetInt("nfs3.mountd.port", 999); NUnit.Framework.Assert.IsTrue(conf.GetInt(NfsConfigKeys.DfsNfsMountdPortKey, 0) == 999); conf.Set("dfs.nfs.exports.allowed.hosts", "host1"); NUnit.Framework.Assert.IsTrue(conf.Get(CommonConfigurationKeys.NfsExportsAllowedHostsKey ).Equals("host1")); conf.SetInt("dfs.nfs.exports.cache.expirytime.millis", 1000); NUnit.Framework.Assert.IsTrue(conf.GetInt(Nfs3Constant.NfsExportsCacheExpirytimeMillisKey , 0) == 1000); conf.SetInt("hadoop.nfs.userupdate.milly", 10); NUnit.Framework.Assert.IsTrue(conf.GetInt(IdMappingConstant.UsergroupidUpdateMillisKey , 0) == 10); conf.Set("dfs.nfs3.dump.dir", "/nfs/tmp"); NUnit.Framework.Assert.IsTrue(conf.Get(NfsConfigKeys.DfsNfsFileDumpDirKey).Equals ("/nfs/tmp")); conf.SetBoolean("dfs.nfs3.enableDump", false); NUnit.Framework.Assert.IsTrue(conf.GetBoolean(NfsConfigKeys.DfsNfsFileDumpKey, true ) == false); conf.SetInt("dfs.nfs3.max.open.files", 500); NUnit.Framework.Assert.IsTrue(conf.GetInt(NfsConfigKeys.DfsNfsMaxOpenFilesKey, 0) == 500); conf.SetInt("dfs.nfs3.stream.timeout", 6000); NUnit.Framework.Assert.IsTrue(conf.GetInt(NfsConfigKeys.DfsNfsStreamTimeoutKey, 0 ) == 6000); conf.Set("dfs.nfs3.export.point", "/dir1"); NUnit.Framework.Assert.IsTrue(conf.Get(NfsConfigKeys.DfsNfsExportPointKey).Equals ("/dir1")); }
public virtual void TestExportPoint() { NfsConfiguration config = new NfsConfiguration(); MiniDFSCluster cluster = null; string exportPoint = "/myexport1"; config.SetStrings(NfsConfigKeys.DfsNfsExportPointKey, exportPoint); // Use emphral port in case tests are running in parallel config.SetInt("nfs3.mountd.port", 0); config.SetInt("nfs3.server.port", 0); try { cluster = new MiniDFSCluster.Builder(config).NumDataNodes(1).Build(); cluster.WaitActive(); // Start nfs Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3 nfsServer = new Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3 (config); nfsServer.StartServiceInternal(false); Mountd mountd = nfsServer.GetMountd(); RpcProgramMountd rpcMount = (RpcProgramMountd)mountd.GetRpcProgram(); NUnit.Framework.Assert.IsTrue(rpcMount.GetExports().Count == 1); string exportInMountd = rpcMount.GetExports()[0]; NUnit.Framework.Assert.IsTrue(exportInMountd.Equals(exportPoint)); } finally { if (cluster != null) { cluster.Shutdown(); } } }
/// <exception cref="System.IO.IOException"/> public Nfs3(NfsConfiguration conf, DatagramSocket registrationSocket, bool allowInsecurePorts ) : base(RpcProgramNfs3.CreateRpcProgramNfs3(conf, registrationSocket, allowInsecurePorts ), conf) { mountd = new Mountd(conf, registrationSocket, allowInsecurePorts); }
/// <exception cref="System.Exception"/> public static void Main(string[] args) { Arrays.Fill(data1, unchecked ((byte)7)); Arrays.Fill(data2, unchecked ((byte)8)); Arrays.Fill(data3, unchecked ((byte)9)); // NFS3 Create request NfsConfiguration conf = new NfsConfiguration(); TestOutOfOrderWrite.WriteClient client = new TestOutOfOrderWrite.WriteClient("localhost" , conf.GetInt(NfsConfigKeys.DfsNfsServerPortKey, NfsConfigKeys.DfsNfsServerPortDefault ), Create(), false); client.Run(); while (handle == null) { Sharpen.Thread.Sleep(1000); System.Console.Out.WriteLine("handle is still null..."); } Log.Info("Send write1 request"); XDR writeReq; writeReq = Write(handle, unchecked ((int)(0x8000005c)), 2000, 1000, data3); Nfs3Utils.WriteChannel(channel, writeReq, 1); writeReq = Write(handle, unchecked ((int)(0x8000005d)), 1000, 1000, data2); Nfs3Utils.WriteChannel(channel, writeReq, 2); writeReq = Write(handle, unchecked ((int)(0x8000005e)), 0, 1000, data1); Nfs3Utils.WriteChannel(channel, writeReq, 3); }
/// <exception cref="System.IO.IOException"/> public static void Main(string[] args) { NfsConfiguration config = new NfsConfiguration(); Org.Apache.Hadoop.Hdfs.Nfs.Mount.Mountd mountd = new Org.Apache.Hadoop.Hdfs.Nfs.Mount.Mountd (config, null, true); mountd.Start(true); }
internal OpenFileCtxCache(NfsConfiguration config, long streamTimeout) { // Insert and delete with openFileMap are synced maxStreams = config.GetInt(NfsConfigKeys.DfsNfsMaxOpenFilesKey, NfsConfigKeys.DfsNfsMaxOpenFilesDefault ); Log.Info("Maximum open streams is " + maxStreams); this.streamTimeout = streamTimeout; streamMonitor = new OpenFileCtxCache.StreamMonitor(this); }
internal DFSClientCache(NfsConfiguration config, int clientCache) { this.config = config; this.clientCache = CacheBuilder.NewBuilder().MaximumSize(clientCache).RemovalListener (ClientRemovalListener()).Build(ClientLoader()); this.inputstreamCache = CacheBuilder.NewBuilder().MaximumSize(DefaultDfsInputstreamCacheSize ).ExpireAfterAccess(DefaultDfsInputstreamCacheTtl, TimeUnit.Seconds).RemovalListener (InputStreamRemovalListener()).Build(InputStreamLoader()); ShutdownHookManager.Get().AddShutdownHook(new DFSClientCache.CacheFinalizer(this) , ShutdownHookPriority); }
public virtual void TestEviction() { NfsConfiguration conf = new NfsConfiguration(); // Only two entries will be in the cache conf.SetInt(NfsConfigKeys.DfsNfsMaxOpenFilesKey, 2); DFSClient dfsClient = Org.Mockito.Mockito.Mock <DFSClient>(); Nfs3FileAttributes attr = new Nfs3FileAttributes(); HdfsDataOutputStream fos = Org.Mockito.Mockito.Mock <HdfsDataOutputStream>(); Org.Mockito.Mockito.When(fos.GetPos()).ThenReturn((long)0); OpenFileCtx context1 = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, new ShellBasedIdMapping(new NfsConfiguration())); OpenFileCtx context2 = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, new ShellBasedIdMapping(new NfsConfiguration())); OpenFileCtx context3 = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, new ShellBasedIdMapping(new NfsConfiguration())); OpenFileCtx context4 = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, new ShellBasedIdMapping(new NfsConfiguration())); OpenFileCtx context5 = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, new ShellBasedIdMapping(new NfsConfiguration())); OpenFileCtxCache cache = new OpenFileCtxCache(conf, 10 * 60 * 100); bool ret = cache.Put(new FileHandle(1), context1); NUnit.Framework.Assert.IsTrue(ret); Sharpen.Thread.Sleep(1000); ret = cache.Put(new FileHandle(2), context2); NUnit.Framework.Assert.IsTrue(ret); ret = cache.Put(new FileHandle(3), context3); NUnit.Framework.Assert.IsFalse(ret); NUnit.Framework.Assert.IsTrue(cache.Size() == 2); // Wait for the oldest stream to be evict-able, insert again Sharpen.Thread.Sleep(NfsConfigKeys.DfsNfsStreamTimeoutMinDefault); NUnit.Framework.Assert.IsTrue(cache.Size() == 2); ret = cache.Put(new FileHandle(3), context3); NUnit.Framework.Assert.IsTrue(ret); NUnit.Framework.Assert.IsTrue(cache.Size() == 2); NUnit.Framework.Assert.IsTrue(cache.Get(new FileHandle(1)) == null); // Test inactive entry is evicted immediately context3.SetActiveStatusForTest(false); ret = cache.Put(new FileHandle(4), context4); NUnit.Framework.Assert.IsTrue(ret); // Now the cache has context2 and context4 // Test eviction failure if all entries have pending work. context2.GetPendingWritesForTest()[new OffsetRange(0, 100)] = new WriteCtx(null, 0, 0, 0, null, null, null, 0, false, null); context4.GetPendingCommitsForTest()[System.Convert.ToInt64(100)] = new OpenFileCtx.CommitCtx (0, null, 0, attr); Sharpen.Thread.Sleep(NfsConfigKeys.DfsNfsStreamTimeoutMinDefault); ret = cache.Put(new FileHandle(5), context5); NUnit.Framework.Assert.IsFalse(ret); }
/// <exception cref="System.IO.IOException"/> internal static void StartService(string[] args, DatagramSocket registrationSocket ) { StringUtils.StartupShutdownMessage(typeof(Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3), args, Log); NfsConfiguration conf = new NfsConfiguration(); bool allowInsecurePorts = conf.GetBoolean(NfsConfigKeys.DfsNfsPortMonitoringDisabledKey , NfsConfigKeys.DfsNfsPortMonitoringDisabledDefault); Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3 nfsServer = new Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3 (conf, registrationSocket, allowInsecurePorts); nfsServer.StartServiceInternal(true); }
/// <exception cref="System.Exception"/> public virtual void Init(DaemonContext context) { System.Console.Error.WriteLine("Initializing privileged NFS client socket..."); NfsConfiguration conf = new NfsConfiguration(); int clientPort = conf.GetInt(NfsConfigKeys.DfsNfsRegistrationPortKey, NfsConfigKeys .DfsNfsRegistrationPortDefault); if (clientPort < 1 || clientPort > 1023) { throw new RuntimeException("Must start privileged NFS server with '" + NfsConfigKeys .DfsNfsRegistrationPortKey + "' configured to a " + "privileged port."); } registrationSocket = new DatagramSocket(new IPEndPoint("localhost", clientPort)); registrationSocket.SetReuseAddress(true); args = context.GetArguments(); }
/// <exception cref="System.IO.IOException"/> public RpcProgramMountd(NfsConfiguration config, DatagramSocket registrationSocket , bool allowInsecurePorts) : base("mountd", "localhost", config.GetInt(NfsConfigKeys.DfsNfsMountdPortKey, NfsConfigKeys .DfsNfsMountdPortDefault), Program, Version1, Version3, registrationSocket, allowInsecurePorts ) { // Note that RPC cache is not enabled exports = new AList <string>(); exports.AddItem(config.Get(NfsConfigKeys.DfsNfsExportPointKey, NfsConfigKeys.DfsNfsExportPointDefault )); this.hostsMatcher = NfsExports.GetInstance(config); this.mounts = Sharpen.Collections.SynchronizedList(new AList <MountEntry>()); UserGroupInformation.SetConfiguration(config); SecurityUtil.Login(config, NfsConfigKeys.DfsNfsKeytabFileKey, NfsConfigKeys.DfsNfsKerberosPrincipalKey ); this.dfsClient = new DFSClient(NameNode.GetAddress(config), config); }
public virtual void TestGetUserGroupInformation() { string userName = "******"; string currentUser = "******"; UserGroupInformation currentUserUgi = UserGroupInformation.CreateUserForTesting(currentUser , new string[0]); NfsConfiguration conf = new NfsConfiguration(); conf.Set(FileSystem.FsDefaultNameKey, "hdfs://localhost"); DFSClientCache cache = new DFSClientCache(conf); UserGroupInformation ugiResult = cache.GetUserGroupInformation(userName, currentUserUgi ); Assert.AssertThat(ugiResult.GetUserName(), IS.Is(userName)); Assert.AssertThat(ugiResult.GetRealUser(), IS.Is(currentUserUgi)); Assert.AssertThat(ugiResult.GetAuthenticationMethod(), IS.Is(UserGroupInformation.AuthenticationMethod .Proxy)); }
public virtual void TestEviction() { NfsConfiguration conf = new NfsConfiguration(); conf.Set(FileSystem.FsDefaultNameKey, "hdfs://localhost"); // Only one entry will be in the cache int MaxCacheSize = 1; DFSClientCache cache = new DFSClientCache(conf, MaxCacheSize); DFSClient c1 = cache.GetDfsClient("test1"); NUnit.Framework.Assert.IsTrue(cache.GetDfsClient("test1").ToString().Contains("ugi=test1" )); NUnit.Framework.Assert.AreEqual(c1, cache.GetDfsClient("test1")); NUnit.Framework.Assert.IsFalse(IsDfsClientClose(c1)); cache.GetDfsClient("test2"); NUnit.Framework.Assert.IsTrue(IsDfsClientClose(c1)); NUnit.Framework.Assert.IsTrue("cache size should be the max size or less", cache. clientCache.Size() <= MaxCacheSize); }
internal WriteManager(IdMappingServiceProvider iug, NfsConfiguration config, bool aixCompatMode) { this.iug = iug; this.config = config; this.aixCompatMode = aixCompatMode; streamTimeout = config.GetLong(NfsConfigKeys.DfsNfsStreamTimeoutKey, NfsConfigKeys .DfsNfsStreamTimeoutDefault); Log.Info("Stream timeout is " + streamTimeout + "ms."); if (streamTimeout < NfsConfigKeys.DfsNfsStreamTimeoutMinDefault) { Log.Info("Reset stream timeout to minimum value " + NfsConfigKeys.DfsNfsStreamTimeoutMinDefault + "ms."); streamTimeout = NfsConfigKeys.DfsNfsStreamTimeoutMinDefault; } maxStreams = config.GetInt(NfsConfigKeys.DfsNfsMaxOpenFilesKey, NfsConfigKeys.DfsNfsMaxOpenFilesDefault ); Log.Info("Maximum open streams is " + maxStreams); this.fileContextCache = new OpenFileCtxCache(config, streamTimeout); }
public virtual void TestGetUserGroupInformationSecure() { string userName = "******"; string currentUser = "******"; NfsConfiguration conf = new NfsConfiguration(); UserGroupInformation currentUserUgi = UserGroupInformation.CreateRemoteUser(currentUser ); currentUserUgi.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod. Kerberos); UserGroupInformation.SetLoginUser(currentUserUgi); DFSClientCache cache = new DFSClientCache(conf); UserGroupInformation ugiResult = cache.GetUserGroupInformation(userName, currentUserUgi ); Assert.AssertThat(ugiResult.GetUserName(), IS.Is(userName)); Assert.AssertThat(ugiResult.GetRealUser(), IS.Is(currentUserUgi)); Assert.AssertThat(ugiResult.GetAuthenticationMethod(), IS.Is(UserGroupInformation.AuthenticationMethod .Proxy)); }
public virtual void TestScan() { NfsConfiguration conf = new NfsConfiguration(); // Only two entries will be in the cache conf.SetInt(NfsConfigKeys.DfsNfsMaxOpenFilesKey, 2); DFSClient dfsClient = Org.Mockito.Mockito.Mock <DFSClient>(); Nfs3FileAttributes attr = new Nfs3FileAttributes(); HdfsDataOutputStream fos = Org.Mockito.Mockito.Mock <HdfsDataOutputStream>(); Org.Mockito.Mockito.When(fos.GetPos()).ThenReturn((long)0); OpenFileCtx context1 = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, new ShellBasedIdMapping(new NfsConfiguration())); OpenFileCtx context2 = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, new ShellBasedIdMapping(new NfsConfiguration())); OpenFileCtx context3 = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, new ShellBasedIdMapping(new NfsConfiguration())); OpenFileCtx context4 = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, new ShellBasedIdMapping(new NfsConfiguration())); OpenFileCtxCache cache = new OpenFileCtxCache(conf, 10 * 60 * 100); // Test cleaning expired entry bool ret = cache.Put(new FileHandle(1), context1); NUnit.Framework.Assert.IsTrue(ret); ret = cache.Put(new FileHandle(2), context2); NUnit.Framework.Assert.IsTrue(ret); Sharpen.Thread.Sleep(NfsConfigKeys.DfsNfsStreamTimeoutMinDefault + 1); cache.Scan(NfsConfigKeys.DfsNfsStreamTimeoutMinDefault); NUnit.Framework.Assert.IsTrue(cache.Size() == 0); // Test cleaning inactive entry ret = cache.Put(new FileHandle(3), context3); NUnit.Framework.Assert.IsTrue(ret); ret = cache.Put(new FileHandle(4), context4); NUnit.Framework.Assert.IsTrue(ret); context3.SetActiveStatusForTest(false); cache.Scan(NfsConfigKeys.DfsNfsStreamTimeoutDefault); NUnit.Framework.Assert.IsTrue(cache.Size() == 1); NUnit.Framework.Assert.IsTrue(cache.Get(new FileHandle(3)) == null); NUnit.Framework.Assert.IsTrue(cache.Get(new FileHandle(4)) != null); }
public virtual void TestStart() { // Start minicluster NfsConfiguration config = new NfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(config).NumDataNodes(1).Build (); cluster.WaitActive(); // Use emphral port in case tests are running in parallel config.SetInt("nfs3.mountd.port", 0); config.SetInt("nfs3.server.port", 0); // Start nfs Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3 nfs3 = new Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3 (config); nfs3.StartServiceInternal(false); RpcProgramMountd mountd = (RpcProgramMountd)nfs3.GetMountd().GetRpcProgram(); mountd.NullOp(new XDR(), 1234, Sharpen.Extensions.GetAddressByName("localhost")); RpcProgramNfs3 nfsd = (RpcProgramNfs3)nfs3.GetRpcProgram(); nfsd.NullProcedure(); cluster.Shutdown(); }
public virtual void TestCheckCommitLargeFileUpload() { // Validate all the commit check return codes OpenFileCtx.COMMIT_STATUS with // large file upload option. DFSClient dfsClient = Org.Mockito.Mockito.Mock <DFSClient>(); Nfs3FileAttributes attr = new Nfs3FileAttributes(); HdfsDataOutputStream fos = Org.Mockito.Mockito.Mock <HdfsDataOutputStream>(); Org.Mockito.Mockito.When(fos.GetPos()).ThenReturn((long)0); NfsConfiguration conf = new NfsConfiguration(); conf.SetBoolean(NfsConfigKeys.LargeFileUpload, true); OpenFileCtx ctx = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, new ShellBasedIdMapping (conf), false, conf); OpenFileCtx.COMMIT_STATUS ret; // Test inactive open file context ctx.SetActiveStatusForTest(false); Org.Jboss.Netty.Channel.Channel ch = Org.Mockito.Mockito.Mock <Org.Jboss.Netty.Channel.Channel >(); ret = ctx.CheckCommit(dfsClient, 0, ch, 1, attr, false); NUnit.Framework.Assert.IsTrue(ret == OpenFileCtx.COMMIT_STATUS.CommitInactiveCtx); ctx.GetPendingWritesForTest()[new OffsetRange(10, 15)] = new WriteCtx(null, 0, 0, 0, null, null, null, 0, false, null); ret = ctx.CheckCommit(dfsClient, 0, ch, 1, attr, false); NUnit.Framework.Assert.IsTrue(ret == OpenFileCtx.COMMIT_STATUS.CommitInactiveWithPendingWrite ); // Test request with non zero commit offset ctx.SetActiveStatusForTest(true); Org.Mockito.Mockito.When(fos.GetPos()).ThenReturn((long)8); ctx.SetNextOffsetForTest(10); OpenFileCtx.COMMIT_STATUS status = ctx.CheckCommitInternal(5, null, 1, attr, false ); NUnit.Framework.Assert.IsTrue(status == OpenFileCtx.COMMIT_STATUS.CommitDoSync); // Do_SYNC state will be updated to FINISHED after data sync ret = ctx.CheckCommit(dfsClient, 5, ch, 1, attr, false); NUnit.Framework.Assert.IsTrue(ret == OpenFileCtx.COMMIT_STATUS.CommitFinished); // Test commit sequential writes status = ctx.CheckCommitInternal(10, ch, 1, attr, false); NUnit.Framework.Assert.IsTrue(status == OpenFileCtx.COMMIT_STATUS.CommitSpecialWait ); ret = ctx.CheckCommit(dfsClient, 10, ch, 1, attr, false); NUnit.Framework.Assert.IsTrue(ret == OpenFileCtx.COMMIT_STATUS.CommitSpecialWait); // Test commit non-sequential writes ConcurrentNavigableMap <long, OpenFileCtx.CommitCtx> commits = ctx.GetPendingCommitsForTest (); NUnit.Framework.Assert.IsTrue(commits.Count == 1); ret = ctx.CheckCommit(dfsClient, 16, ch, 1, attr, false); NUnit.Framework.Assert.IsTrue(ret == OpenFileCtx.COMMIT_STATUS.CommitSpecialSuccess ); NUnit.Framework.Assert.IsTrue(commits.Count == 1); // Test request with zero commit offset Sharpen.Collections.Remove(commits, System.Convert.ToInt64(10)); // There is one pending write [10,15] ret = ctx.CheckCommitInternal(0, ch, 1, attr, false); NUnit.Framework.Assert.IsTrue(ret == OpenFileCtx.COMMIT_STATUS.CommitSpecialWait); ret = ctx.CheckCommitInternal(9, ch, 1, attr, false); NUnit.Framework.Assert.IsTrue(ret == OpenFileCtx.COMMIT_STATUS.CommitSpecialWait); NUnit.Framework.Assert.IsTrue(commits.Count == 2); // Empty pending writes. nextOffset=10, flushed pos=8 Sharpen.Collections.Remove(ctx.GetPendingWritesForTest(), new OffsetRange(10, 15) ); ret = ctx.CheckCommit(dfsClient, 0, ch, 1, attr, false); NUnit.Framework.Assert.IsTrue(ret == OpenFileCtx.COMMIT_STATUS.CommitSpecialWait); // Empty pending writes ctx.SetNextOffsetForTest((long)8); // flushed pos = 8 ret = ctx.CheckCommit(dfsClient, 0, ch, 1, attr, false); NUnit.Framework.Assert.IsTrue(ret == OpenFileCtx.COMMIT_STATUS.CommitFinished); }
public virtual void TestCheckCommitFromReadLargeFileUpload() { // Validate all the commit check return codes OpenFileCtx.COMMIT_STATUS with large file upload option DFSClient dfsClient = Org.Mockito.Mockito.Mock <DFSClient>(); Nfs3FileAttributes attr = new Nfs3FileAttributes(); HdfsDataOutputStream fos = Org.Mockito.Mockito.Mock <HdfsDataOutputStream>(); Org.Mockito.Mockito.When(fos.GetPos()).ThenReturn((long)0); NfsConfiguration config = new NfsConfiguration(); config.SetBoolean(NfsConfigKeys.LargeFileUpload, true); OpenFileCtx ctx = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, new ShellBasedIdMapping (config), false, config); FileHandle h = new FileHandle(1); // fake handle for "/dumpFilePath" OpenFileCtx.COMMIT_STATUS ret; WriteManager wm = new WriteManager(new ShellBasedIdMapping(config), config, false ); NUnit.Framework.Assert.IsTrue(wm.AddOpenFileStream(h, ctx)); // Test inactive open file context ctx.SetActiveStatusForTest(false); Org.Jboss.Netty.Channel.Channel ch = Org.Mockito.Mockito.Mock <Org.Jboss.Netty.Channel.Channel >(); ret = ctx.CheckCommit(dfsClient, 0, ch, 1, attr, true); NUnit.Framework.Assert.AreEqual(OpenFileCtx.COMMIT_STATUS.CommitInactiveCtx, ret); NUnit.Framework.Assert.AreEqual(Nfs3Status.Nfs3Ok, wm.CommitBeforeRead(dfsClient, h, 0)); ctx.GetPendingWritesForTest()[new OffsetRange(10, 15)] = new WriteCtx(null, 0, 0, 0, null, null, null, 0, false, null); ret = ctx.CheckCommit(dfsClient, 0, ch, 1, attr, true); NUnit.Framework.Assert.AreEqual(OpenFileCtx.COMMIT_STATUS.CommitInactiveWithPendingWrite , ret); NUnit.Framework.Assert.AreEqual(Nfs3Status.Nfs3errIo, wm.CommitBeforeRead(dfsClient , h, 0)); // Test request with non zero commit offset ctx.SetActiveStatusForTest(true); Org.Mockito.Mockito.When(fos.GetPos()).ThenReturn((long)6); ctx.SetNextOffsetForTest((long)10); OpenFileCtx.COMMIT_STATUS status = ctx.CheckCommitInternal(5, ch, 1, attr, false); NUnit.Framework.Assert.AreEqual(OpenFileCtx.COMMIT_STATUS.CommitDoSync, status); // Do_SYNC state will be updated to FINISHED after data sync ret = ctx.CheckCommit(dfsClient, 5, ch, 1, attr, true); NUnit.Framework.Assert.AreEqual(OpenFileCtx.COMMIT_STATUS.CommitFinished, ret); NUnit.Framework.Assert.AreEqual(Nfs3Status.Nfs3Ok, wm.CommitBeforeRead(dfsClient, h, 5)); // Test request with sequential writes status = ctx.CheckCommitInternal(9, ch, 1, attr, true); NUnit.Framework.Assert.IsTrue(status == OpenFileCtx.COMMIT_STATUS.CommitSpecialWait ); ret = ctx.CheckCommit(dfsClient, 9, ch, 1, attr, true); NUnit.Framework.Assert.AreEqual(OpenFileCtx.COMMIT_STATUS.CommitSpecialWait, ret); NUnit.Framework.Assert.AreEqual(Nfs3Status.Nfs3errJukebox, wm.CommitBeforeRead(dfsClient , h, 9)); // Test request with non-sequential writes ConcurrentNavigableMap <long, OpenFileCtx.CommitCtx> commits = ctx.GetPendingCommitsForTest (); NUnit.Framework.Assert.IsTrue(commits.Count == 0); ret = ctx.CheckCommit(dfsClient, 16, ch, 1, attr, true); NUnit.Framework.Assert.AreEqual(OpenFileCtx.COMMIT_STATUS.CommitSpecialSuccess, ret ); NUnit.Framework.Assert.AreEqual(0, commits.Count); // commit triggered by read doesn't wait NUnit.Framework.Assert.AreEqual(Nfs3Status.Nfs3Ok, wm.CommitBeforeRead(dfsClient, h, 16)); // Test request with zero commit offset // There is one pending write [10,15] ret = ctx.CheckCommit(dfsClient, 0, ch, 1, attr, true); NUnit.Framework.Assert.AreEqual(OpenFileCtx.COMMIT_STATUS.CommitSpecialWait, ret); NUnit.Framework.Assert.AreEqual(0, commits.Count); NUnit.Framework.Assert.AreEqual(Nfs3Status.Nfs3errJukebox, wm.CommitBeforeRead(dfsClient , h, 0)); // Empty pending writes Sharpen.Collections.Remove(ctx.GetPendingWritesForTest(), new OffsetRange(10, 15) ); ret = ctx.CheckCommit(dfsClient, 0, ch, 1, attr, true); NUnit.Framework.Assert.AreEqual(OpenFileCtx.COMMIT_STATUS.CommitSpecialWait, ret); NUnit.Framework.Assert.AreEqual(Nfs3Status.Nfs3errJukebox, wm.CommitBeforeRead(dfsClient , h, 0)); }
public virtual void TestWriteStableHow() { NfsConfiguration config = new NfsConfiguration(); DFSClient client = null; MiniDFSCluster cluster = null; RpcProgramNfs3 nfsd; SecurityHandler securityHandler = Org.Mockito.Mockito.Mock <SecurityHandler>(); Org.Mockito.Mockito.When(securityHandler.GetUser()).ThenReturn(Runtime.GetProperty ("user.name")); string currentUser = Runtime.GetProperty("user.name"); config.Set(DefaultImpersonationProvider.GetTestProvider().GetProxySuperuserGroupConfKey (currentUser), "*"); config.Set(DefaultImpersonationProvider.GetTestProvider().GetProxySuperuserIpConfKey (currentUser), "*"); ProxyUsers.RefreshSuperUserGroupsConfiguration(config); try { cluster = new MiniDFSCluster.Builder(config).NumDataNodes(1).Build(); cluster.WaitActive(); client = new DFSClient(NameNode.GetAddress(config), config); // Use emphral port in case tests are running in parallel config.SetInt("nfs3.mountd.port", 0); config.SetInt("nfs3.server.port", 0); // Start nfs Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3 nfs3 = new Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3 (config); nfs3.StartServiceInternal(false); nfsd = (RpcProgramNfs3)nfs3.GetRpcProgram(); HdfsFileStatus status = client.GetFileInfo("/"); FileHandle rootHandle = new FileHandle(status.GetFileId()); // Create file1 CREATE3Request createReq = new CREATE3Request(rootHandle, "file1", Nfs3Constant.CreateUnchecked , new SetAttr3(), 0); XDR createXdr = new XDR(); createReq.Serialize(createXdr); CREATE3Response createRsp = nfsd.Create(createXdr.AsReadOnlyWrap(), securityHandler , new IPEndPoint("localhost", 1234)); FileHandle handle = createRsp.GetObjHandle(); // Test DATA_SYNC byte[] buffer = new byte[10]; for (int i = 0; i < 10; i++) { buffer[i] = unchecked ((byte)i); } WRITE3Request writeReq = new WRITE3Request(handle, 0, 10, Nfs3Constant.WriteStableHow .DataSync, ByteBuffer.Wrap(buffer)); XDR writeXdr = new XDR(); writeReq.Serialize(writeXdr); nfsd.Write(writeXdr.AsReadOnlyWrap(), null, 1, securityHandler, new IPEndPoint("localhost" , 1234)); WaitWrite(nfsd, handle, 60000); // Readback READ3Request readReq = new READ3Request(handle, 0, 10); XDR readXdr = new XDR(); readReq.Serialize(readXdr); READ3Response readRsp = nfsd.Read(readXdr.AsReadOnlyWrap(), securityHandler, new IPEndPoint("localhost", 1234)); NUnit.Framework.Assert.IsTrue(Arrays.Equals(buffer, ((byte[])readRsp.GetData().Array ()))); // Test FILE_SYNC // Create file2 CREATE3Request createReq2 = new CREATE3Request(rootHandle, "file2", Nfs3Constant. CreateUnchecked, new SetAttr3(), 0); XDR createXdr2 = new XDR(); createReq2.Serialize(createXdr2); CREATE3Response createRsp2 = nfsd.Create(createXdr2.AsReadOnlyWrap(), securityHandler , new IPEndPoint("localhost", 1234)); FileHandle handle2 = createRsp2.GetObjHandle(); WRITE3Request writeReq2 = new WRITE3Request(handle2, 0, 10, Nfs3Constant.WriteStableHow .FileSync, ByteBuffer.Wrap(buffer)); XDR writeXdr2 = new XDR(); writeReq2.Serialize(writeXdr2); nfsd.Write(writeXdr2.AsReadOnlyWrap(), null, 1, securityHandler, new IPEndPoint("localhost" , 1234)); WaitWrite(nfsd, handle2, 60000); // Readback READ3Request readReq2 = new READ3Request(handle2, 0, 10); XDR readXdr2 = new XDR(); readReq2.Serialize(readXdr2); READ3Response readRsp2 = nfsd.Read(readXdr2.AsReadOnlyWrap(), securityHandler, new IPEndPoint("localhost", 1234)); NUnit.Framework.Assert.IsTrue(Arrays.Equals(buffer, ((byte[])readRsp2.GetData().Array ()))); // FILE_SYNC should sync the file size status = client.GetFileInfo("/file2"); NUnit.Framework.Assert.IsTrue(status.GetLen() == 10); } finally { if (cluster != null) { cluster.Shutdown(); } } }
internal DFSClientCache(NfsConfiguration config) : this(config, DefaultDfsClientCacheSize) { }
public virtual void TestOOOWrites() { NfsConfiguration config = new NfsConfiguration(); MiniDFSCluster cluster = null; RpcProgramNfs3 nfsd; int bufSize = 32; int numOOO = 3; SecurityHandler securityHandler = Org.Mockito.Mockito.Mock <SecurityHandler>(); Org.Mockito.Mockito.When(securityHandler.GetUser()).ThenReturn(Runtime.GetProperty ("user.name")); string currentUser = Runtime.GetProperty("user.name"); config.Set(DefaultImpersonationProvider.GetTestProvider().GetProxySuperuserGroupConfKey (currentUser), "*"); config.Set(DefaultImpersonationProvider.GetTestProvider().GetProxySuperuserIpConfKey (currentUser), "*"); ProxyUsers.RefreshSuperUserGroupsConfiguration(config); // Use emphral port in case tests are running in parallel config.SetInt("nfs3.mountd.port", 0); config.SetInt("nfs3.server.port", 0); try { cluster = new MiniDFSCluster.Builder(config).NumDataNodes(1).Build(); cluster.WaitActive(); Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3 nfs3 = new Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3 (config); nfs3.StartServiceInternal(false); nfsd = (RpcProgramNfs3)nfs3.GetRpcProgram(); DFSClient dfsClient = new DFSClient(NameNode.GetAddress(config), config); HdfsFileStatus status = dfsClient.GetFileInfo("/"); FileHandle rootHandle = new FileHandle(status.GetFileId()); CREATE3Request createReq = new CREATE3Request(rootHandle, "out-of-order-write" + Runtime.CurrentTimeMillis(), Nfs3Constant.CreateUnchecked, new SetAttr3(), 0); XDR createXdr = new XDR(); createReq.Serialize(createXdr); CREATE3Response createRsp = nfsd.Create(createXdr.AsReadOnlyWrap(), securityHandler , new IPEndPoint("localhost", 1234)); FileHandle handle = createRsp.GetObjHandle(); byte[][] oooBuf = new byte[][] { new byte[bufSize], new byte[bufSize], new byte[bufSize ] }; for (int i = 0; i < numOOO; i++) { Arrays.Fill(oooBuf[i], unchecked ((byte)i)); } for (int i_1 = 0; i_1 < numOOO; i_1++) { long offset = (numOOO - 1 - i_1) * bufSize; WRITE3Request writeReq = new WRITE3Request(handle, offset, bufSize, Nfs3Constant.WriteStableHow .Unstable, ByteBuffer.Wrap(oooBuf[i_1])); XDR writeXdr = new XDR(); writeReq.Serialize(writeXdr); nfsd.Write(writeXdr.AsReadOnlyWrap(), null, 1, securityHandler, new IPEndPoint("localhost" , 1234)); } WaitWrite(nfsd, handle, 60000); READ3Request readReq = new READ3Request(handle, bufSize, bufSize); XDR readXdr = new XDR(); readReq.Serialize(readXdr); READ3Response readRsp = nfsd.Read(readXdr.AsReadOnlyWrap(), securityHandler, new IPEndPoint("localhost", config.GetInt(NfsConfigKeys.DfsNfsServerPortKey, NfsConfigKeys .DfsNfsServerPortDefault))); NUnit.Framework.Assert.IsTrue(Arrays.Equals(oooBuf[1], ((byte[])readRsp.GetData() .Array()))); } finally { if (cluster != null) { cluster.Shutdown(); } } }
/// <exception cref="System.IO.IOException"/> public Mountd(NfsConfiguration config, DatagramSocket registrationSocket, bool allowInsecurePorts ) : base(new RpcProgramMountd(config, registrationSocket, allowInsecurePorts)) { }
public virtual void TestCheckCommit() { // Validate all the commit check return codes OpenFileCtx.COMMIT_STATUS, which // includes COMMIT_FINISHED, COMMIT_WAIT, COMMIT_INACTIVE_CTX, // COMMIT_INACTIVE_WITH_PENDING_WRITE, COMMIT_ERROR, and COMMIT_DO_SYNC. DFSClient dfsClient = Org.Mockito.Mockito.Mock <DFSClient>(); Nfs3FileAttributes attr = new Nfs3FileAttributes(); HdfsDataOutputStream fos = Org.Mockito.Mockito.Mock <HdfsDataOutputStream>(); Org.Mockito.Mockito.When(fos.GetPos()).ThenReturn((long)0); NfsConfiguration conf = new NfsConfiguration(); conf.SetBoolean(NfsConfigKeys.LargeFileUpload, false); OpenFileCtx ctx = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, new ShellBasedIdMapping (conf), false, conf); OpenFileCtx.COMMIT_STATUS ret; // Test inactive open file context ctx.SetActiveStatusForTest(false); Org.Jboss.Netty.Channel.Channel ch = Org.Mockito.Mockito.Mock <Org.Jboss.Netty.Channel.Channel >(); ret = ctx.CheckCommit(dfsClient, 0, ch, 1, attr, false); NUnit.Framework.Assert.IsTrue(ret == OpenFileCtx.COMMIT_STATUS.CommitInactiveCtx); ctx.GetPendingWritesForTest()[new OffsetRange(5, 10)] = new WriteCtx(null, 0, 0, 0, null, null, null, 0, false, null); ret = ctx.CheckCommit(dfsClient, 0, ch, 1, attr, false); NUnit.Framework.Assert.IsTrue(ret == OpenFileCtx.COMMIT_STATUS.CommitInactiveWithPendingWrite ); // Test request with non zero commit offset ctx.SetActiveStatusForTest(true); Org.Mockito.Mockito.When(fos.GetPos()).ThenReturn((long)10); ctx.SetNextOffsetForTest(10); OpenFileCtx.COMMIT_STATUS status = ctx.CheckCommitInternal(5, null, 1, attr, false ); NUnit.Framework.Assert.IsTrue(status == OpenFileCtx.COMMIT_STATUS.CommitDoSync); // Do_SYNC state will be updated to FINISHED after data sync ret = ctx.CheckCommit(dfsClient, 5, ch, 1, attr, false); NUnit.Framework.Assert.IsTrue(ret == OpenFileCtx.COMMIT_STATUS.CommitFinished); status = ctx.CheckCommitInternal(10, ch, 1, attr, false); NUnit.Framework.Assert.IsTrue(status == OpenFileCtx.COMMIT_STATUS.CommitDoSync); ret = ctx.CheckCommit(dfsClient, 10, ch, 1, attr, false); NUnit.Framework.Assert.IsTrue(ret == OpenFileCtx.COMMIT_STATUS.CommitFinished); ConcurrentNavigableMap <long, OpenFileCtx.CommitCtx> commits = ctx.GetPendingCommitsForTest (); NUnit.Framework.Assert.IsTrue(commits.Count == 0); ret = ctx.CheckCommit(dfsClient, 11, ch, 1, attr, false); NUnit.Framework.Assert.IsTrue(ret == OpenFileCtx.COMMIT_STATUS.CommitWait); NUnit.Framework.Assert.IsTrue(commits.Count == 1); long key = commits.FirstKey(); NUnit.Framework.Assert.IsTrue(key == 11); // Test request with zero commit offset Sharpen.Collections.Remove(commits, System.Convert.ToInt64(11)); // There is one pending write [5,10] ret = ctx.CheckCommit(dfsClient, 0, ch, 1, attr, false); NUnit.Framework.Assert.IsTrue(ret == OpenFileCtx.COMMIT_STATUS.CommitWait); NUnit.Framework.Assert.IsTrue(commits.Count == 1); key = commits.FirstKey(); NUnit.Framework.Assert.IsTrue(key == 9); // Empty pending writes Sharpen.Collections.Remove(ctx.GetPendingWritesForTest(), new OffsetRange(5, 10)); ret = ctx.CheckCommit(dfsClient, 0, ch, 1, attr, false); NUnit.Framework.Assert.IsTrue(ret == OpenFileCtx.COMMIT_STATUS.CommitFinished); }
internal Nfs3HttpServer(NfsConfiguration conf) { this.conf = conf; }
/// <exception cref="System.IO.IOException"/> public Nfs3(NfsConfiguration conf) : this(conf, null, true) { }