public virtual void TestUse_ExistingEntry_DupId() { // Pre-populate the SCM with one cache entry store.AddResource("key1", "foo.jar"); UserGroupInformation testUGI = UserGroupInformation.GetCurrentUser(); store.AddResourceReference("key1", new SharedCacheResourceReference(CreateAppId(1 , 1L), testUGI.GetShortUserName())); NUnit.Framework.Assert.AreEqual(1, store.GetResourceReferences("key1").Count); long hits = ClientSCMMetrics.GetInstance().GetCacheHits(); // Add a new duplicate appId UseSharedCacheResourceRequest request = recordFactory.NewRecordInstance <UseSharedCacheResourceRequest >(); request.SetResourceKey("key1"); request.SetAppId(CreateAppId(1, 1L)); // Expecting default depth of 3 under the shared cache root dir string expectedPath = testDir.GetAbsolutePath() + "/k/e/y/key1/foo.jar"; NUnit.Framework.Assert.AreEqual(expectedPath, clientSCMProxy.Use(request).GetPath ()); NUnit.Framework.Assert.AreEqual(1, store.GetResourceReferences("key1").Count); NUnit.Framework.Assert.AreEqual("Client SCM metrics aren't updated.", 1, ClientSCMMetrics .GetInstance().GetCacheHits() - hits); }
public SCMMetricsInfo(CleanerMetrics cleanerMetrics, ClientSCMMetrics clientSCMMetrics , SharedCacheUploaderMetrics scmUploaderMetrics) { totalDeletedFiles = cleanerMetrics.GetTotalDeletedFiles(); totalProcessedFiles = cleanerMetrics.GetTotalProcessedFiles(); cacheHits = clientSCMMetrics.GetCacheHits(); cacheMisses = clientSCMMetrics.GetCacheMisses(); cacheReleases = clientSCMMetrics.GetCacheReleases(); acceptedUploads = scmUploaderMetrics.GetAcceptedUploads(); rejectedUploads = scmUploaderMetrics.GetRejectUploads(); }
public virtual void TestUse_MissingEntry() { long misses = ClientSCMMetrics.GetInstance().GetCacheMisses(); UseSharedCacheResourceRequest request = recordFactory.NewRecordInstance <UseSharedCacheResourceRequest >(); request.SetResourceKey("key1"); request.SetAppId(CreateAppId(1, 1L)); NUnit.Framework.Assert.IsNull(clientSCMProxy.Use(request).GetPath()); NUnit.Framework.Assert.AreEqual("Client SCM metrics aren't updated.", 1, ClientSCMMetrics .GetInstance().GetCacheMisses() - misses); }
public virtual void TestRelease_MissingEntry() { long releases = ClientSCMMetrics.GetInstance().GetCacheReleases(); ReleaseSharedCacheResourceRequest request = recordFactory.NewRecordInstance <ReleaseSharedCacheResourceRequest >(); request.SetResourceKey("key2"); request.SetAppId(CreateAppId(2, 2L)); clientSCMProxy.Release(request); NUnit.Framework.Assert.IsNotNull(store.GetResourceReferences("key2")); NUnit.Framework.Assert.AreEqual(0, store.GetResourceReferences("key2").Count); NUnit.Framework.Assert.AreEqual("Client SCM metrics were updated when a release did not happen." , 0, ClientSCMMetrics.GetInstance().GetCacheReleases() - releases); }
/// <exception cref="System.Exception"/> protected override void ServiceStart() { Configuration conf = GetConfig(); this.metrics = ClientSCMMetrics.GetInstance(); YarnRPC rpc = YarnRPC.Create(conf); this.server = rpc.GetServer(typeof(ClientSCMProtocol), this, clientBindAddress, conf , null, conf.GetInt(YarnConfiguration.ScmClientServerThreadCount, YarnConfiguration .DefaultScmClientServerThreadCount)); // Secret manager null for now (security not supported) // TODO (YARN-2774): Enable service authorization this.server.Start(); clientBindAddress = conf.UpdateConnectAddr(YarnConfiguration.ScmClientServerAddress , server.GetListenerAddress()); base.ServiceStart(); }
public virtual void TestRelease_ExistingEntry_NonExistantAppId() { // Pre-populate the SCM with one cache entry store.AddResource("key1", "foo.jar"); store.AddResourceReference("key1", new SharedCacheResourceReference(CreateAppId(1 , 1L), "user")); NUnit.Framework.Assert.AreEqual(1, store.GetResourceReferences("key1").Count); long releases = ClientSCMMetrics.GetInstance().GetCacheReleases(); ReleaseSharedCacheResourceRequest request = recordFactory.NewRecordInstance <ReleaseSharedCacheResourceRequest >(); request.SetResourceKey("key1"); request.SetAppId(CreateAppId(2, 2L)); clientSCMProxy.Release(request); NUnit.Framework.Assert.AreEqual(1, store.GetResourceReferences("key1").Count); NUnit.Framework.Assert.AreEqual("Client SCM metrics were updated when a release did not happen" , 0, ClientSCMMetrics.GetInstance().GetCacheReleases() - releases); }
public virtual void TestRelease_ExistingEntry_WithAppId() { // Pre-populate the SCM with one cache entry store.AddResource("key1", "foo.jar"); UserGroupInformation testUGI = UserGroupInformation.GetCurrentUser(); store.AddResourceReference("key1", new SharedCacheResourceReference(CreateAppId(1 , 1L), testUGI.GetShortUserName())); NUnit.Framework.Assert.AreEqual(1, store.GetResourceReferences("key1").Count); long releases = ClientSCMMetrics.GetInstance().GetCacheReleases(); ReleaseSharedCacheResourceRequest request = recordFactory.NewRecordInstance <ReleaseSharedCacheResourceRequest >(); request.SetResourceKey("key1"); request.SetAppId(CreateAppId(1, 1L)); clientSCMProxy.Release(request); NUnit.Framework.Assert.AreEqual(0, store.GetResourceReferences("key1").Count); NUnit.Framework.Assert.AreEqual("Client SCM metrics aren't updated.", 1, ClientSCMMetrics .GetInstance().GetCacheReleases() - releases); }
protected override void Render(HtmlBlock.Block html) { SCMMetricsInfo metricsInfo = new SCMMetricsInfo(CleanerMetrics.GetInstance(), ClientSCMMetrics .GetInstance(), SharedCacheUploaderMetrics.GetInstance()); Info("Shared Cache Manager overview").("Started on:", Times.Format(scm.GetStartTime ())).("Cache hits: ", metricsInfo.GetCacheHits()).("Cache misses: ", metricsInfo .GetCacheMisses()).("Cache releases: ", metricsInfo.GetCacheReleases()).("Accepted uploads: " , metricsInfo.GetAcceptedUploads()).("Rejected uploads: ", metricsInfo.GetRejectUploads ()).("Deleted files by the cleaner: ", metricsInfo.GetTotalDeletedFiles()).("Processed files by the cleaner: " , metricsInfo.GetTotalProcessedFiles()); html.(typeof(InfoBlock)); }