public virtual void TestUse_ExistingEntry_DupId()
{
// Pre-populate the SCM with one cache entry
store.AddResource("key1", "foo.jar");
UserGroupInformation testUGI = UserGroupInformation.GetCurrentUser();
store.AddResourceReference("key1", new SharedCacheResourceReference(CreateAppId(1
, 1L), testUGI.GetShortUserName()));
NUnit.Framework.Assert.AreEqual(1, store.GetResourceReferences("key1").Count);
long hits = ClientSCMMetrics.GetInstance().GetCacheHits();
// Add a new duplicate appId
UseSharedCacheResourceRequest request = recordFactory.NewRecordInstance <UseSharedCacheResourceRequest
>();
request.SetResourceKey("key1");
request.SetAppId(CreateAppId(1, 1L));
// Expecting default depth of 3 under the shared cache root dir
string expectedPath = testDir.GetAbsolutePath() + "/k/e/y/key1/foo.jar";
NUnit.Framework.Assert.AreEqual(expectedPath, clientSCMProxy.Use(request).GetPath
());
NUnit.Framework.Assert.AreEqual(1, store.GetResourceReferences("key1").Count);
NUnit.Framework.Assert.AreEqual("Client SCM metrics aren't updated.", 1, ClientSCMMetrics
.GetInstance().GetCacheHits() - hits);
}
public SCMMetricsInfo(CleanerMetrics cleanerMetrics, ClientSCMMetrics clientSCMMetrics
, SharedCacheUploaderMetrics scmUploaderMetrics)
{
totalDeletedFiles = cleanerMetrics.GetTotalDeletedFiles();
totalProcessedFiles = cleanerMetrics.GetTotalProcessedFiles();
cacheHits = clientSCMMetrics.GetCacheHits();
cacheMisses = clientSCMMetrics.GetCacheMisses();
cacheReleases = clientSCMMetrics.GetCacheReleases();
acceptedUploads = scmUploaderMetrics.GetAcceptedUploads();
rejectedUploads = scmUploaderMetrics.GetRejectUploads();
}
public virtual void TestUse_MissingEntry()
{
long misses = ClientSCMMetrics.GetInstance().GetCacheMisses();
UseSharedCacheResourceRequest request = recordFactory.NewRecordInstance <UseSharedCacheResourceRequest
>();
request.SetResourceKey("key1");
request.SetAppId(CreateAppId(1, 1L));
NUnit.Framework.Assert.IsNull(clientSCMProxy.Use(request).GetPath());
NUnit.Framework.Assert.AreEqual("Client SCM metrics aren't updated.", 1, ClientSCMMetrics
.GetInstance().GetCacheMisses() - misses);
}
public virtual void TestRelease_MissingEntry()
{
long releases = ClientSCMMetrics.GetInstance().GetCacheReleases();
ReleaseSharedCacheResourceRequest request = recordFactory.NewRecordInstance <ReleaseSharedCacheResourceRequest
>();
request.SetResourceKey("key2");
request.SetAppId(CreateAppId(2, 2L));
clientSCMProxy.Release(request);
NUnit.Framework.Assert.IsNotNull(store.GetResourceReferences("key2"));
NUnit.Framework.Assert.AreEqual(0, store.GetResourceReferences("key2").Count);
NUnit.Framework.Assert.AreEqual("Client SCM metrics were updated when a release did not happen."
, 0, ClientSCMMetrics.GetInstance().GetCacheReleases() - releases);
}
/// <exception cref="System.Exception"/>
protected override void ServiceStart()
{
Configuration conf = GetConfig();
this.metrics = ClientSCMMetrics.GetInstance();
YarnRPC rpc = YarnRPC.Create(conf);
this.server = rpc.GetServer(typeof(ClientSCMProtocol), this, clientBindAddress, conf
, null, conf.GetInt(YarnConfiguration.ScmClientServerThreadCount, YarnConfiguration
.DefaultScmClientServerThreadCount));
// Secret manager null for now (security not supported)
// TODO (YARN-2774): Enable service authorization
this.server.Start();
clientBindAddress = conf.UpdateConnectAddr(YarnConfiguration.ScmClientServerAddress
, server.GetListenerAddress());
base.ServiceStart();
}
public virtual void TestRelease_ExistingEntry_NonExistantAppId()
{
// Pre-populate the SCM with one cache entry
store.AddResource("key1", "foo.jar");
store.AddResourceReference("key1", new SharedCacheResourceReference(CreateAppId(1
, 1L), "user"));
NUnit.Framework.Assert.AreEqual(1, store.GetResourceReferences("key1").Count);
long releases = ClientSCMMetrics.GetInstance().GetCacheReleases();
ReleaseSharedCacheResourceRequest request = recordFactory.NewRecordInstance <ReleaseSharedCacheResourceRequest
>();
request.SetResourceKey("key1");
request.SetAppId(CreateAppId(2, 2L));
clientSCMProxy.Release(request);
NUnit.Framework.Assert.AreEqual(1, store.GetResourceReferences("key1").Count);
NUnit.Framework.Assert.AreEqual("Client SCM metrics were updated when a release did not happen"
, 0, ClientSCMMetrics.GetInstance().GetCacheReleases() - releases);
}
public virtual void TestRelease_ExistingEntry_WithAppId()
{
// Pre-populate the SCM with one cache entry
store.AddResource("key1", "foo.jar");
UserGroupInformation testUGI = UserGroupInformation.GetCurrentUser();
store.AddResourceReference("key1", new SharedCacheResourceReference(CreateAppId(1
, 1L), testUGI.GetShortUserName()));
NUnit.Framework.Assert.AreEqual(1, store.GetResourceReferences("key1").Count);
long releases = ClientSCMMetrics.GetInstance().GetCacheReleases();
ReleaseSharedCacheResourceRequest request = recordFactory.NewRecordInstance <ReleaseSharedCacheResourceRequest
>();
request.SetResourceKey("key1");
request.SetAppId(CreateAppId(1, 1L));
clientSCMProxy.Release(request);
NUnit.Framework.Assert.AreEqual(0, store.GetResourceReferences("key1").Count);
NUnit.Framework.Assert.AreEqual("Client SCM metrics aren't updated.", 1, ClientSCMMetrics
.GetInstance().GetCacheReleases() - releases);
}
protected override void Render(HtmlBlock.Block html)
{
SCMMetricsInfo metricsInfo = new SCMMetricsInfo(CleanerMetrics.GetInstance(), ClientSCMMetrics
.GetInstance(), SharedCacheUploaderMetrics.GetInstance());
Info("Shared Cache Manager overview").("Started on:", Times.Format(scm.GetStartTime
())).("Cache hits: ", metricsInfo.GetCacheHits()).("Cache misses: ", metricsInfo
.GetCacheMisses()).("Cache releases: ", metricsInfo.GetCacheReleases()).("Accepted uploads: "
, metricsInfo.GetAcceptedUploads()).("Rejected uploads: ", metricsInfo.GetRejectUploads
()).("Deleted files by the cleaner: ", metricsInfo.GetTotalDeletedFiles()).("Processed files by the cleaner: "
, metricsInfo.GetTotalProcessedFiles());
html.(typeof(InfoBlock));
}
