/// <exception cref="System.IO.IOException"/>
private NMStateStoreService.RecoveredUserResources LoadUserLocalizedResources(LeveldbIterator
iter, string keyPrefix)
{
NMStateStoreService.RecoveredUserResources userResources = new NMStateStoreService.RecoveredUserResources
();
while (iter.HasNext())
{
KeyValuePair <byte[], byte[]> entry = iter.PeekNext();
string key = JniDBFactory.AsString(entry.Key);
if (!key.StartsWith(keyPrefix))
{
break;
}
if (key.StartsWith(LocalizationFilecacheSuffix, keyPrefix.Length))
{
userResources.privateTrackerState = LoadResourceTrackerState(iter, keyPrefix + LocalizationFilecacheSuffix
);
}
else
{
if (key.StartsWith(LocalizationAppcacheSuffix, keyPrefix.Length))
{
int appIdStartPos = keyPrefix.Length + LocalizationAppcacheSuffix.Length;
int appIdEndPos = key.IndexOf('/', appIdStartPos);
if (appIdEndPos < 0)
{
throw new IOException("Unable to determine appID in resource key: " + key);
}
ApplicationId appId = ConverterUtils.ToApplicationId(Sharpen.Runtime.Substring(key
, appIdStartPos, appIdEndPos));
userResources.appTrackerStates[appId] = LoadResourceTrackerState(iter, Sharpen.Runtime.Substring
(key, 0, appIdEndPos + 1));
}
else
{
throw new IOException("Unexpected user resource key " + key);
}
}
}
return(userResources);
}
/// <exception cref="System.Exception"/>
public virtual void TestNMProxyRetry()
{
containerManager.Start();
containerManager.SetBlockNewContainerRequests(false);
StartContainersRequest allRequests = Records.NewRecord <StartContainersRequest>();
ApplicationId appId = ApplicationId.NewInstance(1, 1);
ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1);
Token nmToken = context.GetNMTokenSecretManager().CreateNMToken(attemptId, context
.GetNodeId(), user);
IPEndPoint address = conf.GetSocketAddr(YarnConfiguration.NmBindHost, YarnConfiguration
.NmAddress, YarnConfiguration.DefaultNmAddress, YarnConfiguration.DefaultNmPort);
Org.Apache.Hadoop.Security.Token.Token <NMTokenIdentifier> token = ConverterUtils.
ConvertFromYarn(nmToken, SecurityUtil.BuildTokenService(address));
UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser(user);
ugi.AddToken(token);
ContainerManagementProtocol proxy = NMProxy.CreateNMProxy <ContainerManagementProtocol
>(conf, ugi, YarnRPC.Create(conf), address);
retryCount = 0;
shouldThrowNMNotYetReadyException = false;
proxy.StartContainers(allRequests);
NUnit.Framework.Assert.AreEqual(5, retryCount);
retryCount = 0;
shouldThrowNMNotYetReadyException = false;
proxy.StopContainers(Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <StopContainersRequest
>());
NUnit.Framework.Assert.AreEqual(5, retryCount);
retryCount = 0;
shouldThrowNMNotYetReadyException = false;
proxy.GetContainerStatuses(Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <GetContainerStatusesRequest
>());
NUnit.Framework.Assert.AreEqual(5, retryCount);
retryCount = 0;
shouldThrowNMNotYetReadyException = true;
proxy.StartContainers(allRequests);
NUnit.Framework.Assert.AreEqual(5, retryCount);
}
internal static void BuildCatalog()
{
styleCatalog = new StyleCatalog();
var paths = GetDefaultStyleCatalogPaths();
foreach (var editorUssPath in AssetDatabase.GetAllAssetPaths().Where(IsEditorStyleSheet))
{
paths.Add(editorUssPath);
}
styleCatalog.Load(paths);
if (CanEnableExtendedStyles())
{
// Update gui skin style layouts
var skin = GUIUtility.GetDefaultSkin();
if (skin != null)
{
ConverterUtils.ResetSkinToPristine(skin, EditorGUIUtility.isProSkin ? SkinTarget.Dark : SkinTarget.Light);
UpdateGUIStyleProperties(skin);
}
}
}
protected internal virtual ContainerManagementProtocol NewProxy(YarnRPC rpc, string
containerManagerBindAddr, ContainerId containerId, Token token)
{
if (token == null)
{
throw new SecretManager.InvalidToken("No NMToken sent for " + containerManagerBindAddr
);
}
IPEndPoint cmAddr = NetUtils.CreateSocketAddr(containerManagerBindAddr);
ContainerManagementProtocolProxy.Log.Info("Opening proxy : " + containerManagerBindAddr
);
// the user in createRemoteUser in this context has to be ContainerID
UserGroupInformation user = UserGroupInformation.CreateRemoteUser(containerId.GetApplicationAttemptId
().ToString());
Org.Apache.Hadoop.Security.Token.Token <NMTokenIdentifier> nmToken = ConverterUtils
.ConvertFromYarn(token, cmAddr);
user.AddToken(nmToken);
return(NMProxy.CreateNMProxy <ContainerManagementProtocol>(this._enclosing.conf, user
, rpc, cmAddr));
}
public virtual Response GetLogs(string containerIdStr, string filename)
{
ContainerId containerId;
try
{
containerId = ConverterUtils.ToContainerId(containerIdStr);
}
catch (ArgumentException)
{
return(Response.Status(Response.Status.BadRequest).Build());
}
FilePath logFile = null;
try
{
logFile = ContainerLogsUtils.GetContainerLogFile(containerId, filename, request.GetRemoteUser
(), nmContext);
}
catch (NotFoundException ex)
{
return(Response.Status(Response.Status.NotFound).Entity(ex.Message).Build());
}
catch (YarnException ex)
{
return(Response.ServerError().Entity(ex.Message).Build());
}
try
{
FileInputStream fis = ContainerLogsUtils.OpenLogFileForRead(containerIdStr, logFile
, nmContext);
StreamingOutput stream = new _StreamingOutput_240(fis);
return(Response.Ok(stream).Build());
}
catch (IOException ex)
{
return(Response.ServerError().Entity(ex.Message).Build());
}
}
/// <exception cref="System.IO.IOException"/>
private static void ConfigureTask(JobConf job, Task task, Credentials credentials
, Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jt)
{
job.SetCredentials(credentials);
ApplicationAttemptId appAttemptId = ConverterUtils.ToContainerId(Runtime.Getenv(ApplicationConstants.Environment
.ContainerId.ToString())).GetApplicationAttemptId();
Log.Debug("APPLICATION_ATTEMPT_ID: " + appAttemptId);
// Set it in conf, so as to be able to be used the the OutputCommitter.
job.SetInt(MRJobConfig.ApplicationAttemptId, appAttemptId.GetAttemptId());
// set tcp nodelay
job.SetBoolean("ipc.client.tcpnodelay", true);
job.SetClass(MRConfig.TaskLocalOutputClass, typeof(YarnOutputFiles), typeof(MapOutputFile
));
// set the jobToken and shuffle secrets into task
task.SetJobTokenSecret(JobTokenSecretManager.CreateSecretKey(jt.GetPassword()));
byte[] shuffleSecret = TokenCache.GetShuffleSecretKey(credentials);
if (shuffleSecret == null)
{
Log.Warn("Shuffle secret missing from task credentials." + " Using job token secret as shuffle secret."
);
shuffleSecret = jt.GetPassword();
}
task.SetShuffleSecret(JobTokenSecretManager.CreateSecretKey(shuffleSecret));
// setup the child's MRConfig.LOCAL_DIR.
ConfigureLocalDirs(task, job);
// setup the child's attempt directories
// Do the task-type specific localization
task.LocalizeConfiguration(job);
// Set up the DistributedCache related configs
MRApps.SetupDistributedCacheLocal(job);
// Overwrite the localized task jobconf which is linked to in the current
// work-dir.
Path localTaskFile = new Path(MRJobConfig.JobConfFile);
WriteLocalJobFile(localTaskFile, job);
task.SetJobFile(localTaskFile.ToString());
task.SetConf(job);
}
public virtual ContainerInfo GetNodeContainer(string id)
{
ContainerId containerId = null;
Init();
try
{
containerId = ConverterUtils.ToContainerId(id);
}
catch (Exception)
{
throw new BadRequestException("invalid container id, " + id);
}
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= nmContext.GetContainers()[containerId];
if (container == null)
{
throw new NotFoundException("container with id, " + id + ", not found");
}
return(new ContainerInfo(this.nmContext, container, uriInfo.GetBaseUri().ToString
(), webapp.Name()));
}
private NodeId VerifyAndGetNodeId(HtmlBlock.Block html)
{
string nodeIdStr = $(YarnWebParams.NmNodename);
if (nodeIdStr == null || nodeIdStr.IsEmpty())
{
html.H1().("Cannot get container logs without a NodeId").();
return(null);
}
NodeId nodeId = null;
try
{
nodeId = ConverterUtils.ToNodeId(nodeIdStr);
}
catch (ArgumentException)
{
html.H1().("Cannot get container logs. Invalid nodeId: " + nodeIdStr).();
return(null);
}
return(nodeId);
}
public TaskAttemptInfo(TaskAttempt ta, TaskType type, bool isRunning)
{
TaskAttemptReport report = ta.GetReport();
this.type = type.ToString();
this.id = MRApps.ToString(ta.GetID());
this.nodeHttpAddress = ta.GetNodeHttpAddress();
this.startTime = report.GetStartTime();
this.finishTime = report.GetFinishTime();
this.assignedContainerId = ConverterUtils.ToString(report.GetContainerId());
this.assignedContainer = report.GetContainerId();
this.progress = report.GetProgress() * 100;
this.status = report.GetStateString();
this.state = report.GetTaskAttemptState();
this.elapsedTime = Times.Elapsed(this.startTime, this.finishTime, isRunning);
if (this.elapsedTime == -1)
{
this.elapsedTime = 0;
}
this.diagnostics = report.GetDiagnosticInfo();
this.rack = ta.GetNodeRackName();
}
internal static ResourceLocalizationSpec GetMockRsrc(Random r, LocalResourceVisibility
vis, Path p)
{
ResourceLocalizationSpec resourceLocalizationSpec = Org.Mockito.Mockito.Mock <ResourceLocalizationSpec
>();
LocalResource rsrc = Org.Mockito.Mockito.Mock <LocalResource>();
string name = long.ToHexString(r.NextLong());
URL uri = Org.Mockito.Mockito.Mock <URL>();
Org.Mockito.Mockito.When(uri.GetScheme()).ThenReturn("file");
Org.Mockito.Mockito.When(uri.GetHost()).ThenReturn(null);
Org.Mockito.Mockito.When(uri.GetFile()).ThenReturn("/local/" + vis + "/" + name);
Org.Mockito.Mockito.When(rsrc.GetResource()).ThenReturn(uri);
Org.Mockito.Mockito.When(rsrc.GetSize()).ThenReturn(r.Next(1024) + 1024L);
Org.Mockito.Mockito.When(rsrc.GetTimestamp()).ThenReturn(r.Next(1024) + 2048L);
Org.Mockito.Mockito.When(rsrc.GetType()).ThenReturn(LocalResourceType.File);
Org.Mockito.Mockito.When(rsrc.GetVisibility()).ThenReturn(vis);
Org.Mockito.Mockito.When(resourceLocalizationSpec.GetResource()).ThenReturn(rsrc);
Org.Mockito.Mockito.When(resourceLocalizationSpec.GetDestinationDirectory()).ThenReturn
(ConverterUtils.GetYarnUrlFromPath(p));
return(resourceLocalizationSpec);
}
internal async void ScrapeWebsite(LotteryDetails lotteryDetails, Dictionary <string, string> parameters)
{
try
{
RaiseEvent(LottoWebScrapingStages.CONNECTING);
IHtmlDocument documentForSession = await GetWebsiteDOMAsync(GenerateParameters(lotteryDetails));
RaiseEvent(LottoWebScrapingStages.SESSION_CREATION);
Dictionary <string, string> sessionParam = GetSessionBasedParameters(lotteryDetails, documentForSession);
RaiseEvent(LottoWebScrapingStages.SEARCHING_DATA);
IHtmlDocument document = await GetWebsiteDOMAsync(sessionParam);
RaiseEvent(LottoWebScrapingStages.SCRAPING);
List <LotteryDrawResult> lotteryDrawResultArr = GetScrapeResults(lotteryDetails, document);
int countCtr = 1;
LotteryDrawResultDao lotteryDao = LotteryDrawResultDaoImpl.GetInstance();
foreach (LotteryDrawResult scrapeResult in lotteryDrawResultArr.ToList())
{
LotteryDrawResult result = lotteryDao.GetLotteryDrawResultByDrawDate(lotteryDetails.GameMode, scrapeResult.GetDrawDate());
if (result == null && !scrapeResult.IsDrawResulSequenceEmpty())
{
newRecordsCount++;
lotteryDao.InsertDrawDate(scrapeResult);
}
RaiseEvent(LottoWebScrapingStages.INSERT, ConverterUtils.GetPercentageFloored(countCtr++, lotteryDrawResultArr.Count), scrapeResult.GetExtractedDrawnResultDetails());
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
RaiseEvent(LottoWebScrapingStages.ERROR, 0, ex.Message);
}
finally
{
RaiseEvent(LottoWebScrapingStages.FINISH);
}
}
public virtual void TestMRAppMasterSuccessLock()
{
string applicationAttemptIdStr = "appattempt_1317529182569_0004_000002";
string containerIdStr = "container_1317529182569_0004_000002_1";
string userName = "******";
JobConf conf = new JobConf();
conf.Set(MRJobConfig.MrAmStagingDir, stagingDir);
ApplicationAttemptId applicationAttemptId = ConverterUtils.ToApplicationAttemptId
(applicationAttemptIdStr);
JobId jobId = TypeConverter.ToYarn(TypeConverter.FromYarn(applicationAttemptId.GetApplicationId
()));
Path start = MRApps.GetStartJobCommitFile(conf, userName, jobId);
Path end = MRApps.GetEndJobCommitSuccessFile(conf, userName, jobId);
FileSystem fs = FileSystem.Get(conf);
fs.Create(start).Close();
fs.Create(end).Close();
ContainerId containerId = ConverterUtils.ToContainerId(containerIdStr);
MRAppMaster appMaster = new MRAppMasterTest(applicationAttemptId, containerId, "host"
, -1, -1, Runtime.CurrentTimeMillis(), false, false);
bool caught = false;
try
{
MRAppMaster.InitAndStartAppMaster(appMaster, conf, userName);
}
catch (IOException e)
{
//The IO Exception is expected
Log.Info("Caught expected Exception", e);
caught = true;
}
NUnit.Framework.Assert.IsTrue(caught);
NUnit.Framework.Assert.IsTrue(appMaster.errorHappenedShutDown);
NUnit.Framework.Assert.AreEqual(JobStateInternal.Succeeded, appMaster.forcedState
);
appMaster.Stop();
// verify the final status is SUCCEEDED
VerifyFailedStatus((MRAppMasterTest)appMaster, "SUCCEEDED");
}
internal static void RefreshSkin()
{
if (!CanEnableExtendedStyles())
{
return;
}
GUIStyle.onDraw = StylePainter.DrawStyle;
// Update gui skin style layouts
var skin = GUIUtility.GetDefaultSkin();
if (skin != null)
{
// TODO: Emit OnStyleCatalogLoaded
if (Path.GetFileName(Path.GetDirectoryName(Application.dataPath)) == "editor_resources")
{
ConverterUtils.ResetSkinToPristine(skin, EditorGUIUtility.isProSkin ? SkinTarget.Dark : SkinTarget.Light);
}
skin.font = GetNormalFont();
UpdateGUIStyleProperties(skin);
}
}
private ContainerId VerifyAndGetContainerId(HtmlBlock.Block html)
{
string containerIdStr = $(YarnWebParams.ContainerId);
if (containerIdStr == null || containerIdStr.IsEmpty())
{
html.H1().("Cannot get container logs without a ContainerId").();
return(null);
}
ContainerId containerId = null;
try
{
containerId = ConverterUtils.ToContainerId(containerIdStr);
}
catch (ArgumentException)
{
html.H1().("Cannot get container logs for invalid containerId: " + containerIdStr
).();
return(null);
}
return(containerId);
}
internal virtual bool VerifyAccess()
{
// if it is in the public cache, it's trivially OK
if (resource.GetVisibility() == LocalResourceVisibility.Public)
{
return(true);
}
Path remotePath;
try
{
remotePath = ConverterUtils.GetPathFromYarnURL(resource.GetResource());
}
catch (URISyntaxException e)
{
throw new IOException("Invalid resource", e);
}
// get the file status of the HDFS file
FileSystem remoteFs = remotePath.GetFileSystem(conf);
FileStatus status = remoteFs.GetFileStatus(remotePath);
// check to see if the file has been modified in any way
if (status.GetModificationTime() != resource.GetTimestamp())
{
Log.Warn("The remote file " + remotePath + " has changed since it's localized; will not consider it for upload"
);
return(false);
}
// check for the user ownership
if (status.GetOwner().Equals(user))
{
return(true);
}
// the user owns the file
// check if the file is publicly readable otherwise
return(FileIsPublic(remotePath, remoteFs, status));
}
public virtual void VerifyTaskAttemptGeneric(TaskAttempt ta, TaskType ttype, string
id, string state, string type, string rack, string nodeHttpAddress, string diagnostics
, string assignedContainerId, long startTime, long finishTime, long elapsedTime,
float progress)
{
TaskAttemptId attid = ta.GetID();
string attemptId = MRApps.ToString(attid);
WebServicesTestUtils.CheckStringMatch("id", attemptId, id);
WebServicesTestUtils.CheckStringMatch("type", ttype.ToString(), type);
WebServicesTestUtils.CheckStringMatch("state", ta.GetState().ToString(), state);
WebServicesTestUtils.CheckStringMatch("rack", ta.GetNodeRackName(), rack);
WebServicesTestUtils.CheckStringMatch("nodeHttpAddress", ta.GetNodeHttpAddress(),
nodeHttpAddress);
string expectDiag = string.Empty;
IList <string> diagnosticsList = ta.GetDiagnostics();
if (diagnosticsList != null && !diagnostics.IsEmpty())
{
StringBuilder b = new StringBuilder();
foreach (string diag in diagnosticsList)
{
b.Append(diag);
}
expectDiag = b.ToString();
}
WebServicesTestUtils.CheckStringMatch("diagnostics", expectDiag, diagnostics);
WebServicesTestUtils.CheckStringMatch("assignedContainerId", ConverterUtils.ToString
(ta.GetAssignedContainerID()), assignedContainerId);
NUnit.Framework.Assert.AreEqual("startTime wrong", ta.GetLaunchTime(), startTime);
NUnit.Framework.Assert.AreEqual("finishTime wrong", ta.GetFinishTime(), finishTime
);
NUnit.Framework.Assert.AreEqual("elapsedTime wrong", finishTime - startTime, elapsedTime
);
NUnit.Framework.Assert.AreEqual("progress wrong", ta.GetProgress() * 100, progress
, 1e-3f);
}
/// <summary>
/// Loop through for a time-bounded interval waiting to
/// read the process id from a file generated by a running process.
/// </summary>
/// <param name="pidFilePath">File from which to read the process id</param>
/// <returns>Process ID</returns>
/// <exception cref="System.Exception"/>
private string GetContainerPid(Path pidFilePath)
{
string containerIdStr = ConverterUtils.ToString(container.GetContainerId());
string processId = null;
Log.Debug("Accessing pid for container " + containerIdStr + " from pid file " + pidFilePath
);
int sleepCounter = 0;
int sleepInterval = 100;
// loop waiting for pid file to show up
// until our timer expires in which case we admit defeat
while (true)
{
processId = ProcessIdFileReader.GetProcessId(pidFilePath);
if (processId != null)
{
Log.Debug("Got pid " + processId + " for container " + containerIdStr);
break;
}
else
{
if ((sleepCounter * sleepInterval) > maxKillWaitTime)
{
Log.Info("Could not get pid for " + containerIdStr + ". Waited for " + maxKillWaitTime
+ " ms.");
break;
}
else
{
++sleepCounter;
Sharpen.Thread.Sleep(sleepInterval);
}
}
}
return(processId);
}
/// <exception cref="System.Exception"/>
public virtual void TestFSRMStateStore()
{
HdfsConfiguration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
try
{
fsTester = new TestFSRMStateStore.TestFSRMStateStoreTester(this, cluster, false);
// If the state store is FileSystemRMStateStore then add corrupted entry.
// It should discard the entry and remove it from file system.
FSDataOutputStream fsOut = null;
FileSystemRMStateStore fileSystemRMStateStore = (FileSystemRMStateStore)fsTester.
GetRMStateStore();
string appAttemptIdStr3 = "appattempt_1352994193343_0001_000003";
ApplicationAttemptId attemptId3 = ConverterUtils.ToApplicationAttemptId(appAttemptIdStr3
);
Path appDir = fsTester.store.GetAppDir(attemptId3.GetApplicationId().ToString());
Path tempAppAttemptFile = new Path(appDir, attemptId3.ToString() + ".tmp");
fsOut = fileSystemRMStateStore.fs.Create(tempAppAttemptFile, false);
fsOut.Write(Sharpen.Runtime.GetBytesForString("Some random data "));
fsOut.Close();
TestRMAppStateStore(fsTester);
NUnit.Framework.Assert.IsFalse(fsTester.workingDirPathURI.GetFileSystem(conf).Exists
(tempAppAttemptFile));
TestRMDTSecretManagerStateStore(fsTester);
TestCheckVersion(fsTester);
TestEpoch(fsTester);
TestAppDeletion(fsTester);
TestDeleteStore(fsTester);
TestAMRMTokenSecretManagerStateStore(fsTester);
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestDuplicateRMAppDeletion()
{
TestZKRMStateStore.TestZKRMStateStoreTester zkTester = new TestZKRMStateStore.TestZKRMStateStoreTester
(this);
long submitTime = Runtime.CurrentTimeMillis();
long startTime = Runtime.CurrentTimeMillis() + 1234;
RMStateStore store = zkTester.GetRMStateStore();
RMStateStoreTestBase.TestDispatcher dispatcher = new RMStateStoreTestBase.TestDispatcher
();
store.SetRMDispatcher(dispatcher);
ApplicationAttemptId attemptIdRemoved = ConverterUtils.ToApplicationAttemptId("appattempt_1352994193343_0002_000001"
);
ApplicationId appIdRemoved = attemptIdRemoved.GetApplicationId();
StoreApp(store, appIdRemoved, submitTime, startTime);
StoreAttempt(store, attemptIdRemoved, "container_1352994193343_0002_01_000001", null
, null, dispatcher);
ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl();
context.SetApplicationId(appIdRemoved);
ApplicationStateData appStateRemoved = ApplicationStateData.NewInstance(submitTime
, startTime, context, "user1");
appStateRemoved.attempts[attemptIdRemoved] = null;
store.RemoveApplicationStateInternal(appStateRemoved);
try
{
store.RemoveApplicationStateInternal(appStateRemoved);
}
catch (KeeperException.NoNodeException)
{
NUnit.Framework.Assert.Fail("NoNodeException should not happen.");
}
store.Close();
}
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/>
/// <exception cref="System.Exception"/>
public virtual void TestNodeSingleContainersHelper(string media)
{
WebResource r = Resource();
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = new MockApp(1);
nmContext.GetApplications()[app.GetAppId()] = app;
Dictionary <string, string> hash = AddAppContainers(app);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app2 = new MockApp(2);
nmContext.GetApplications()[app2.GetAppId()] = app2;
AddAppContainers(app2);
foreach (string id in hash.Keys)
{
ClientResponse response = r.Path("ws").Path("v1").Path("node").Path("containers")
.Path(id).Accept(media).Get <ClientResponse>();
NUnit.Framework.Assert.AreEqual(MediaType.ApplicationJsonType, response.GetType()
);
JSONObject json = response.GetEntity <JSONObject>();
VerifyNodeContainerInfo(json.GetJSONObject("container"), nmContext.GetContainers(
)[ConverterUtils.ToContainerId(id)]);
}
}
public BreastCancerSampleItemDefinitionBuilderUI()
{
InitializeComponent();
headers = ConverterUtils.GetItems <BreastCancerSampleItem, SampleInfoAttribute>();
propertyNames = (from h in headers
orderby h.PropertyName
select h.PropertyName).ToList();
defaultValues = new BindingList <DefaultValue>(
(from pn in propertyNames
select new DefaultValue()
{
PropertyName = pn,
Value = string.Empty
}).ToList());
propertyNames.Insert(0, string.Empty);
colPropertyName.Items.AddRange(propertyNames.ToArray());
this.Text = Constants.GetSqhVanderbiltTitle(title, version);
}
public virtual void Logs()
{
string containerIdStr = $(ContainerId);
ContainerId containerId = null;
try
{
containerId = ConverterUtils.ToContainerId(containerIdStr);
}
catch (ArgumentException)
{
Render(typeof(ContainerLogsPage));
return;
}
ApplicationId appId = containerId.GetApplicationAttemptId().GetApplicationId();
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = nmContext.GetApplications()[appId];
if (app == null && nmConf.GetBoolean(YarnConfiguration.LogAggregationEnabled, YarnConfiguration
.DefaultLogAggregationEnabled))
{
string logServerUrl = nmConf.Get(YarnConfiguration.YarnLogServerUrl);
string redirectUrl = null;
if (logServerUrl == null || logServerUrl.IsEmpty())
{
redirectUrl = "false";
}
else
{
redirectUrl = Url(logServerUrl, nmContext.GetNodeId().ToString(), containerIdStr,
containerIdStr, $(AppOwner));
}
Set(ContainerLogsPage.RedirectUrl, redirectUrl);
}
Render(typeof(ContainerLogsPage));
}
public virtual void TestMRAppMasterMissingStaging()
{
string applicationAttemptIdStr = "appattempt_1317529182569_0004_000002";
string containerIdStr = "container_1317529182569_0004_000002_1";
string userName = "******";
JobConf conf = new JobConf();
conf.Set(MRJobConfig.MrAmStagingDir, stagingDir);
ApplicationAttemptId applicationAttemptId = ConverterUtils.ToApplicationAttemptId
(applicationAttemptIdStr);
//Delete the staging directory
FilePath dir = new FilePath(stagingDir);
if (dir.Exists())
{
FileUtils.DeleteDirectory(dir);
}
ContainerId containerId = ConverterUtils.ToContainerId(containerIdStr);
MRAppMaster appMaster = new MRAppMasterTest(applicationAttemptId, containerId, "host"
, -1, -1, Runtime.CurrentTimeMillis(), false, false);
bool caught = false;
try
{
MRAppMaster.InitAndStartAppMaster(appMaster, conf, userName);
}
catch (IOException e)
{
//The IO Exception is expected
Log.Info("Caught expected Exception", e);
caught = true;
}
NUnit.Framework.Assert.IsTrue(caught);
NUnit.Framework.Assert.IsTrue(appMaster.errorHappenedShutDown);
//Copying the history file is disabled, but it is not really visible from
//here
NUnit.Framework.Assert.AreEqual(JobStateInternal.Error, appMaster.forcedState);
appMaster.Stop();
}
protected override void Render(HtmlBlock.Block html)
{
string redirectUrl = $(RedirectUrl);
if (redirectUrl != null && redirectUrl.Equals("false"))
{
html.H1("Failed while trying to construct the redirect url to the log" + " server. Log Server url may not be configured"
);
}
//Intentional fallthrough.
ContainerId containerId;
try
{
containerId = ConverterUtils.ToContainerId($(YarnWebParams.ContainerId));
}
catch (ArgumentException)
{
html.H1("Invalid container ID: " + $(YarnWebParams.ContainerId));
return;
}
try
{
if ($(ContainerLogType).IsEmpty())
{
IList <FilePath> logFiles = ContainerLogsUtils.GetContainerLogDirs(containerId, Request
().GetRemoteUser(), nmContext);
PrintLogFileDirectory(html, logFiles);
}
else
{
FilePath logFile = ContainerLogsUtils.GetContainerLogFile(containerId, $(ContainerLogType
), Request().GetRemoteUser(), nmContext);
PrintLogFile(html, logFile);
}
}
/// <summary>Wrap API resource to match against cache of localized resources.</summary>
/// <param name="resource">Resource requested by container</param>
/// <exception cref="Sharpen.URISyntaxException">If the path is malformed</exception>
public LocalResourceRequest(LocalResource resource)
: this(ConverterUtils.GetPathFromYarnURL(resource.GetResource()), resource.GetTimestamp
(), resource.GetType(), resource.GetVisibility(), resource.GetPattern())
{
}
private void SetMaxFileSizeHumanReadable()
{
lblFileMaxSizeReadable.Text = ConverterUtils.HumanReadableFileSize(long.Parse(numericUpDownMaxFileSize.Value.ToString()), 2);
}
/// <summary>Cleanup the container.</summary>
/// <remarks>
/// Cleanup the container.
/// Cancels the launch if launch has not started yet or signals
/// the executor to not execute the process if not already done so.
/// Also, sends a SIGTERM followed by a SIGKILL to the process if
/// the process id is available.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
public virtual void CleanupContainer()
{
// dispatcher not typed
ContainerId containerId = container.GetContainerId();
string containerIdStr = ConverterUtils.ToString(containerId);
Log.Info("Cleaning up container " + containerIdStr);
try
{
context.GetNMStateStore().StoreContainerKilled(containerId);
}
catch (IOException e)
{
Log.Error("Unable to mark container " + containerId + " killed in store", e);
}
// launch flag will be set to true if process already launched
bool alreadyLaunched = !shouldLaunchContainer.CompareAndSet(false, true);
if (!alreadyLaunched)
{
Log.Info("Container " + containerIdStr + " not launched." + " No cleanup needed to be done"
);
return;
}
Log.Debug("Marking container " + containerIdStr + " as inactive");
// this should ensure that if the container process has not launched
// by this time, it will never be launched
exec.DeactivateContainer(containerId);
if (Log.IsDebugEnabled())
{
Log.Debug("Getting pid for container " + containerIdStr + " to kill" + " from pid file "
+ (pidFilePath != null ? pidFilePath.ToString() : "null"));
}
// however the container process may have already started
try
{
// get process id from pid file if available
// else if shell is still active, get it from the shell
string processId = null;
if (pidFilePath != null)
{
processId = GetContainerPid(pidFilePath);
}
// kill process
if (processId != null)
{
string user = container.GetUser();
Log.Debug("Sending signal to pid " + processId + " as user " + user + " for container "
+ containerIdStr);
ContainerExecutor.Signal signal = sleepDelayBeforeSigKill > 0 ? ContainerExecutor.Signal
.Term : ContainerExecutor.Signal.Kill;
bool result = exec.SignalContainer(user, processId, signal);
Log.Debug("Sent signal " + signal + " to pid " + processId + " as user " + user +
" for container " + containerIdStr + ", result=" + (result ? "success" : "failed"
));
if (sleepDelayBeforeSigKill > 0)
{
new ContainerExecutor.DelayedProcessKiller(container, user, processId, sleepDelayBeforeSigKill
, ContainerExecutor.Signal.Kill, exec).Start();
}
}
}
catch (Exception e)
{
string message = "Exception when trying to cleanup container " + containerIdStr +
": " + StringUtils.StringifyException(e);
Log.Warn(message);
dispatcher.GetEventHandler().Handle(new ContainerDiagnosticsUpdateEvent(containerId
, message));
}
finally
{
// cleanup pid file if present
if (pidFilePath != null)
{
FileContext lfs = FileContext.GetLocalFSFileContext();
lfs.Delete(pidFilePath, false);
lfs.Delete(pidFilePath.Suffix(ExitCodeFileSuffix), false);
}
}
}
public virtual int Call()
{
// dispatcher not typed
ContainerLaunchContext launchContext = container.GetLaunchContext();
IDictionary <Path, IList <string> > localResources = null;
ContainerId containerID = container.GetContainerId();
string containerIdStr = ConverterUtils.ToString(containerID);
IList <string> command = launchContext.GetCommands();
int ret = -1;
// CONTAINER_KILLED_ON_REQUEST should not be missed if the container
// is already at KILLING
if (container.GetContainerState() == ContainerState.Killing)
{
dispatcher.GetEventHandler().Handle(new ContainerExitEvent(containerID, ContainerEventType
.ContainerKilledOnRequest, Shell.Windows ? ContainerExecutor.ExitCode.ForceKilled
.GetExitCode() : ContainerExecutor.ExitCode.Terminated.GetExitCode(), "Container terminated before launch."
));
return(0);
}
try
{
localResources = container.GetLocalizedResources();
if (localResources == null)
{
throw RPCUtil.GetRemoteException("Unable to get local resources when Container "
+ containerID + " is at " + container.GetContainerState());
}
string user = container.GetUser();
// /////////////////////////// Variable expansion
// Before the container script gets written out.
IList <string> newCmds = new AList <string>(command.Count);
string appIdStr = app.GetAppId().ToString();
string relativeContainerLogDir = Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Launcher.ContainerLaunch
.GetRelativeContainerLogDir(appIdStr, containerIdStr);
Path containerLogDir = dirsHandler.GetLogPathForWrite(relativeContainerLogDir, false
);
foreach (string str in command)
{
// TODO: Should we instead work via symlinks without this grammar?
newCmds.AddItem(ExpandEnvironment(str, containerLogDir));
}
launchContext.SetCommands(newCmds);
IDictionary <string, string> environment = launchContext.GetEnvironment();
// Make a copy of env to iterate & do variable expansion
foreach (KeyValuePair <string, string> entry in environment)
{
string value = entry.Value;
value = ExpandEnvironment(value, containerLogDir);
entry.SetValue(value);
}
// /////////////////////////// End of variable expansion
FileContext lfs = FileContext.GetLocalFSFileContext();
Path nmPrivateContainerScriptPath = dirsHandler.GetLocalPathForWrite(GetContainerPrivateDir
(appIdStr, containerIdStr) + Path.Separator + ContainerScript);
Path nmPrivateTokensPath = dirsHandler.GetLocalPathForWrite(GetContainerPrivateDir
(appIdStr, containerIdStr) + Path.Separator + string.Format(ContainerLocalizer.TokenFileNameFmt
, containerIdStr));
Path nmPrivateClasspathJarDir = dirsHandler.GetLocalPathForWrite(GetContainerPrivateDir
(appIdStr, containerIdStr));
DataOutputStream containerScriptOutStream = null;
DataOutputStream tokensOutStream = null;
// Select the working directory for the container
Path containerWorkDir = dirsHandler.GetLocalPathForWrite(ContainerLocalizer.Usercache
+ Path.Separator + user + Path.Separator + ContainerLocalizer.Appcache + Path.Separator
+ appIdStr + Path.Separator + containerIdStr, LocalDirAllocator.SizeUnknown, false
);
string pidFileSubpath = GetPidFileSubpath(appIdStr, containerIdStr);
// pid file should be in nm private dir so that it is not
// accessible by users
pidFilePath = dirsHandler.GetLocalPathForWrite(pidFileSubpath);
IList <string> localDirs = dirsHandler.GetLocalDirs();
IList <string> logDirs = dirsHandler.GetLogDirs();
IList <string> containerLogDirs = new AList <string>();
foreach (string logDir in logDirs)
{
containerLogDirs.AddItem(logDir + Path.Separator + relativeContainerLogDir);
}
if (!dirsHandler.AreDisksHealthy())
{
ret = ContainerExitStatus.DisksFailed;
throw new IOException("Most of the disks failed. " + dirsHandler.GetDisksHealthReport
(false));
}
try
{
// /////////// Write out the container-script in the nmPrivate space.
IList <Path> appDirs = new AList <Path>(localDirs.Count);
foreach (string localDir in localDirs)
{
Path usersdir = new Path(localDir, ContainerLocalizer.Usercache);
Path userdir = new Path(usersdir, user);
Path appsdir = new Path(userdir, ContainerLocalizer.Appcache);
appDirs.AddItem(new Path(appsdir, appIdStr));
}
containerScriptOutStream = lfs.Create(nmPrivateContainerScriptPath, EnumSet.Of(CreateFlag
.Create, CreateFlag.Overwrite));
// Set the token location too.
environment[ApplicationConstants.ContainerTokenFileEnvName] = new Path(containerWorkDir
, FinalContainerTokensFile).ToUri().GetPath();
// Sanitize the container's environment
SanitizeEnv(environment, containerWorkDir, appDirs, containerLogDirs, localResources
, nmPrivateClasspathJarDir);
// Write out the environment
exec.WriteLaunchEnv(containerScriptOutStream, environment, localResources, launchContext
.GetCommands());
// /////////// End of writing out container-script
// /////////// Write out the container-tokens in the nmPrivate space.
tokensOutStream = lfs.Create(nmPrivateTokensPath, EnumSet.Of(CreateFlag.Create, CreateFlag
.Overwrite));
Credentials creds = container.GetCredentials();
creds.WriteTokenStorageToStream(tokensOutStream);
}
finally
{
// /////////// End of writing out container-tokens
IOUtils.Cleanup(Log, containerScriptOutStream, tokensOutStream);
}
// LaunchContainer is a blocking call. We are here almost means the
// container is launched, so send out the event.
dispatcher.GetEventHandler().Handle(new ContainerEvent(containerID, ContainerEventType
.ContainerLaunched));
context.GetNMStateStore().StoreContainerLaunched(containerID);
// Check if the container is signalled to be killed.
if (!shouldLaunchContainer.CompareAndSet(false, true))
{
Log.Info("Container " + containerIdStr + " not launched as " + "cleanup already called"
);
ret = ContainerExecutor.ExitCode.Terminated.GetExitCode();
}
else
{
exec.ActivateContainer(containerID, pidFilePath);
ret = exec.LaunchContainer(container, nmPrivateContainerScriptPath, nmPrivateTokensPath
, user, appIdStr, containerWorkDir, localDirs, logDirs);
}
}
catch (Exception e)
{
Log.Warn("Failed to launch container.", e);
dispatcher.GetEventHandler().Handle(new ContainerExitEvent(containerID, ContainerEventType
.ContainerExitedWithFailure, ret, e.Message));
return(ret);
}
finally
{
completed.Set(true);
exec.DeactivateContainer(containerID);
try
{
context.GetNMStateStore().StoreContainerCompleted(containerID, ret);
}
catch (IOException)
{
Log.Error("Unable to set exit code for container " + containerID);
}
}
if (Log.IsDebugEnabled())
{
Log.Debug("Container " + containerIdStr + " completed with exit code " + ret);
}
if (ret == ContainerExecutor.ExitCode.ForceKilled.GetExitCode() || ret == ContainerExecutor.ExitCode
.Terminated.GetExitCode())
{
// If the process was killed, Send container_cleanedup_after_kill and
// just break out of this method.
dispatcher.GetEventHandler().Handle(new ContainerExitEvent(containerID, ContainerEventType
.ContainerKilledOnRequest, ret, "Container exited with a non-zero exit code " +
ret));
return(ret);
}
if (ret != 0)
{
Log.Warn("Container exited with a non-zero exit code " + ret);
this.dispatcher.GetEventHandler().Handle(new ContainerExitEvent(containerID, ContainerEventType
.ContainerExitedWithFailure, ret, "Container exited with a non-zero exit code "
+ ret));
return(ret);
}
Log.Info("Container " + containerIdStr + " succeeded ");
dispatcher.GetEventHandler().Handle(new ContainerEvent(containerID, ContainerEventType
.ContainerExitedWithSuccess));
return(0);
}
/// <summary>
/// Set up the distributed cache by localizing the resources, and updating
/// the configuration with references to the localized resources.
/// </summary>
/// <param name="conf"/>
/// <exception cref="System.IO.IOException"/>
public virtual void Setup(JobConf conf)
{
FilePath workDir = new FilePath(Runtime.GetProperty("user.dir"));
// Generate YARN local resources objects corresponding to the distributed
// cache configuration
IDictionary <string, LocalResource> localResources = new LinkedHashMap <string, LocalResource
>();
MRApps.SetupDistributedCache(conf, localResources);
// Generating unique numbers for FSDownload.
AtomicLong uniqueNumberGenerator = new AtomicLong(Runtime.CurrentTimeMillis());
// Find which resources are to be put on the local classpath
IDictionary <string, Path> classpaths = new Dictionary <string, Path>();
Path[] archiveClassPaths = DistributedCache.GetArchiveClassPaths(conf);
if (archiveClassPaths != null)
{
foreach (Path p in archiveClassPaths)
{
classpaths[p.ToUri().GetPath().ToString()] = p;
}
}
Path[] fileClassPaths = DistributedCache.GetFileClassPaths(conf);
if (fileClassPaths != null)
{
foreach (Path p in fileClassPaths)
{
classpaths[p.ToUri().GetPath().ToString()] = p;
}
}
// Localize the resources
LocalDirAllocator localDirAllocator = new LocalDirAllocator(MRConfig.LocalDir);
FileContext localFSFileContext = FileContext.GetLocalFSFileContext();
UserGroupInformation ugi = UserGroupInformation.GetCurrentUser();
ExecutorService exec = null;
try
{
ThreadFactory tf = new ThreadFactoryBuilder().SetNameFormat("LocalDistributedCacheManager Downloader #%d"
).Build();
exec = Executors.NewCachedThreadPool(tf);
Path destPath = localDirAllocator.GetLocalPathForWrite(".", conf);
IDictionary <LocalResource, Future <Path> > resourcesToPaths = Maps.NewHashMap();
foreach (LocalResource resource in localResources.Values)
{
Callable <Path> download = new FSDownload(localFSFileContext, ugi, conf, new Path(
destPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet())), resource
);
Future <Path> future = exec.Submit(download);
resourcesToPaths[resource] = future;
}
foreach (KeyValuePair <string, LocalResource> entry in localResources)
{
LocalResource resource_1 = entry.Value;
Path path;
try
{
path = resourcesToPaths[resource_1].Get();
}
catch (Exception e)
{
throw new IOException(e);
}
catch (ExecutionException e)
{
throw new IOException(e);
}
string pathString = path.ToUri().ToString();
string link = entry.Key;
string target = new FilePath(path.ToUri()).GetPath();
Symlink(workDir, target, link);
if (resource_1.GetType() == LocalResourceType.Archive)
{
localArchives.AddItem(pathString);
}
else
{
if (resource_1.GetType() == LocalResourceType.File)
{
localFiles.AddItem(pathString);
}
else
{
if (resource_1.GetType() == LocalResourceType.Pattern)
{
//PATTERN is not currently used in local mode
throw new ArgumentException("Resource type PATTERN is not " + "implemented yet. "
+ resource_1.GetResource());
}
}
}
Path resourcePath;
try
{
resourcePath = ConverterUtils.GetPathFromYarnURL(resource_1.GetResource());
}
catch (URISyntaxException e)
{
throw new IOException(e);
}
Log.Info(string.Format("Localized %s as %s", resourcePath, path));
string cp = resourcePath.ToUri().GetPath();
if (classpaths.Keys.Contains(cp))
{
localClasspaths.AddItem(path.ToUri().GetPath().ToString());
}
}
}
finally
{
if (exec != null)
{
exec.Shutdown();
}
}
// Update the configuration object with localized data.
if (!localArchives.IsEmpty())
{
conf.Set(MRJobConfig.CacheLocalarchives, StringUtils.ArrayToString(Sharpen.Collections.ToArray
(localArchives, new string[localArchives.Count])));
}
if (!localFiles.IsEmpty())
{
conf.Set(MRJobConfig.CacheLocalfiles, StringUtils.ArrayToString(Sharpen.Collections.ToArray
(localFiles, new string[localArchives.Count])));
}
setupCalled = true;
}
public AppInfo(ResourceManager rm, RMApp app, bool hasAccess, string schemePrefix
)
{
// these are ok for any user to see
// these are only allowed if acls allow
// preemption info fields
// JAXB needs this
this.schemePrefix = schemePrefix;
if (app != null)
{
string trackingUrl = app.GetTrackingUrl();
this.state = app.CreateApplicationState();
this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.IsEmpty() || YarnApplicationState
.New == this.state || YarnApplicationState.NewSaving == this.state || YarnApplicationState
.Submitted == this.state || YarnApplicationState.Accepted == this.state;
this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app.GetFinishTime(
) == 0 ? "ApplicationMaster" : "History");
if (!trackingUrlIsNotReady)
{
this.trackingUrl = WebAppUtils.GetURLWithScheme(schemePrefix, trackingUrl);
this.trackingUrlPretty = this.trackingUrl;
}
else
{
this.trackingUrlPretty = "UNASSIGNED";
}
this.applicationId = app.GetApplicationId();
this.applicationType = app.GetApplicationType();
this.appIdNum = app.GetApplicationId().GetId().ToString();
this.id = app.GetApplicationId().ToString();
this.user = app.GetUser().ToString();
this.name = app.GetName().ToString();
this.queue = app.GetQueue().ToString();
this.progress = app.GetProgress() * 100;
this.diagnostics = app.GetDiagnostics().ToString();
if (diagnostics == null || diagnostics.IsEmpty())
{
this.diagnostics = string.Empty;
}
if (app.GetApplicationTags() != null && !app.GetApplicationTags().IsEmpty())
{
this.applicationTags = Joiner.On(',').Join(app.GetApplicationTags());
}
this.finalStatus = app.GetFinalApplicationStatus();
this.clusterId = ResourceManager.GetClusterTimeStamp();
if (hasAccess)
{
this.startedTime = app.GetStartTime();
this.finishedTime = app.GetFinishTime();
this.elapsedTime = Times.Elapsed(app.GetStartTime(), app.GetFinishTime());
RMAppAttempt attempt = app.GetCurrentAppAttempt();
if (attempt != null)
{
Container masterContainer = attempt.GetMasterContainer();
if (masterContainer != null)
{
this.amContainerLogsExist = true;
this.amContainerLogs = WebAppUtils.GetRunningLogURL(schemePrefix + masterContainer
.GetNodeHttpAddress(), ConverterUtils.ToString(masterContainer.GetId()), app.GetUser
());
this.amHostHttpAddress = masterContainer.GetNodeHttpAddress();
}
ApplicationResourceUsageReport resourceReport = attempt.GetApplicationResourceUsageReport
();
if (resourceReport != null)
{
Resource usedResources = resourceReport.GetUsedResources();
allocatedMB = usedResources.GetMemory();
allocatedVCores = usedResources.GetVirtualCores();
runningContainers = resourceReport.GetNumUsedContainers();
}
resourceRequests = ((AbstractYarnScheduler)rm.GetRMContext().GetScheduler()).GetPendingResourceRequestsForAttempt
(attempt.GetAppAttemptId());
}
}
// copy preemption info fields
RMAppMetrics appMetrics = app.GetRMAppMetrics();
numAMContainerPreempted = appMetrics.GetNumAMContainersPreempted();
preemptedResourceMB = appMetrics.GetResourcePreempted().GetMemory();
numNonAMContainerPreempted = appMetrics.GetNumNonAMContainersPreempted();
preemptedResourceVCores = appMetrics.GetResourcePreempted().GetVirtualCores();
memorySeconds = appMetrics.GetMemorySeconds();
vcoreSeconds = appMetrics.GetVcoreSeconds();
}
}
