/// <summary>given user name - get all the groups.</summary>
/// <remarks>
/// given user name - get all the groups.
/// Needs to happen before creating the test users
/// </remarks>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void TestGetServerSideGroups()
{
// get the user name
SystemProcess pp = Runtime.GetRuntime().Exec("whoami");
BufferedReader br = new BufferedReader(new InputStreamReader(pp.GetInputStream())
);
string userName = br.ReadLine().Trim();
// If on windows domain, token format is DOMAIN\\user and we want to
// extract only the user name
if (Shell.Windows)
{
int sp = userName.LastIndexOf('\\');
if (sp != -1)
{
userName = Runtime.Substring(userName, sp + 1);
}
// user names are case insensitive on Windows. Make consistent
userName = StringUtils.ToLowerCase(userName);
}
// get the groups
pp = Runtime.GetRuntime().Exec(Shell.Windows ? Shell.Winutils + " groups -F" : "id -Gn"
);
br = new BufferedReader(new InputStreamReader(pp.GetInputStream()));
string line = br.ReadLine();
System.Console.Out.WriteLine(userName + ":" + line);
ICollection <string> groups = new LinkedHashSet <string>();
string[] tokens = line.Split(Shell.TokenSeparatorRegex);
foreach (string s in tokens)
{
groups.AddItem(s);
}
UserGroupInformation login = UserGroupInformation.GetCurrentUser();
string loginUserName = login.GetShortUserName();
if (Shell.Windows)
{
// user names are case insensitive on Windows. Make consistent
loginUserName = StringUtils.ToLowerCase(loginUserName);
}
Assert.Equal(userName, loginUserName);
string[] gi = login.GetGroupNames();
Assert.Equal(groups.Count, gi.Length);
for (int i = 0; i < gi.Length; i++)
{
Assert.True(groups.Contains(gi[i]));
}
UserGroupInformation fakeUser = UserGroupInformation.CreateRemoteUser("foo.bar");
fakeUser.DoAs(new _PrivilegedExceptionAction_248(login, fakeUser));
}
public virtual void TestProxyUserFromEnvironment()
{
string proxyUser = "******";
Runtime.SetProperty(UserGroupInformation.HadoopProxyUser, proxyUser);
UserGroupInformation ugi = UserGroupInformation.GetLoginUser();
Assert.Equal(proxyUser, ugi.GetUserName());
UserGroupInformation realUgi = ugi.GetRealUser();
NUnit.Framework.Assert.IsNotNull(realUgi);
// get the expected real user name
SystemProcess pp = Runtime.GetRuntime().Exec("whoami");
BufferedReader br = new BufferedReader(new InputStreamReader(pp.GetInputStream())
);
string realUser = br.ReadLine().Trim();
// On Windows domain joined machine, whoami returns the username
// in the DOMAIN\\username format, so we trim the domain part before
// the comparison. We don't have to special case for Windows
// given that Unix systems do not allow slashes in usernames.
int backslashIndex = realUser.IndexOf('\\');
if (backslashIndex != -1)
{
realUser = Runtime.Substring(realUser, backslashIndex + 1);
}
Assert.Equal(realUser, realUgi.GetUserName());
}
/// <summary>Execute a command and return a single line of output as a String</summary>
/// <param name="dir">Working directory for the command</param>
/// <param name="command">as component array</param>
/// <param name="encoding"></param>
/// <returns>the one-line output of the command</returns>
protected internal static string ReadPipe(FilePath dir, string[] command, string
encoding)
{
try
{
SystemProcess p = Runtime.GetRuntime().Exec(command, null, dir);
BufferedReader lineRead = new BufferedReader(new InputStreamReader(p.GetInputStream
(), encoding));
string r = null;
try
{
r = lineRead.ReadLine();
}
finally
{
p.GetOutputStream().Close();
p.GetErrorStream().Close();
lineRead.Close();
}
for (; ;)
{
try
{
if (p.WaitFor() == 0 && r != null && r.Length > 0)
{
return(r);
}
break;
}
catch (Exception)
{
}
}
}
catch (IOException e)
{
// Stop bothering me, I have a zombie to reap.
if (SystemReader.GetInstance().GetProperty("jgit.fs.debug") != null)
{
System.Console.Error.WriteLine(e);
}
}
// Ignore error (but report)
return(null);
}
/// <summary>Run a hook script in the repository, returning the exit status.</summary>
/// <remarks>Run a hook script in the repository, returning the exit status.</remarks>
/// <param name="db">repository the script should see in GIT_DIR environment</param>
/// <param name="hook">
/// path of the hook script to execute, must be executable file
/// type on this platform
/// </param>
/// <param name="args">arguments to pass to the hook script</param>
/// <returns>exit status code of the invoked hook</returns>
/// <exception cref="System.IO.IOException">the hook could not be executed</exception>
/// <exception cref="System.Exception">the caller was interrupted before the hook completed
/// </exception>
protected internal virtual int RunHook(Repository db, FilePath hook, params string
[] args)
{
string[] argv = new string[1 + args.Length];
argv[0] = hook.GetAbsolutePath();
System.Array.Copy(args, 0, argv, 1, args.Length);
IDictionary <string, string> env = CloneEnv();
env.Put("GIT_DIR", db.Directory.GetAbsolutePath());
PutPersonIdent(env, "AUTHOR", author);
PutPersonIdent(env, "COMMITTER", committer);
FilePath cwd = db.WorkTree;
SystemProcess p = Runtime.GetRuntime().Exec(argv, ToEnvArray(env), cwd);
p.GetOutputStream().Close();
p.GetErrorStream().Close();
p.GetInputStream().Close();
return(p.WaitFor());
}
/// <summary>Execute a command and return a single line of output as a String</summary>
/// <param name="dir">Working directory for the command</param>
/// <param name="command">as component array</param>
/// <param name="encoding"></param>
/// <returns>the one-line output of the command</returns>
protected internal static string ReadPipe(FilePath dir, string[] command, string
encoding)
{
try
{
SystemProcess p = Runtime.GetRuntime().Exec(command, null, dir);
BufferedReader lineRead = new BufferedReader(new InputStreamReader(p.GetInputStream
(), encoding));
string r = null;
try
{
r = lineRead.ReadLine();
}
finally
{
p.GetOutputStream().Close();
p.GetErrorStream().Close();
lineRead.Close();
}
for (; ;)
{
try
{
if (p.WaitFor() == 0 && r != null && r.Length > 0)
{
return(r);
}
break;
}
catch (Exception)
{
}
}
}
catch (IOException)
{
}
// Stop bothering me, I have a zombie to reap.
// ignore
return(null);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public virtual void LaunchAM(ApplicationAttemptId attemptId)
{
Credentials credentials = new Credentials();
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> token = rmClient.GetAMRMToken
(attemptId.GetApplicationId());
// Service will be empty but that's okay, we are just passing down only
// AMRMToken down to the real AM which eventually sets the correct
// service-address.
credentials.AddToken(token.GetService(), token);
FilePath tokenFile = FilePath.CreateTempFile("unmanagedAMRMToken", string.Empty,
new FilePath(Runtime.GetProperty("user.dir")));
try
{
FileUtil.Chmod(tokenFile.GetAbsolutePath(), "600");
}
catch (Exception ex)
{
throw new RuntimeException(ex);
}
tokenFile.DeleteOnExit();
DataOutputStream os = new DataOutputStream(new FileOutputStream(tokenFile, true));
credentials.WriteTokenStorageToStream(os);
os.Close();
IDictionary <string, string> env = Sharpen.Runtime.GetEnv();
AList <string> envAMList = new AList <string>();
bool setClasspath = false;
foreach (KeyValuePair <string, string> entry in env)
{
string key = entry.Key;
string value = entry.Value;
if (key.Equals("CLASSPATH"))
{
setClasspath = true;
if (classpath != null)
{
value = value + FilePath.pathSeparator + classpath;
}
}
envAMList.AddItem(key + "=" + value);
}
if (!setClasspath && classpath != null)
{
envAMList.AddItem("CLASSPATH=" + classpath);
}
ContainerId containerId = ContainerId.NewContainerId(attemptId, 0);
string hostname = Sharpen.Runtime.GetLocalHost().GetHostName();
envAMList.AddItem(ApplicationConstants.Environment.ContainerId.ToString() + "=" +
containerId);
envAMList.AddItem(ApplicationConstants.Environment.NmHost.ToString() + "=" + hostname
);
envAMList.AddItem(ApplicationConstants.Environment.NmHttpPort.ToString() + "=0");
envAMList.AddItem(ApplicationConstants.Environment.NmPort.ToString() + "=0");
envAMList.AddItem(ApplicationConstants.Environment.LocalDirs.ToString() + "= /tmp"
);
envAMList.AddItem(ApplicationConstants.AppSubmitTimeEnv + "=" + Runtime.CurrentTimeMillis
());
envAMList.AddItem(ApplicationConstants.ContainerTokenFileEnvName + "=" + tokenFile
.GetAbsolutePath());
string[] envAM = new string[envAMList.Count];
SystemProcess amProc = Runtime.GetRuntime().Exec(amCmd, Sharpen.Collections.ToArray
(envAMList, envAM));
BufferedReader errReader = new BufferedReader(new InputStreamReader(amProc.GetErrorStream
(), Sharpen.Extensions.GetEncoding("UTF-8")));
BufferedReader inReader = new BufferedReader(new InputStreamReader(amProc.GetInputStream
(), Sharpen.Extensions.GetEncoding("UTF-8")));
// read error and input streams as this would free up the buffers
// free the error stream buffer
Sharpen.Thread errThread = new _Thread_244(errReader);
Sharpen.Thread outThread = new _Thread_258(inReader);
try
{
errThread.Start();
outThread.Start();
}
catch (InvalidOperationException)
{
}
// wait for the process to finish and check the exit code
try
{
int exitCode = amProc.WaitFor();
Log.Info("AM process exited with value: " + exitCode);
}
catch (Exception e)
{
Sharpen.Runtime.PrintStackTrace(e);
}
finally
{
amCompleted = true;
}
try
{
// make sure that the error thread exits
// on Windows these threads sometimes get stuck and hang the execution
// timeout and join later after destroying the process.
errThread.Join();
outThread.Join();
errReader.Close();
inReader.Close();
}
catch (Exception ie)
{
Log.Info("ShellExecutor: Interrupted while reading the error/out stream", ie);
}
catch (IOException ioe)
{
Log.Warn("Error while closing the error/out stream", ioe);
}
amProc.Destroy();
}
/// <summary>Execute a command and return a single line of output as a String</summary>
/// <param name="dir">Working directory for the command</param>
/// <param name="command">as component array</param>
/// <param name="encoding"></param>
/// <returns>the one-line output of the command</returns>
protected internal static string ReadPipe(FilePath dir, string[] command, string
encoding)
{
bool debug = System.Boolean.Parse(SystemReader.GetInstance().GetProperty("jgit.fs.debug"
));
try
{
if (debug)
{
System.Console.Error.WriteLine("readpipe " + Arrays.AsList(command) + "," + dir);
}
SystemProcess p = Runtime.GetRuntime().Exec(command, null, dir);
BufferedReader lineRead = new BufferedReader(new InputStreamReader(p.GetInputStream
(), encoding));
p.GetOutputStream().Close();
AtomicBoolean gooblerFail = new AtomicBoolean(false);
Sharpen.Thread gobbler = new _Thread_300(p, debug, gooblerFail);
// ignore
// Just print on stderr for debugging
// Just print on stderr for debugging
gobbler.Start();
string r = null;
try
{
r = lineRead.ReadLine();
if (debug)
{
System.Console.Error.WriteLine("readpipe may return '" + r + "'");
System.Console.Error.WriteLine("(ignoring remaing output:");
}
string l;
while ((l = lineRead.ReadLine()) != null)
{
if (debug)
{
System.Console.Error.WriteLine(l);
}
}
}
finally
{
p.GetErrorStream().Close();
lineRead.Close();
}
for (; ;)
{
try
{
int rc = p.WaitFor();
gobbler.Join();
if (rc == 0 && r != null && r.Length > 0 && !gooblerFail.Get())
{
return(r);
}
if (debug)
{
System.Console.Error.WriteLine("readpipe rc=" + rc);
}
break;
}
catch (Exception)
{
}
}
}
catch (IOException e)
{
// Stop bothering me, I have a zombie to reap.
if (debug)
{
System.Console.Error.WriteLine(e);
}
}
// Ignore error (but report)
if (debug)
{
System.Console.Error.WriteLine("readpipe returns null");
}
return(null);
}
public static bool UpdateMapInternal(BiMap <int, string> map, string mapName, string
command, string regex, IDictionary <int, int> staticMapping)
{
bool updated = false;
BufferedReader br = null;
try
{
SystemProcess process = Runtime.GetRuntime().Exec(new string[] { "bash", "-c", command });
br = new BufferedReader(new InputStreamReader(process.GetInputStream(), Encoding.
Default));
string line = null;
while ((line = br.ReadLine()) != null)
{
string[] nameId = line.Split(regex);
if ((nameId == null) || (nameId.Length != 2))
{
throw new IOException("Can't parse " + mapName + " list entry:" + line);
}
Log.Debug("add to " + mapName + "map:" + nameId[0] + " id:" + nameId[1]);
// HDFS can't differentiate duplicate names with simple authentication
int key = staticMapping[ParseId(nameId[1])];
string value = nameId[0];
if (map.Contains(key))
{
string prevValue = map[key];
if (value.Equals(prevValue))
{
// silently ignore equivalent entries
continue;
}
ReportDuplicateEntry("Got multiple names associated with the same id: ", key, value
, key, prevValue);
continue;
}
if (map.ContainsValue(value))
{
int prevKey = map.Inverse()[value];
ReportDuplicateEntry("Got multiple ids associated with the same name: ", key, value
, prevKey, value);
continue;
}
map[key] = value;
updated = true;
}
Log.Debug("Updated " + mapName + " map size: " + map.Count);
}
catch (IOException e)
{
Log.Error("Can't update " + mapName + " map");
throw;
}
finally
{
if (br != null)
{
try
{
br.Close();
}
catch (IOException e1)
{
Log.Error("Can't close BufferedReader of command result", e1);
}
}
}
return(updated);
}