public override string ToString()
{
return(string.Format("[AchievementGroupInfo: id={0}, name={1}, isIncrement={2}, isInGame={3}, requirementArray={4}, currentCompleted={5}, rewardedIndex={6}]",
id,
name,
isIncrement,
isInGame,
StringUtils.ArrayToString(achievementInfos),
currentCompleted,
rewardedIndex
));
}
// TODO: Don't create again and again.
/// <exception cref="System.IO.IOException"/>
private ContainerLaunchContext CreateAMContainerLaunchContext(ApplicationSubmissionContext
applicationMasterContext, ContainerId containerID)
{
// Construct the actual Container
ContainerLaunchContext container = applicationMasterContext.GetAMContainerSpec();
Log.Info("Command to launch container " + containerID + " : " + StringUtils.ArrayToString
(Sharpen.Collections.ToArray(container.GetCommands(), new string[0])));
// Finalize the container
SetupTokens(container, containerID);
return(container);
}
public void ArrayToStringTest()
{
string[] test = new string[] { "a", "b", "c" };
string expected1 = "a, b, c";
string expected2 = "a,b,c";
string expected3 = "a b c";
string expected4 = "afoobfooc";
Assert.AreEqual(expected1, StringUtils.ArrayToString(test));
Assert.AreEqual(expected2, StringUtils.ArrayToString(test, ","));
Assert.AreEqual(expected3, StringUtils.ArrayToString(test, " "));
Assert.AreEqual(expected4, StringUtils.ArrayToString(test, "foo"));
}
public virtual string[] GetQueues(string queue)
{
Log.Debug("CSConf - getQueues called for: queuePrefix=" + GetQueuePrefix(queue));
string[] queues = GetStrings(GetQueuePrefix(queue) + Queues);
IList <string> trimmedQueueNames = new AList <string>();
if (null != queues)
{
foreach (string s in queues)
{
trimmedQueueNames.AddItem(s.Trim());
}
queues = Sharpen.Collections.ToArray(trimmedQueueNames, new string[0]);
}
Log.Debug("CSConf - getQueues: queuePrefix=" + GetQueuePrefix(queue) + ", queues="
+ ((queues == null) ? string.Empty : StringUtils.ArrayToString(queues)));
return(queues);
}
/// <summary>
/// Set up the DistributedCache related configs to make
/// <see cref="Org.Apache.Hadoop.Mapreduce.Filecache.DistributedCache.GetLocalCacheFiles(Org.Apache.Hadoop.Conf.Configuration)
/// "/>
/// and
/// <see cref="Org.Apache.Hadoop.Mapreduce.Filecache.DistributedCache.GetLocalCacheArchives(Org.Apache.Hadoop.Conf.Configuration)
/// "/>
/// working.
/// </summary>
/// <param name="conf"/>
/// <exception cref="System.IO.IOException"/>
public static void SetupDistributedCacheLocal(Configuration conf)
{
string localWorkDir = Runtime.Getenv("PWD");
// ^ ^ all symlinks are created in the current work-dir
// Update the configuration object with localized archives.
URI[] cacheArchives = DistributedCache.GetCacheArchives(conf);
if (cacheArchives != null)
{
IList <string> localArchives = new AList <string>();
for (int i = 0; i < cacheArchives.Length; ++i)
{
URI u = cacheArchives[i];
Path p = new Path(u);
Path name = new Path((null == u.GetFragment()) ? p.GetName() : u.GetFragment());
string linkName = name.ToUri().GetPath();
localArchives.AddItem(new Path(localWorkDir, linkName).ToUri().GetPath());
}
if (!localArchives.IsEmpty())
{
conf.Set(MRJobConfig.CacheLocalarchives, StringUtils.ArrayToString(Sharpen.Collections.ToArray
(localArchives, new string[localArchives.Count])));
}
}
// Update the configuration object with localized files.
URI[] cacheFiles = DistributedCache.GetCacheFiles(conf);
if (cacheFiles != null)
{
IList <string> localFiles = new AList <string>();
for (int i = 0; i < cacheFiles.Length; ++i)
{
URI u = cacheFiles[i];
Path p = new Path(u);
Path name = new Path((null == u.GetFragment()) ? p.GetName() : u.GetFragment());
string linkName = name.ToUri().GetPath();
localFiles.AddItem(new Path(localWorkDir, linkName).ToUri().GetPath());
}
if (!localFiles.IsEmpty())
{
conf.Set(MRJobConfig.CacheLocalfiles, StringUtils.ArrayToString(Sharpen.Collections.ToArray
(localFiles, new string[localFiles.Count])));
}
}
}
/// <returns>the option set to be used in command line parsing</returns>
private Options GetOptions()
{
Options cliopt = new Options();
cliopt.AddOption(ConfigOption.Maps);
cliopt.AddOption(ConfigOption.Reduces);
cliopt.AddOption(ConfigOption.PacketSize);
cliopt.AddOption(ConfigOption.Ops);
cliopt.AddOption(ConfigOption.Duration);
cliopt.AddOption(ConfigOption.ExitOnError);
cliopt.AddOption(ConfigOption.SleepTime);
cliopt.AddOption(ConfigOption.TruncateWait);
cliopt.AddOption(ConfigOption.Files);
cliopt.AddOption(ConfigOption.DirSize);
cliopt.AddOption(ConfigOption.BaseDir);
cliopt.AddOption(ConfigOption.ResultFile);
cliopt.AddOption(ConfigOption.Cleanup);
{
string[] distStrs = new string[Constants.Distribution.Values().Length];
Constants.Distribution[] distValues = Constants.Distribution.Values();
for (int i = 0; i < distValues.Length; ++i)
{
distStrs[i] = distValues[i].LowerName();
}
string opdesc = string.Format(Constants.OpDescr, StringUtils.ArrayToString(distStrs
));
foreach (Constants.OperationType type in Constants.OperationType.Values())
{
string opname = type.LowerName();
cliopt.AddOption(new Option(opname, true, opdesc));
}
}
cliopt.AddOption(ConfigOption.ReplicationAm);
cliopt.AddOption(ConfigOption.BlockSize);
cliopt.AddOption(ConfigOption.ReadSize);
cliopt.AddOption(ConfigOption.WriteSize);
cliopt.AddOption(ConfigOption.AppendSize);
cliopt.AddOption(ConfigOption.TruncateSize);
cliopt.AddOption(ConfigOption.RandomSeed);
cliopt.AddOption(ConfigOption.QueueName);
cliopt.AddOption(ConfigOption.Help);
return(cliopt);
}
/// <summary>
/// Set up the distributed cache by localizing the resources, and updating
/// the configuration with references to the localized resources.
/// </summary>
/// <param name="conf"/>
/// <exception cref="System.IO.IOException"/>
public virtual void Setup(JobConf conf)
{
FilePath workDir = new FilePath(Runtime.GetProperty("user.dir"));
// Generate YARN local resources objects corresponding to the distributed
// cache configuration
IDictionary <string, LocalResource> localResources = new LinkedHashMap <string, LocalResource
>();
MRApps.SetupDistributedCache(conf, localResources);
// Generating unique numbers for FSDownload.
AtomicLong uniqueNumberGenerator = new AtomicLong(Runtime.CurrentTimeMillis());
// Find which resources are to be put on the local classpath
IDictionary <string, Path> classpaths = new Dictionary <string, Path>();
Path[] archiveClassPaths = DistributedCache.GetArchiveClassPaths(conf);
if (archiveClassPaths != null)
{
foreach (Path p in archiveClassPaths)
{
classpaths[p.ToUri().GetPath().ToString()] = p;
}
}
Path[] fileClassPaths = DistributedCache.GetFileClassPaths(conf);
if (fileClassPaths != null)
{
foreach (Path p in fileClassPaths)
{
classpaths[p.ToUri().GetPath().ToString()] = p;
}
}
// Localize the resources
LocalDirAllocator localDirAllocator = new LocalDirAllocator(MRConfig.LocalDir);
FileContext localFSFileContext = FileContext.GetLocalFSFileContext();
UserGroupInformation ugi = UserGroupInformation.GetCurrentUser();
ExecutorService exec = null;
try
{
ThreadFactory tf = new ThreadFactoryBuilder().SetNameFormat("LocalDistributedCacheManager Downloader #%d"
).Build();
exec = Executors.NewCachedThreadPool(tf);
Path destPath = localDirAllocator.GetLocalPathForWrite(".", conf);
IDictionary <LocalResource, Future <Path> > resourcesToPaths = Maps.NewHashMap();
foreach (LocalResource resource in localResources.Values)
{
Callable <Path> download = new FSDownload(localFSFileContext, ugi, conf, new Path(
destPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet())), resource
);
Future <Path> future = exec.Submit(download);
resourcesToPaths[resource] = future;
}
foreach (KeyValuePair <string, LocalResource> entry in localResources)
{
LocalResource resource_1 = entry.Value;
Path path;
try
{
path = resourcesToPaths[resource_1].Get();
}
catch (Exception e)
{
throw new IOException(e);
}
catch (ExecutionException e)
{
throw new IOException(e);
}
string pathString = path.ToUri().ToString();
string link = entry.Key;
string target = new FilePath(path.ToUri()).GetPath();
Symlink(workDir, target, link);
if (resource_1.GetType() == LocalResourceType.Archive)
{
localArchives.AddItem(pathString);
}
else
{
if (resource_1.GetType() == LocalResourceType.File)
{
localFiles.AddItem(pathString);
}
else
{
if (resource_1.GetType() == LocalResourceType.Pattern)
{
//PATTERN is not currently used in local mode
throw new ArgumentException("Resource type PATTERN is not " + "implemented yet. "
+ resource_1.GetResource());
}
}
}
Path resourcePath;
try
{
resourcePath = ConverterUtils.GetPathFromYarnURL(resource_1.GetResource());
}
catch (URISyntaxException e)
{
throw new IOException(e);
}
Log.Info(string.Format("Localized %s as %s", resourcePath, path));
string cp = resourcePath.ToUri().GetPath();
if (classpaths.Keys.Contains(cp))
{
localClasspaths.AddItem(path.ToUri().GetPath().ToString());
}
}
}
finally
{
if (exec != null)
{
exec.Shutdown();
}
}
// Update the configuration object with localized data.
if (!localArchives.IsEmpty())
{
conf.Set(MRJobConfig.CacheLocalarchives, StringUtils.ArrayToString(Sharpen.Collections.ToArray
(localArchives, new string[localArchives.Count])));
}
if (!localFiles.IsEmpty())
{
conf.Set(MRJobConfig.CacheLocalfiles, StringUtils.ArrayToString(Sharpen.Collections.ToArray
(localFiles, new string[localArchives.Count])));
}
setupCalled = true;
}
void DrawIgnoredFilesAndFolders()
{
EditorGUI.indentLevel++;
EditorGUILayout.LabelField(new GUIContent("Ignored Files and Folders", "File and folders that will be ignored when adding folders to change files"));
EditorGUI.indentLevel--;
EditorGUILayout.BeginVertical(_style.IndentedBox());
EditorGUILayout.BeginHorizontal();
EditorGUILayout.LabelField(new GUIContent("Default Ignored Files", "These files are always ignored when adding folders"), GUILayout.ExpandWidth(false));
EditorGUILayout.LabelField(StringUtils.ArrayToString(IgnoredFiles.DefaultList), EditorStyles.wordWrappedLabel);
EditorGUILayout.EndHorizontal();
EditorGUILayout.LabelField(new GUIContent("Custom Ignored Files", "Add filenames and wildcards entries here to have them ignored when adding folders"), GUILayout.ExpandWidth(false));
EditorGUI.indentLevel++;
bool update = false;
int remove = -1;
for (int ii = 0; ii < _customIgnoredFiles.Count; ++ii)
{
EditorGUILayout.BeginHorizontal();
EditorGUI.BeginChangeCheck();
_customIgnoredFiles[ii] = EditorGUILayout.TextField(_customIgnoredFiles[ii]);
if (EditorGUI.EndChangeCheck())
{
update = true;
}
EditorGUILayout.Space();
if (_style.MinusButton("Remove ignored file pattern"))
{
bool removeCheck = true;
if (!string.IsNullOrEmpty(_customIgnoredFiles[ii].Trim()))
{
removeCheck = EditorUtility.DisplayDialog("Remove ignored file pattern?", "Are you sure you want to remove \"" + _customIgnoredFiles[ii] + "\" from the ignore list?", "Remove", "Cancel");
}
if (removeCheck)
{
remove = ii;
}
}
EditorGUILayout.EndHorizontal();
GUILayout.Space(2);
}
if (remove > -1)
{
IgnoredFiles.Remove(_customIgnoredFiles[remove]);
_customIgnoredFiles.RemoveAt(remove);
_settings.SetDirty();
_settings.Save();
}
if (update)
{
IgnoredFiles.SetIngnoredFiles(_customIgnoredFiles.ToArray());
_settings.SetDirty();
_settings.Save();
update = false;
}
EditorGUILayout.BeginHorizontal();
EditorGUILayout.Space();
if (_style.PlusButton("Add new ignored file pattern"))
{
_customIgnoredFiles.Add("");
}
EditorGUILayout.EndHorizontal();
EditorGUI.indentLevel--;
GUILayout.Space(4);
EditorGUILayout.EndVertical();
}
public void EmptyArrayToStringTest()
{
Assert.AreEqual("", StringUtils.ArrayToString(null));
Assert.AreEqual("", StringUtils.ArrayToString(new string[] {}));
}
public override string ToString()
{
return(string.Format("[RFGEProdMap: gwidth2={0}, nf={1}]",
StringUtils.ArrayToString(gwidth2), numFeatures));
}
public Gamma PrecisionAverageConditional(Gamma precision)
{
Console.WriteLine("{0}.PrecisionAverageConditional. precision: {1}",
typeof(KEP_CGFacOpIns), precision);
var incom = new IKEPDist[] { new DGamma(precision) };
Gamma predictOut;
// The array should have length 1 as we have one incoming message.
Vector[] randomFeatures = null;
watch.Start();
bool isUn = true;
bool onlineReady = toPrecisionMap.IsOnlineReady();
double[] uncertainty = null;
if (onlineReady)
{
randomFeatures = toPrecisionMap.GenAllRandomFeatures(incom);
Debug.Assert(randomFeatures.Length == 2,
"Should have 2 feature vectors: one for shape, one for rate.");
double[] thresh = toPrecisionMap.GetUncertaintyThreshold();
uncertainty = toPrecisionMap.EstimateUncertainty(randomFeatures);
isUn = MatrixUtils.SomeGeq(uncertainty, thresh);
}
watch.Stop();
if (isUn)
{
watch.Start();
// Operator is not certain. Query the oracle.
Gamma oracleOut = CGFacOp.PrecisionAverageConditional(precision);
DGamma target = new DGamma(oracleOut);
// TODO: This line can be improved by passing the random features
toPrecisionMap.UpdateOperator(target, incom);
watch.Stop();
if (IsRecordMessages)
{
if (onlineReady)
{
// What to do if this is improper ?
DGamma rawOut = toPrecisionMap.MapToDistFromRandomFeatures(randomFeatures);
predictOut = (Gamma)rawOut.GetWrappedDistribution();
double[] logPredVar = uncertainty;
record.Record(precision, predictOut, true, logPredVar, oracleOut);
}
else
{
// Not record the predicted messages
double[] logPredVar = new double[] { double.NaN, double.NaN };
record.Record(precision, null, true, logPredVar, oracleOut);
}
}
return(oracleOut);
}
else
{
// Operator is sure
watch.Start();
DGamma rawOut = toPrecisionMap.MapToDistFromRandomFeatures(randomFeatures);
predictOut = (Gamma)rawOut.GetWrappedDistribution();
watch.Stop();
double[] logPredVar = uncertainty;
Console.WriteLine(" ** Certain with log predictive variance: {0}",
StringUtils.ArrayToString(logPredVar));
Console.WriteLine("Predicted outgoing: {0}", predictOut);
Gamma?oracleOut = null;
if (IsPrintTrueWhenCertain)
{
oracleOut = CGFacOp.PrecisionAverageConditional(precision);
Console.WriteLine("oracle outgoing: {0}", oracleOut);
}
Console.WriteLine();
if (IsRecordMessages)
{
// compute oracle's outgoing
if (oracleOut == null)
{
oracleOut = CGFacOp.PrecisionAverageConditional(precision);
}
record.Record(precision, predictOut, false, logPredVar, oracleOut.Value);
}
return(predictOut);
}
}
// Input output paths for this..
// these are all dummy and does not test
// much in map reduce except for the command line
// params
/// <exception cref="System.Exception"/>
public virtual void TestJobShell()
{
MiniDFSCluster dfs = null;
MiniMRCluster mr = null;
FileSystem fs = null;
Path testFile = new Path(input, "testfile");
try
{
Configuration conf = new Configuration();
//start the mini mr and dfs cluster.
dfs = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build();
fs = dfs.GetFileSystem();
FSDataOutputStream stream = fs.Create(testFile);
stream.Write(Sharpen.Runtime.GetBytesForString("teststring"));
stream.Close();
mr = new MiniMRCluster(2, fs.GetUri().ToString(), 1);
FilePath thisbuildDir = new FilePath(buildDir, "jobCommand");
NUnit.Framework.Assert.IsTrue("create build dir", thisbuildDir.Mkdirs());
FilePath f = new FilePath(thisbuildDir, "files_tmp");
FileOutputStream fstream = new FileOutputStream(f);
fstream.Write(Sharpen.Runtime.GetBytesForString("somestrings"));
fstream.Close();
FilePath f1 = new FilePath(thisbuildDir, "files_tmp1");
fstream = new FileOutputStream(f1);
fstream.Write(Sharpen.Runtime.GetBytesForString("somestrings"));
fstream.Close();
// copy files to dfs
Path cachePath = new Path("/cacheDir");
if (!fs.Mkdirs(cachePath))
{
throw new IOException("Mkdirs failed to create " + cachePath.ToString());
}
Path localCachePath = new Path(Runtime.GetProperty("test.cache.data"));
Path txtPath = new Path(localCachePath, new Path("test.txt"));
Path jarPath = new Path(localCachePath, new Path("test.jar"));
Path zipPath = new Path(localCachePath, new Path("test.zip"));
Path tarPath = new Path(localCachePath, new Path("test.tar"));
Path tgzPath = new Path(localCachePath, new Path("test.tgz"));
fs.CopyFromLocalFile(txtPath, cachePath);
fs.CopyFromLocalFile(jarPath, cachePath);
fs.CopyFromLocalFile(zipPath, cachePath);
// construct options for -files
string[] files = new string[3];
files[0] = f.ToString();
files[1] = f1.ToString() + "#localfilelink";
files[2] = fs.GetUri().Resolve(cachePath + "/test.txt#dfsfilelink").ToString();
// construct options for -libjars
string[] libjars = new string[2];
libjars[0] = "build/test/mapred/testjar/testjob.jar";
libjars[1] = fs.GetUri().Resolve(cachePath + "/test.jar").ToString();
// construct options for archives
string[] archives = new string[3];
archives[0] = tgzPath.ToString();
archives[1] = tarPath + "#tarlink";
archives[2] = fs.GetUri().Resolve(cachePath + "/test.zip#ziplink").ToString();
string[] args = new string[10];
args[0] = "-files";
args[1] = StringUtils.ArrayToString(files);
args[2] = "-libjars";
// the testjob.jar as a temporary jar file
// rather than creating its own
args[3] = StringUtils.ArrayToString(libjars);
args[4] = "-archives";
args[5] = StringUtils.ArrayToString(archives);
args[6] = "-D";
args[7] = "mapred.output.committer.class=testjar.CustomOutputCommitter";
args[8] = input.ToString();
args[9] = output.ToString();
JobConf jobConf = mr.CreateJobConf();
//before running the job, verify that libjar is not in client classpath
NUnit.Framework.Assert.IsTrue("libjar not in client classpath", LoadLibJar(jobConf
) == null);
int ret = ToolRunner.Run(jobConf, new ExternalMapReduce(), args);
//after running the job, verify that libjar is in the client classpath
NUnit.Framework.Assert.IsTrue("libjar added to client classpath", LoadLibJar(jobConf
) != null);
NUnit.Framework.Assert.IsTrue("not failed ", ret != -1);
f.Delete();
thisbuildDir.Delete();
}
finally
{
if (dfs != null)
{
dfs.Shutdown();
}
if (mr != null)
{
mr.Shutdown();
}
}
}
public virtual void SetQueues(string queue, string[] subQueues)
{
Set(GetQueuePrefix(queue) + Queues, StringUtils.ArrayToString(subQueues));
Log.Debug("CSConf - setQueues: qPrefix=" + GetQueuePrefix(queue) + ", queues=" +
StringUtils.ArrayToString(subQueues));
}
