/// <exception cref="System.IO.IOException"/>
public static void AddClasspathToEnv(IDictionary <string, string> environment, string
classpathEnvVar, Configuration conf)
{
MRApps.AddToEnvironment(environment, classpathEnvVar, MRJobConfig.JobJar + Path.Separator
+ MRJobConfig.JobJar, conf);
MRApps.AddToEnvironment(environment, classpathEnvVar, MRJobConfig.JobJar + Path.Separator
+ "classes" + Path.Separator, conf);
MRApps.AddToEnvironment(environment, classpathEnvVar, MRJobConfig.JobJar + Path.Separator
+ "lib" + Path.Separator + "*", conf);
MRApps.AddToEnvironment(environment, classpathEnvVar, CrossPlatformifyMREnv(conf,
ApplicationConstants.Environment.Pwd) + Path.Separator + "*", conf);
// a * in the classpath will only find a .jar, so we need to filter out
// all .jars and add everything else
AddToClasspathIfNotJar(DistributedCache.GetFileClassPaths(conf), DistributedCache
.GetCacheFiles(conf), conf, environment, classpathEnvVar);
AddToClasspathIfNotJar(DistributedCache.GetArchiveClassPaths(conf), DistributedCache
.GetCacheArchives(conf), conf, environment, classpathEnvVar);
}
/// <summary>
/// Set up the DistributedCache related configs to make
/// <see cref="Org.Apache.Hadoop.Mapreduce.Filecache.DistributedCache.GetLocalCacheFiles(Org.Apache.Hadoop.Conf.Configuration)
/// "/>
/// and
/// <see cref="Org.Apache.Hadoop.Mapreduce.Filecache.DistributedCache.GetLocalCacheArchives(Org.Apache.Hadoop.Conf.Configuration)
/// "/>
/// working.
/// </summary>
/// <param name="conf"/>
/// <exception cref="System.IO.IOException"/>
public static void SetupDistributedCacheLocal(Configuration conf)
{
string localWorkDir = Runtime.Getenv("PWD");
// ^ ^ all symlinks are created in the current work-dir
// Update the configuration object with localized archives.
URI[] cacheArchives = DistributedCache.GetCacheArchives(conf);
if (cacheArchives != null)
{
IList <string> localArchives = new AList <string>();
for (int i = 0; i < cacheArchives.Length; ++i)
{
URI u = cacheArchives[i];
Path p = new Path(u);
Path name = new Path((null == u.GetFragment()) ? p.GetName() : u.GetFragment());
string linkName = name.ToUri().GetPath();
localArchives.AddItem(new Path(localWorkDir, linkName).ToUri().GetPath());
}
if (!localArchives.IsEmpty())
{
conf.Set(MRJobConfig.CacheLocalarchives, StringUtils.ArrayToString(Sharpen.Collections.ToArray
(localArchives, new string[localArchives.Count])));
}
}
// Update the configuration object with localized files.
URI[] cacheFiles = DistributedCache.GetCacheFiles(conf);
if (cacheFiles != null)
{
IList <string> localFiles = new AList <string>();
for (int i = 0; i < cacheFiles.Length; ++i)
{
URI u = cacheFiles[i];
Path p = new Path(u);
Path name = new Path((null == u.GetFragment()) ? p.GetName() : u.GetFragment());
string linkName = name.ToUri().GetPath();
localFiles.AddItem(new Path(localWorkDir, linkName).ToUri().GetPath());
}
if (!localFiles.IsEmpty())
{
conf.Set(MRJobConfig.CacheLocalfiles, StringUtils.ArrayToString(Sharpen.Collections.ToArray
(localFiles, new string[localFiles.Count])));
}
}
}
/// <summary>Get cache files set in the Configuration</summary>
/// <returns>A URI array of the files set in the Configuration</returns>
/// <exception cref="System.IO.IOException"/>
public virtual URI[] GetCacheFiles()
{
return(DistributedCache.GetCacheFiles(conf));
}
/// <exception cref="System.Exception"/>
public virtual void TestDeprecatedFunctions()
{
DistributedCache.AddLocalArchives(conf, "Test Local Archives 1");
NUnit.Framework.Assert.AreEqual("Test Local Archives 1", conf.Get(DistributedCache
.CacheLocalarchives));
NUnit.Framework.Assert.AreEqual(1, DistributedCache.GetLocalCacheArchives(conf).Length
);
NUnit.Framework.Assert.AreEqual("Test Local Archives 1", DistributedCache.GetLocalCacheArchives
(conf)[0].GetName());
DistributedCache.AddLocalArchives(conf, "Test Local Archives 2");
NUnit.Framework.Assert.AreEqual("Test Local Archives 1,Test Local Archives 2", conf
.Get(DistributedCache.CacheLocalarchives));
NUnit.Framework.Assert.AreEqual(2, DistributedCache.GetLocalCacheArchives(conf).Length
);
NUnit.Framework.Assert.AreEqual("Test Local Archives 2", DistributedCache.GetLocalCacheArchives
(conf)[1].GetName());
DistributedCache.SetLocalArchives(conf, "Test Local Archives 3");
NUnit.Framework.Assert.AreEqual("Test Local Archives 3", conf.Get(DistributedCache
.CacheLocalarchives));
NUnit.Framework.Assert.AreEqual(1, DistributedCache.GetLocalCacheArchives(conf).Length
);
NUnit.Framework.Assert.AreEqual("Test Local Archives 3", DistributedCache.GetLocalCacheArchives
(conf)[0].GetName());
DistributedCache.AddLocalFiles(conf, "Test Local Files 1");
NUnit.Framework.Assert.AreEqual("Test Local Files 1", conf.Get(DistributedCache.CacheLocalfiles
));
NUnit.Framework.Assert.AreEqual(1, DistributedCache.GetLocalCacheFiles(conf).Length
);
NUnit.Framework.Assert.AreEqual("Test Local Files 1", DistributedCache.GetLocalCacheFiles
(conf)[0].GetName());
DistributedCache.AddLocalFiles(conf, "Test Local Files 2");
NUnit.Framework.Assert.AreEqual("Test Local Files 1,Test Local Files 2", conf.Get
(DistributedCache.CacheLocalfiles));
NUnit.Framework.Assert.AreEqual(2, DistributedCache.GetLocalCacheFiles(conf).Length
);
NUnit.Framework.Assert.AreEqual("Test Local Files 2", DistributedCache.GetLocalCacheFiles
(conf)[1].GetName());
DistributedCache.SetLocalFiles(conf, "Test Local Files 3");
NUnit.Framework.Assert.AreEqual("Test Local Files 3", conf.Get(DistributedCache.CacheLocalfiles
));
NUnit.Framework.Assert.AreEqual(1, DistributedCache.GetLocalCacheFiles(conf).Length
);
NUnit.Framework.Assert.AreEqual("Test Local Files 3", DistributedCache.GetLocalCacheFiles
(conf)[0].GetName());
DistributedCache.SetArchiveTimestamps(conf, "1234567890");
NUnit.Framework.Assert.AreEqual(1234567890, conf.GetLong(DistributedCache.CacheArchivesTimestamps
, 0));
NUnit.Framework.Assert.AreEqual(1, DistributedCache.GetArchiveTimestamps(conf).Length
);
NUnit.Framework.Assert.AreEqual(1234567890, DistributedCache.GetArchiveTimestamps
(conf)[0]);
DistributedCache.SetFileTimestamps(conf, "1234567890");
NUnit.Framework.Assert.AreEqual(1234567890, conf.GetLong(DistributedCache.CacheFilesTimestamps
, 0));
NUnit.Framework.Assert.AreEqual(1, DistributedCache.GetFileTimestamps(conf).Length
);
NUnit.Framework.Assert.AreEqual(1234567890, DistributedCache.GetFileTimestamps(conf
)[0]);
DistributedCache.CreateAllSymlink(conf, new FilePath("Test Job Cache Dir"), new FilePath
("Test Work Dir"));
NUnit.Framework.Assert.IsNull(conf.Get(DistributedCache.CacheSymlink));
NUnit.Framework.Assert.IsTrue(DistributedCache.GetSymlink(conf));
NUnit.Framework.Assert.IsTrue(symlinkFile.CreateNewFile());
FileStatus fileStatus = DistributedCache.GetFileStatus(conf, symlinkFile.ToURI());
NUnit.Framework.Assert.IsNotNull(fileStatus);
NUnit.Framework.Assert.AreEqual(fileStatus.GetModificationTime(), DistributedCache
.GetTimestamp(conf, symlinkFile.ToURI()));
NUnit.Framework.Assert.IsTrue(symlinkFile.Delete());
DistributedCache.AddCacheArchive(symlinkFile.ToURI(), conf);
NUnit.Framework.Assert.AreEqual(symlinkFile.ToURI().ToString(), conf.Get(DistributedCache
.CacheArchives));
NUnit.Framework.Assert.AreEqual(1, DistributedCache.GetCacheArchives(conf).Length
);
NUnit.Framework.Assert.AreEqual(symlinkFile.ToURI(), DistributedCache.GetCacheArchives
(conf)[0]);
DistributedCache.AddCacheFile(symlinkFile.ToURI(), conf);
NUnit.Framework.Assert.AreEqual(symlinkFile.ToURI().ToString(), conf.Get(DistributedCache
.CacheFiles));
NUnit.Framework.Assert.AreEqual(1, DistributedCache.GetCacheFiles(conf).Length);
NUnit.Framework.Assert.AreEqual(symlinkFile.ToURI(), DistributedCache.GetCacheFiles
(conf)[0]);
}
/// <exception cref="System.IO.IOException"/>
public static void SetupDistributedCache(Configuration conf, IDictionary <string,
LocalResource> localResources)
{
// Cache archives
ParseDistributedCacheArtifacts(conf, localResources, LocalResourceType.Archive, DistributedCache
.GetCacheArchives(conf), DistributedCache.GetArchiveTimestamps(conf), GetFileSizes
(conf, MRJobConfig.CacheArchivesSizes), DistributedCache.GetArchiveVisibilities(
conf));
// Cache files
ParseDistributedCacheArtifacts(conf, localResources, LocalResourceType.File, DistributedCache
.GetCacheFiles(conf), DistributedCache.GetFileTimestamps(conf), GetFileSizes(conf
, MRJobConfig.CacheFilesSizes), DistributedCache.GetFileVisibilities(conf));
}
/// <exception cref="System.IO.IOException"/>
private static void SetupPipesJob(JobConf conf)
{
// default map output types to Text
if (!GetIsJavaMapper(conf))
{
conf.SetMapRunnerClass(typeof(PipesMapRunner));
// Save the user's partitioner and hook in our's.
SetJavaPartitioner(conf, conf.GetPartitionerClass());
conf.SetPartitionerClass(typeof(PipesPartitioner));
}
if (!GetIsJavaReducer(conf))
{
conf.SetReducerClass(typeof(PipesReducer));
if (!GetIsJavaRecordWriter(conf))
{
conf.SetOutputFormat(typeof(NullOutputFormat));
}
}
string textClassname = typeof(Text).FullName;
SetIfUnset(conf, MRJobConfig.MapOutputKeyClass, textClassname);
SetIfUnset(conf, MRJobConfig.MapOutputValueClass, textClassname);
SetIfUnset(conf, MRJobConfig.OutputKeyClass, textClassname);
SetIfUnset(conf, MRJobConfig.OutputValueClass, textClassname);
// Use PipesNonJavaInputFormat if necessary to handle progress reporting
// from C++ RecordReaders ...
if (!GetIsJavaRecordReader(conf) && !GetIsJavaMapper(conf))
{
conf.SetClass(Org.Apache.Hadoop.Mapred.Pipes.Submitter.InputFormat, conf.GetInputFormat
().GetType(), typeof(InputFormat));
conf.SetInputFormat(typeof(PipesNonJavaInputFormat));
}
string exec = GetExecutable(conf);
if (exec == null)
{
throw new ArgumentException("No application program defined.");
}
// add default debug script only when executable is expressed as
// <path>#<executable>
if (exec.Contains("#"))
{
// set default gdb commands for map and reduce task
string defScript = "$HADOOP_PREFIX/src/c++/pipes/debug/pipes-default-script";
SetIfUnset(conf, MRJobConfig.MapDebugScript, defScript);
SetIfUnset(conf, MRJobConfig.ReduceDebugScript, defScript);
}
URI[] fileCache = DistributedCache.GetCacheFiles(conf);
if (fileCache == null)
{
fileCache = new URI[1];
}
else
{
URI[] tmp = new URI[fileCache.Length + 1];
System.Array.Copy(fileCache, 0, tmp, 1, fileCache.Length);
fileCache = tmp;
}
try
{
fileCache[0] = new URI(exec);
}
catch (URISyntaxException e)
{
IOException ie = new IOException("Problem parsing execable URI " + exec);
Sharpen.Extensions.InitCause(ie, e);
throw ie;
}
DistributedCache.SetCacheFiles(fileCache, conf);
}
