/// <exception cref="System.IO.IOException"/> public static void AddClasspathToEnv(IDictionary <string, string> environment, string classpathEnvVar, Configuration conf) { MRApps.AddToEnvironment(environment, classpathEnvVar, MRJobConfig.JobJar + Path.Separator + MRJobConfig.JobJar, conf); MRApps.AddToEnvironment(environment, classpathEnvVar, MRJobConfig.JobJar + Path.Separator + "classes" + Path.Separator, conf); MRApps.AddToEnvironment(environment, classpathEnvVar, MRJobConfig.JobJar + Path.Separator + "lib" + Path.Separator + "*", conf); MRApps.AddToEnvironment(environment, classpathEnvVar, CrossPlatformifyMREnv(conf, ApplicationConstants.Environment.Pwd) + Path.Separator + "*", conf); // a * in the classpath will only find a .jar, so we need to filter out // all .jars and add everything else AddToClasspathIfNotJar(DistributedCache.GetFileClassPaths(conf), DistributedCache .GetCacheFiles(conf), conf, environment, classpathEnvVar); AddToClasspathIfNotJar(DistributedCache.GetArchiveClassPaths(conf), DistributedCache .GetCacheArchives(conf), conf, environment, classpathEnvVar); }
/// <summary> /// Set up the DistributedCache related configs to make /// <see cref="Org.Apache.Hadoop.Mapreduce.Filecache.DistributedCache.GetLocalCacheFiles(Org.Apache.Hadoop.Conf.Configuration) /// "/> /// and /// <see cref="Org.Apache.Hadoop.Mapreduce.Filecache.DistributedCache.GetLocalCacheArchives(Org.Apache.Hadoop.Conf.Configuration) /// "/> /// working. /// </summary> /// <param name="conf"/> /// <exception cref="System.IO.IOException"/> public static void SetupDistributedCacheLocal(Configuration conf) { string localWorkDir = Runtime.Getenv("PWD"); // ^ ^ all symlinks are created in the current work-dir // Update the configuration object with localized archives. URI[] cacheArchives = DistributedCache.GetCacheArchives(conf); if (cacheArchives != null) { IList <string> localArchives = new AList <string>(); for (int i = 0; i < cacheArchives.Length; ++i) { URI u = cacheArchives[i]; Path p = new Path(u); Path name = new Path((null == u.GetFragment()) ? p.GetName() : u.GetFragment()); string linkName = name.ToUri().GetPath(); localArchives.AddItem(new Path(localWorkDir, linkName).ToUri().GetPath()); } if (!localArchives.IsEmpty()) { conf.Set(MRJobConfig.CacheLocalarchives, StringUtils.ArrayToString(Sharpen.Collections.ToArray (localArchives, new string[localArchives.Count]))); } } // Update the configuration object with localized files. URI[] cacheFiles = DistributedCache.GetCacheFiles(conf); if (cacheFiles != null) { IList <string> localFiles = new AList <string>(); for (int i = 0; i < cacheFiles.Length; ++i) { URI u = cacheFiles[i]; Path p = new Path(u); Path name = new Path((null == u.GetFragment()) ? p.GetName() : u.GetFragment()); string linkName = name.ToUri().GetPath(); localFiles.AddItem(new Path(localWorkDir, linkName).ToUri().GetPath()); } if (!localFiles.IsEmpty()) { conf.Set(MRJobConfig.CacheLocalfiles, StringUtils.ArrayToString(Sharpen.Collections.ToArray (localFiles, new string[localFiles.Count]))); } } }
/// <summary>Get cache files set in the Configuration</summary> /// <returns>A URI array of the files set in the Configuration</returns> /// <exception cref="System.IO.IOException"/> public virtual URI[] GetCacheFiles() { return(DistributedCache.GetCacheFiles(conf)); }
/// <exception cref="System.Exception"/> public virtual void TestDeprecatedFunctions() { DistributedCache.AddLocalArchives(conf, "Test Local Archives 1"); NUnit.Framework.Assert.AreEqual("Test Local Archives 1", conf.Get(DistributedCache .CacheLocalarchives)); NUnit.Framework.Assert.AreEqual(1, DistributedCache.GetLocalCacheArchives(conf).Length ); NUnit.Framework.Assert.AreEqual("Test Local Archives 1", DistributedCache.GetLocalCacheArchives (conf)[0].GetName()); DistributedCache.AddLocalArchives(conf, "Test Local Archives 2"); NUnit.Framework.Assert.AreEqual("Test Local Archives 1,Test Local Archives 2", conf .Get(DistributedCache.CacheLocalarchives)); NUnit.Framework.Assert.AreEqual(2, DistributedCache.GetLocalCacheArchives(conf).Length ); NUnit.Framework.Assert.AreEqual("Test Local Archives 2", DistributedCache.GetLocalCacheArchives (conf)[1].GetName()); DistributedCache.SetLocalArchives(conf, "Test Local Archives 3"); NUnit.Framework.Assert.AreEqual("Test Local Archives 3", conf.Get(DistributedCache .CacheLocalarchives)); NUnit.Framework.Assert.AreEqual(1, DistributedCache.GetLocalCacheArchives(conf).Length ); NUnit.Framework.Assert.AreEqual("Test Local Archives 3", DistributedCache.GetLocalCacheArchives (conf)[0].GetName()); DistributedCache.AddLocalFiles(conf, "Test Local Files 1"); NUnit.Framework.Assert.AreEqual("Test Local Files 1", conf.Get(DistributedCache.CacheLocalfiles )); NUnit.Framework.Assert.AreEqual(1, DistributedCache.GetLocalCacheFiles(conf).Length ); NUnit.Framework.Assert.AreEqual("Test Local Files 1", DistributedCache.GetLocalCacheFiles (conf)[0].GetName()); DistributedCache.AddLocalFiles(conf, "Test Local Files 2"); NUnit.Framework.Assert.AreEqual("Test Local Files 1,Test Local Files 2", conf.Get (DistributedCache.CacheLocalfiles)); NUnit.Framework.Assert.AreEqual(2, DistributedCache.GetLocalCacheFiles(conf).Length ); NUnit.Framework.Assert.AreEqual("Test Local Files 2", DistributedCache.GetLocalCacheFiles (conf)[1].GetName()); DistributedCache.SetLocalFiles(conf, "Test Local Files 3"); NUnit.Framework.Assert.AreEqual("Test Local Files 3", conf.Get(DistributedCache.CacheLocalfiles )); NUnit.Framework.Assert.AreEqual(1, DistributedCache.GetLocalCacheFiles(conf).Length ); NUnit.Framework.Assert.AreEqual("Test Local Files 3", DistributedCache.GetLocalCacheFiles (conf)[0].GetName()); DistributedCache.SetArchiveTimestamps(conf, "1234567890"); NUnit.Framework.Assert.AreEqual(1234567890, conf.GetLong(DistributedCache.CacheArchivesTimestamps , 0)); NUnit.Framework.Assert.AreEqual(1, DistributedCache.GetArchiveTimestamps(conf).Length ); NUnit.Framework.Assert.AreEqual(1234567890, DistributedCache.GetArchiveTimestamps (conf)[0]); DistributedCache.SetFileTimestamps(conf, "1234567890"); NUnit.Framework.Assert.AreEqual(1234567890, conf.GetLong(DistributedCache.CacheFilesTimestamps , 0)); NUnit.Framework.Assert.AreEqual(1, DistributedCache.GetFileTimestamps(conf).Length ); NUnit.Framework.Assert.AreEqual(1234567890, DistributedCache.GetFileTimestamps(conf )[0]); DistributedCache.CreateAllSymlink(conf, new FilePath("Test Job Cache Dir"), new FilePath ("Test Work Dir")); NUnit.Framework.Assert.IsNull(conf.Get(DistributedCache.CacheSymlink)); NUnit.Framework.Assert.IsTrue(DistributedCache.GetSymlink(conf)); NUnit.Framework.Assert.IsTrue(symlinkFile.CreateNewFile()); FileStatus fileStatus = DistributedCache.GetFileStatus(conf, symlinkFile.ToURI()); NUnit.Framework.Assert.IsNotNull(fileStatus); NUnit.Framework.Assert.AreEqual(fileStatus.GetModificationTime(), DistributedCache .GetTimestamp(conf, symlinkFile.ToURI())); NUnit.Framework.Assert.IsTrue(symlinkFile.Delete()); DistributedCache.AddCacheArchive(symlinkFile.ToURI(), conf); NUnit.Framework.Assert.AreEqual(symlinkFile.ToURI().ToString(), conf.Get(DistributedCache .CacheArchives)); NUnit.Framework.Assert.AreEqual(1, DistributedCache.GetCacheArchives(conf).Length ); NUnit.Framework.Assert.AreEqual(symlinkFile.ToURI(), DistributedCache.GetCacheArchives (conf)[0]); DistributedCache.AddCacheFile(symlinkFile.ToURI(), conf); NUnit.Framework.Assert.AreEqual(symlinkFile.ToURI().ToString(), conf.Get(DistributedCache .CacheFiles)); NUnit.Framework.Assert.AreEqual(1, DistributedCache.GetCacheFiles(conf).Length); NUnit.Framework.Assert.AreEqual(symlinkFile.ToURI(), DistributedCache.GetCacheFiles (conf)[0]); }
/// <exception cref="System.IO.IOException"/> public static void SetupDistributedCache(Configuration conf, IDictionary <string, LocalResource> localResources) { // Cache archives ParseDistributedCacheArtifacts(conf, localResources, LocalResourceType.Archive, DistributedCache .GetCacheArchives(conf), DistributedCache.GetArchiveTimestamps(conf), GetFileSizes (conf, MRJobConfig.CacheArchivesSizes), DistributedCache.GetArchiveVisibilities( conf)); // Cache files ParseDistributedCacheArtifacts(conf, localResources, LocalResourceType.File, DistributedCache .GetCacheFiles(conf), DistributedCache.GetFileTimestamps(conf), GetFileSizes(conf , MRJobConfig.CacheFilesSizes), DistributedCache.GetFileVisibilities(conf)); }
/// <exception cref="System.IO.IOException"/> private static void SetupPipesJob(JobConf conf) { // default map output types to Text if (!GetIsJavaMapper(conf)) { conf.SetMapRunnerClass(typeof(PipesMapRunner)); // Save the user's partitioner and hook in our's. SetJavaPartitioner(conf, conf.GetPartitionerClass()); conf.SetPartitionerClass(typeof(PipesPartitioner)); } if (!GetIsJavaReducer(conf)) { conf.SetReducerClass(typeof(PipesReducer)); if (!GetIsJavaRecordWriter(conf)) { conf.SetOutputFormat(typeof(NullOutputFormat)); } } string textClassname = typeof(Text).FullName; SetIfUnset(conf, MRJobConfig.MapOutputKeyClass, textClassname); SetIfUnset(conf, MRJobConfig.MapOutputValueClass, textClassname); SetIfUnset(conf, MRJobConfig.OutputKeyClass, textClassname); SetIfUnset(conf, MRJobConfig.OutputValueClass, textClassname); // Use PipesNonJavaInputFormat if necessary to handle progress reporting // from C++ RecordReaders ... if (!GetIsJavaRecordReader(conf) && !GetIsJavaMapper(conf)) { conf.SetClass(Org.Apache.Hadoop.Mapred.Pipes.Submitter.InputFormat, conf.GetInputFormat ().GetType(), typeof(InputFormat)); conf.SetInputFormat(typeof(PipesNonJavaInputFormat)); } string exec = GetExecutable(conf); if (exec == null) { throw new ArgumentException("No application program defined."); } // add default debug script only when executable is expressed as // <path>#<executable> if (exec.Contains("#")) { // set default gdb commands for map and reduce task string defScript = "$HADOOP_PREFIX/src/c++/pipes/debug/pipes-default-script"; SetIfUnset(conf, MRJobConfig.MapDebugScript, defScript); SetIfUnset(conf, MRJobConfig.ReduceDebugScript, defScript); } URI[] fileCache = DistributedCache.GetCacheFiles(conf); if (fileCache == null) { fileCache = new URI[1]; } else { URI[] tmp = new URI[fileCache.Length + 1]; System.Array.Copy(fileCache, 0, tmp, 1, fileCache.Length); fileCache = tmp; } try { fileCache[0] = new URI(exec); } catch (URISyntaxException e) { IOException ie = new IOException("Problem parsing execable URI " + exec); Sharpen.Extensions.InitCause(ie, e); throw ie; } DistributedCache.SetCacheFiles(fileCache, conf); }