示例#1
0
 /// <summary>Write key value to an output file name.</summary>
 /// <remarks>
 /// Write key value to an output file name.
 /// Gets the record writer from job's output format.
 /// Job's output format should be a FileOutputFormat.
 /// </remarks>
 /// <param name="key">the key</param>
 /// <param name="value">the value</param>
 /// <param name="baseOutputPath">
 /// base-output path to write the record to.
 /// Note: Framework will generate unique filename for the baseOutputPath
 /// </param>
 /// <exception cref="System.IO.IOException"/>
 /// <exception cref="System.Exception"/>
 public virtual void Write(KEYOUT key, VALUEOUT value, string baseOutputPath)
 {
     CheckBaseOutputPath(baseOutputPath);
     if (jobOutputFormatContext == null)
     {
         jobOutputFormatContext = new TaskAttemptContextImpl(context.GetConfiguration(), context
                                                             .GetTaskAttemptID(), new MultipleOutputs.WrappedStatusReporter(context));
     }
     GetRecordWriter(jobOutputFormatContext, baseOutputPath).Write(key, value);
 }
            /// <exception cref="System.IO.IOException"/>
            public virtual void Setup <_T0>(TaskInputOutputContext <_T0> context)
            {
                Configuration conf = context.GetConfiguration();

                Path[]     localFiles    = context.GetLocalCacheFiles();
                URI[]      files         = context.GetCacheFiles();
                Path[]     localArchives = context.GetLocalCacheArchives();
                URI[]      archives      = context.GetCacheArchives();
                FileSystem fs            = LocalFileSystem.Get(conf);

                // Check that 2 files and 2 archives are present
                NUnit.Framework.Assert.AreEqual(2, localFiles.Length);
                NUnit.Framework.Assert.AreEqual(2, localArchives.Length);
                NUnit.Framework.Assert.AreEqual(2, files.Length);
                NUnit.Framework.Assert.AreEqual(2, archives.Length);
                // Check the file name
                NUnit.Framework.Assert.IsTrue(files[0].GetPath().EndsWith("distributed.first"));
                NUnit.Framework.Assert.IsTrue(files[1].GetPath().EndsWith("distributed.second.jar"
                                                                          ));
                // Check lengths of the files
                NUnit.Framework.Assert.AreEqual(1, fs.GetFileStatus(localFiles[0]).GetLen());
                NUnit.Framework.Assert.IsTrue(fs.GetFileStatus(localFiles[1]).GetLen() > 1);
                // Check extraction of the archive
                NUnit.Framework.Assert.IsTrue(fs.Exists(new Path(localArchives[0], "distributed.jar.inside3"
                                                                 )));
                NUnit.Framework.Assert.IsTrue(fs.Exists(new Path(localArchives[1], "distributed.jar.inside4"
                                                                 )));
                // Check the class loaders
                Log.Info("Java Classpath: " + Runtime.GetProperty("java.class.path"));
                ClassLoader cl = Sharpen.Thread.CurrentThread().GetContextClassLoader();

                // Both the file and the archive were added to classpath, so both
                // should be reachable via the class loader.
                NUnit.Framework.Assert.IsNotNull(cl.GetResource("distributed.jar.inside2"));
                NUnit.Framework.Assert.IsNotNull(cl.GetResource("distributed.jar.inside3"));
                NUnit.Framework.Assert.IsNull(cl.GetResource("distributed.jar.inside4"));
                // Check that the symlink for the renaming was created in the cwd;
                NUnit.Framework.Assert.IsTrue("symlink distributed.first.symlink doesn't exist",
                                              symlinkFile.Exists());
                NUnit.Framework.Assert.AreEqual("symlink distributed.first.symlink length not 1",
                                                1, symlinkFile.Length());
                //This last one is a difference between MRv2 and MRv1
                NUnit.Framework.Assert.IsTrue("second file should be symlinked too", expectedAbsentSymlinkFile
                                              .Exists());
            }