예제 #1
0
            /// <exception cref="System.IO.IOException"/>
            /// <exception cref="System.Exception"/>
            protected override void Setup(Mapper.Context context)
            {
                Configuration conf = context.GetConfiguration();

                NUnit.Framework.Assert.AreEqual(prop, conf.Get("a"));
                WriteFlag(conf, "map.setup." + name);
            }
            /// <exception cref="System.IO.IOException"/>
            /// <exception cref="System.Exception"/>
            protected override void Map(object key, Text value, Mapper.Context context)
            {
                // Make one mapper slower for speculative execution
                TaskAttemptID taid               = context.GetTaskAttemptID();
                long          sleepTime          = 100;
                Configuration conf               = context.GetConfiguration();
                bool          test_speculate_map = conf.GetBoolean(MRJobConfig.MapSpeculative, false);

                // IF TESTING MAPPER SPECULATIVE EXECUTION:
                //   Make the "*_m_000000_0" attempt take much longer than the others.
                //   When speculative execution is enabled, this should cause the attempt
                //   to be killed and restarted. At that point, the attempt ID will be
                //   "*_m_000000_1", so sleepTime will still remain 100ms.
                if ((taid.GetTaskType() == TaskType.Map) && test_speculate_map && (taid.GetTaskID
                                                                                       ().GetId() == 0) && (taid.GetId() == 0))
                {
                    sleepTime = 10000;
                }
                try
                {
                    Sharpen.Thread.Sleep(sleepTime);
                }
                catch (Exception)
                {
                }
                // Ignore
                context.Write(value, new IntWritable(1));
            }
예제 #3
0
        protected internal override void Setup(Mapper.Context context)
        {
            Configuration conf = context.GetConfiguration();

            pattern = Sharpen.Pattern.Compile(conf.Get(Pattern));
            group   = conf.GetInt(Group, 0);
        }
예제 #4
0
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="System.Exception"/>
        protected internal override void Setup(Mapper.Context context)
        {
            // Find the Mapper from the TaggedInputSplit.
            TaggedInputSplit inputSplit = (TaggedInputSplit)context.GetInputSplit();

            mapper = (Mapper <K1, V1, K2, V2>)ReflectionUtils.NewInstance(inputSplit.GetMapperClass
                                                                              (), context.GetConfiguration());
        }
예제 #5
0
            /// <summary>Save the configuration value that we need to write the data.</summary>
            protected override void Setup(Mapper.Context context)
            {
                Configuration conf = context.GetConfiguration();

                numBytesToWrite   = conf.GetLong(BytesPerMap, 1 * 1024 * 1024 * 1024);
                minWordsInKey     = conf.GetInt(MinKey, 5);
                wordsInKeyRange   = (conf.GetInt(MaxKey, 10) - minWordsInKey);
                minWordsInValue   = conf.GetInt(MinValue, 10);
                wordsInValueRange = (conf.GetInt(MaxValue, 100) - minWordsInValue);
            }
예제 #6
0
            /// <summary>
            /// Save the values out of the configuaration that we need to write
            /// the data.
            /// </summary>
            protected override void Setup(Mapper.Context context)
            {
                Configuration conf = context.GetConfiguration();

                numBytesToWrite = conf.GetLong(BytesPerMap, 1 * 1024 * 1024 * 1024);
                minKeySize      = conf.GetInt(MinKey, 10);
                keySizeRange    = conf.GetInt(MaxKey, 1000) - minKeySize;
                minValueSize    = conf.GetInt(MinValue, 0);
                valueSizeRange  = conf.GetInt(MaxValue, 20000) - minValueSize;
            }
예제 #7
0
            /// <exception cref="System.IO.IOException"/>
            /// <exception cref="System.Exception"/>
            protected override void Setup(Mapper.Context context)
            {
                base.Setup(context);
                Configuration conf     = context.GetConfiguration();
                string        ioSortMb = conf.Get(MRJobConfig.IoSortMb);

                if (!TestIoSortMb.Equals(ioSortMb))
                {
                    throw new IOException("io.sort.mb expected: " + TestIoSortMb + ", actual: " + ioSortMb
                                          );
                }
            }
예제 #8
0
            protected override void Setup(Mapper.Context context)
            {
                this.context = context;
                Configuration conf = context.GetConfiguration();

                depth  = conf.GetInt(Pentomino.Depth, PentDepth);
                width  = conf.GetInt(Pentomino.Width, PentWidth);
                height = conf.GetInt(Pentomino.Height, PentHeight);
                pent   = (Pentomino)ReflectionUtils.NewInstance(conf.GetClass(Pentomino.Class, typeof(
                                                                                  OneSidedPentomino)), conf);
                pent.Initialize(width, height);
                pent.SetPrinter(new DistributedPentomino.PentMap.SolutionCatcher(this));
            }
예제 #9
0
                /// <summary>Partitions sigma into parts</summary>
                /// <exception cref="System.IO.IOException"/>
                /// <exception cref="System.Exception"/>
                protected override void Map(NullWritable nw, SummationWritable sigma, Mapper.Context
                                            context)
                {
                    Configuration conf   = context.GetConfiguration();
                    int           nParts = conf.GetInt(NParts, 0);

                    Summation[] parts = sigma.GetElement().Partition(nParts);
                    for (int i = 0; i < parts.Length; ++i)
                    {
                        context.Write(new IntWritable(i), new SummationWritable(parts[i]));
                        Log.Info("parts[" + i + "] = " + parts[i]);
                    }
                }
예제 #10
0
            /// <exception cref="System.IO.IOException"/>
            protected override void Map(LongWritable key, Text value, Mapper.Context context)
            {
                StringBuilder sb = new StringBuilder(512);

                for (int i = 0; i < 1000; i++)
                {
                    sb.Append("a");
                }
                context.SetStatus(sb.ToString());
                int progressStatusLength = context.GetConfiguration().GetInt(MRConfig.ProgressStatusLenLimitKey
                                                                             , MRConfig.ProgressStatusLenLimitDefault);

                if (context.GetStatus().Length > progressStatusLength)
                {
                    throw new IOException("Status is not truncated");
                }
            }
예제 #11
0
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="System.Exception"/>
        protected internal override void Setup(Mapper.Context context)
        {
            Configuration conf = context.GetConfiguration();

            this.fieldSeparator        = conf.Get(FieldSelectionHelper.DataFieldSeperator, "\t");
            this.mapOutputKeyValueSpec = conf.Get(FieldSelectionHelper.MapOutputKeyValueSpec,
                                                  "0-:");
            try
            {
                this.ignoreInputKey = typeof(TextInputFormat).GetCanonicalName().Equals(context.GetInputFormatClass
                                                                                            ().GetCanonicalName());
            }
            catch (TypeLoadException e)
            {
                throw new IOException("Input format class not found", e);
            }
            allMapValueFieldsFrom = FieldSelectionHelper.ParseOutputKeyValueSpec(mapOutputKeyValueSpec
                                                                                 , mapOutputKeyFieldList, mapOutputValueFieldList);
            Log.Info(FieldSelectionHelper.SpecToString(fieldSeparator, mapOutputKeyValueSpec,
                                                       allMapValueFieldsFrom, mapOutputKeyFieldList, mapOutputValueFieldList) + "\nignoreInputKey:"
                     + ignoreInputKey);
        }
예제 #12
0
 protected override void Setup(Mapper.Context context)
 {
     ioEx = context.GetConfiguration().GetBoolean("multithreaded.ioException", false);
     rtEx = context.GetConfiguration().GetBoolean("multithreaded.runtimeException", false
                                                  );
 }
예제 #13
0
            /// <exception cref="System.IO.IOException"/>
            /// <exception cref="System.Exception"/>
            internal MapRunner(MultithreadedMapper <K1, V1, K2, V2> _enclosing, Mapper.Context
                               context)
            {
                this._enclosing = _enclosing;
                reader          = new MultithreadedMapper.SubMapRecordReader(this);
                this.mapper     = ReflectionUtils.NewInstance(this._enclosing.mapClass, context.GetConfiguration
                                                                  ());
                MapContext <K1, V1, K2, V2> mapContext = new MapContextImpl <K1, V1, K2, V2>(this._enclosing
                                                                                             .outer.GetConfiguration(), this._enclosing.outer.GetTaskAttemptID(), this.reader
                                                                                             , new MultithreadedMapper.SubMapRecordWriter(this), context.GetOutputCommitter()
                                                                                             , new MultithreadedMapper.SubMapStatusReporter(this), this._enclosing.outer.GetInputSplit
                                                                                                 ());

                this.subcontext = new WrappedMapper <K1, V1, K2, V2>().GetMapContext(mapContext);
                this.reader.Initialize(context.GetInputSplit(), context);
            }
예제 #14
0
 protected internal override void Setup(Mapper.Context context)
 {
     chain = new Org.Apache.Hadoop.Mapreduce.Lib.Chain.Chain(true);
     chain.Setup(context.GetConfiguration());
 }
예제 #15
0
            // my sleep class
            /// <summary>attempts to access tokenCache as from client</summary>
            /// <exception cref="System.IO.IOException"/>
            /// <exception cref="System.Exception"/>
            protected override void Map(IntWritable key, IntWritable value, Mapper.Context context
                                        )
            {
                // get context token storage:
                Credentials contextCredentials = context.GetCredentials();
                ICollection <Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> > contextTokenCollection
                    = contextCredentials.GetAllTokens();

                foreach (Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> t in contextTokenCollection)
                {
                    System.Console.Out.WriteLine("Context token: [" + t + "]");
                }
                if (contextTokenCollection.Count != 2)
                {
                    // one job token and one delegation token
                    // fail the test:
                    throw new RuntimeException("Exactly 2 tokens are expected in the contextTokenCollection: "
                                               + "one job token and one delegation token, but was found " + contextTokenCollection
                                               .Count + " tokens.");
                }
                Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> dt = contextCredentials.GetToken
                                                                                  (new Text(DelegationTokenKey));
                if (dt == null)
                {
                    throw new RuntimeException("Token for key [" + DelegationTokenKey + "] not found in the job context."
                                               );
                }
                string tokenFile0 = context.GetConfiguration().Get(MRJobConfig.MapreduceJobCredentialsBinary
                                                                   );

                if (tokenFile0 != null)
                {
                    throw new RuntimeException("Token file key [" + MRJobConfig.MapreduceJobCredentialsBinary
                                               + "] found in the configuration. It should have been removed from the configuration."
                                               );
                }
                string tokenFile = context.GetConfiguration().Get(KeySecurityTokenFileName);

                if (tokenFile == null)
                {
                    throw new RuntimeException("Token file key [" + KeySecurityTokenFileName + "] not found in the job configuration."
                                               );
                }
                Credentials binaryCredentials = new Credentials();

                binaryCredentials.ReadTokenStorageStream(new DataInputStream(new FileInputStream(
                                                                                 tokenFile)));
                ICollection <Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> > binaryTokenCollection
                    = binaryCredentials.GetAllTokens();

                if (binaryTokenCollection.Count != 1)
                {
                    throw new RuntimeException("The token collection read from file [" + tokenFile +
                                               "] must have size = 1.");
                }
                Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> binTok = binaryTokenCollection
                                                                                  .GetEnumerator().Next();
                System.Console.Out.WriteLine("The token read from binary file: t = [" + binTok +
                                             "]");
                // Verify that dt is same as the token in the file:
                if (!dt.Equals(binTok))
                {
                    throw new RuntimeException("Delegation token in job is not same as the token passed in file:"
                                               + " tokenInFile=[" + binTok + "], dt=[" + dt + "].");
                }
                // Now test the user tokens.
                UserGroupInformation ugi = UserGroupInformation.GetCurrentUser();
                // Print all the UGI tokens for diagnostic purposes:
                ICollection <Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> > ugiTokenCollection
                    = ugi.GetTokens();

                foreach (Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> t_1 in ugiTokenCollection)
                {
                    System.Console.Out.WriteLine("UGI token: [" + t_1 + "]");
                }
                Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> ugiToken = ugi.GetCredentials
                                                                                        ().GetToken(new Text(DelegationTokenKey));
                if (ugiToken == null)
                {
                    throw new RuntimeException("Token for key [" + DelegationTokenKey + "] not found among the UGI tokens."
                                               );
                }
                if (!ugiToken.Equals(binTok))
                {
                    throw new RuntimeException("UGI token is not same as the token passed in binary file:"
                                               + " tokenInBinFile=[" + binTok + "], ugiTok=[" + ugiToken + "].");
                }
                base.Map(key, value, context);
            }
예제 #16
0
 protected override void Setup(Mapper.Context context)
 {
     srcs = context.GetConfiguration().GetInt("testdatamerge.sources", 0);
     NUnit.Framework.Assert.IsTrue("Invalid src count: " + srcs, srcs > 0);
 }
예제 #17
0
 /// <exception cref="System.IO.IOException"/>
 /// <exception cref="System.Exception"/>
 protected override void Map(LongWritable key, Text value, Mapper.Context context)
 {
     WriteFlag(context.GetConfiguration(), "map." + name + ".value." + value);
     context.Write(key, new Text(value + name));
 }
예제 #18
0
 /// <exception cref="System.IO.IOException"/>
 /// <exception cref="System.Exception"/>
 protected internal override void Setup(Mapper.Context context)
 {
     ValueAggregatorJobBase.Setup(context.GetConfiguration());
 }
예제 #19
0
            /// <exception cref="System.IO.IOException"/>
            protected override void Setup(Mapper.Context context)
            {
                Configuration conf = context.GetConfiguration();

                Path[] localFiles    = context.GetLocalCacheFiles();
                URI[]  files         = context.GetCacheFiles();
                Path[] localArchives = context.GetLocalCacheArchives();
                URI[]  archives      = context.GetCacheArchives();
                // Check that 4 (2 + appjar + DistrubutedCacheChecker jar) files
                // and 2 archives are present
                NUnit.Framework.Assert.AreEqual(4, localFiles.Length);
                NUnit.Framework.Assert.AreEqual(4, files.Length);
                NUnit.Framework.Assert.AreEqual(2, localArchives.Length);
                NUnit.Framework.Assert.AreEqual(2, archives.Length);
                // Check lengths of the files
                IDictionary <string, Path> filesMap = PathsToMap(localFiles);

                NUnit.Framework.Assert.IsTrue(filesMap.Contains("distributed.first.symlink"));
                NUnit.Framework.Assert.AreEqual(1, localFs.GetFileStatus(filesMap["distributed.first.symlink"
                                                                         ]).GetLen());
                NUnit.Framework.Assert.IsTrue(filesMap.Contains("distributed.second.jar"));
                NUnit.Framework.Assert.IsTrue(localFs.GetFileStatus(filesMap["distributed.second.jar"
                                                                    ]).GetLen() > 1);
                // Check extraction of the archive
                IDictionary <string, Path> archivesMap = PathsToMap(localArchives);

                NUnit.Framework.Assert.IsTrue(archivesMap.Contains("distributed.third.jar"));
                NUnit.Framework.Assert.IsTrue(localFs.Exists(new Path(archivesMap["distributed.third.jar"
                                                                      ], "distributed.jar.inside3")));
                NUnit.Framework.Assert.IsTrue(archivesMap.Contains("distributed.fourth.jar"));
                NUnit.Framework.Assert.IsTrue(localFs.Exists(new Path(archivesMap["distributed.fourth.jar"
                                                                      ], "distributed.jar.inside4")));
                // Check the class loaders
                Log.Info("Java Classpath: " + Runtime.GetProperty("java.class.path"));
                ClassLoader cl = Sharpen.Thread.CurrentThread().GetContextClassLoader();

                // Both the file and the archive should have been added to classpath, so
                // both should be reachable via the class loader.
                NUnit.Framework.Assert.IsNotNull(cl.GetResource("distributed.jar.inside2"));
                NUnit.Framework.Assert.IsNotNull(cl.GetResource("distributed.jar.inside3"));
                NUnit.Framework.Assert.IsNotNull(cl.GetResource("distributed.jar.inside4"));
                // The Job Jar should have been extracted to a folder named "job.jar" and
                // added to the classpath; the two jar files in the lib folder in the Job
                // Jar should have also been added to the classpath
                NUnit.Framework.Assert.IsNotNull(cl.GetResource("job.jar/"));
                NUnit.Framework.Assert.IsNotNull(cl.GetResource("job.jar/lib/lib1.jar"));
                NUnit.Framework.Assert.IsNotNull(cl.GetResource("job.jar/lib/lib2.jar"));
                // Check that the symlink for the renaming was created in the cwd;
                FilePath symlinkFile = new FilePath("distributed.first.symlink");

                NUnit.Framework.Assert.IsTrue(symlinkFile.Exists());
                NUnit.Framework.Assert.AreEqual(1, symlinkFile.Length());
                // Check that the symlink for the Job Jar was created in the cwd and
                // points to the extracted directory
                FilePath jobJarDir = new FilePath("job.jar");

                if (Shell.Windows)
                {
                    NUnit.Framework.Assert.IsTrue(IsWindowsSymlinkedDirectory(jobJarDir));
                }
                else
                {
                    NUnit.Framework.Assert.IsTrue(FileUtils.IsSymlink(jobJarDir));
                    NUnit.Framework.Assert.IsTrue(jobJarDir.IsDirectory());
                }
            }
예제 #20
0
 /// <exception cref="System.IO.IOException"/>
 /// <exception cref="System.Exception"/>
 protected override void Cleanup(Mapper.Context context)
 {
     WriteFlag(context.GetConfiguration(), "map.cleanup." + name);
 }