コード例 #1
0
 /// <exception cref="System.IO.IOException"/>
 public virtual void Map(WritableComparable key, Writable val, OutputCollector <WritableComparable
                                                                                , Writable> output, Reporter reporter)
 {
     NUnit.Framework.Assert.IsNotNull("Mapper not configured!", loader);
     // load the memory
     loader.Load();
     // work as identity mapper
     output.Collect(key, val);
 }
コード例 #2
0
            /// <exception cref="System.IO.IOException"/>
            public virtual void Map(WritableComparable key, Writable value, OutputCollector <BytesWritable
                                                                                             , BytesWritable> output, Reporter reporter)
            {
                int itemCount = 0;

                while (numBytesToWrite > 0)
                {
                    int keyLength = minKeySize + (keySizeRange != 0 ? random.Next(keySizeRange) : 0);
                    randomKey.SetSize(keyLength);
                    RandomizeBytes(randomKey.GetBytes(), 0, randomKey.GetLength());
                    int valueLength = minValueSize + (valueSizeRange != 0 ? random.Next(valueSizeRange
                                                                                        ) : 0);
                    randomValue.SetSize(valueLength);
                    RandomizeBytes(randomValue.GetBytes(), 0, randomValue.GetLength());
                    output.Collect(randomKey, randomValue);
                    numBytesToWrite -= keyLength + valueLength;
                    reporter.IncrCounter(ThreadedMapBenchmark.Counters.BytesWritten, 1);
                    reporter.IncrCounter(ThreadedMapBenchmark.Counters.RecordsWritten, 1);
                    if (++itemCount % 200 == 0)
                    {
                        reporter.SetStatus("wrote record " + itemCount + ". " + numBytesToWrite + " bytes left."
                                           );
                    }
                }
                reporter.SetStatus("done with " + itemCount + " records.");
            }
コード例 #3
0
ファイル: SortValidator.cs プロジェクト: orf53975/hadoop.net
                /// <exception cref="System.IO.IOException"/>
                public virtual void Map(BytesWritable key, BytesWritable value, OutputCollector <BytesWritable
                                                                                                 , IntWritable> output, Reporter reporter)
                {
                    // newKey = (key, value)
                    BytesWritable keyValue = new BytesWritable(Pair(key, value));

                    // output (newKey, value)
                    output.Collect(keyValue, this.value);
                }
コード例 #4
0
 /// <exception cref="System.IO.IOException"/>
 public override RecordReader <K, V> GetRecordReader(InputSplit split, JobConf job,
                                                     Reporter reporter)
 {
     return(null);
 }
コード例 #5
0
ファイル: SortValidator.cs プロジェクト: orf53975/hadoop.net
                /// <exception cref="System.IO.IOException"/>
                public virtual void Map(WritableComparable key, Writable value, OutputCollector <IntWritable
                                                                                                 , SortValidator.RecordStatsChecker.RecordStatsWritable> output, Reporter reporter
                                        )
                {
                    // Set up rawKey and rawValue on the first call to 'map'
                    if (recordId == -1)
                    {
                        rawKey   = CreateRaw(key.GetType());
                        rawValue = CreateRaw(value.GetType());
                    }
                    ++recordId;
                    if (this.key == sortOutput)
                    {
                        // Check if keys are 'sorted' if this
                        // record is from sort's output
                        if (prevKey == null)
                        {
                            prevKey  = key;
                            keyClass = prevKey.GetType();
                        }
                        else
                        {
                            // Sanity check
                            if (keyClass != key.GetType())
                            {
                                throw new IOException("Type mismatch in key: expected " + keyClass.FullName + ", received "
                                                      + key.GetType().FullName);
                            }
                            // Check if they were sorted correctly
                            if (prevKey.CompareTo(key) > 0)
                            {
                                throw new IOException("The 'map-reduce' framework wrongly" + " classifed (" + prevKey
                                                      + ") > (" + key + ") " + "for record# " + recordId);
                            }
                            prevKey = key;
                        }
                        // Check if the sorted output is 'partitioned' right
                        int keyPartition = partitioner.GetPartition(key, value, noSortReducers);
                        if (partition != keyPartition)
                        {
                            throw new IOException("Partitions do not match for record# " + recordId + " ! - '"
                                                  + partition + "' v/s '" + keyPartition + "'");
                        }
                    }
                    // Construct the record-stats and output (this.key, record-stats)
                    byte[] keyBytes    = rawKey.GetRawBytes(key);
                    int    keyBytesLen = rawKey.GetRawBytesLength(key);

                    byte[] valueBytes       = rawValue.GetRawBytes(value);
                    int    valueBytesLen    = rawValue.GetRawBytesLength(value);
                    int    keyValueChecksum = (WritableComparator.HashBytes(keyBytes, keyBytesLen) ^ WritableComparator
                                               .HashBytes(valueBytes, valueBytesLen));

                    output.Collect(this.key, new SortValidator.RecordStatsChecker.RecordStatsWritable
                                       ((keyBytesLen + valueBytesLen), 1, keyValueChecksum));
                }
コード例 #6
0
ファイル: SortValidator.cs プロジェクト: orf53975/hadoop.net
                /// <exception cref="System.IO.IOException"/>
                public virtual void Reduce(IntWritable key, IEnumerator <SortValidator.RecordStatsChecker.RecordStatsWritable
                                                                         > values, OutputCollector <IntWritable, SortValidator.RecordStatsChecker.RecordStatsWritable
                                                                                                    > output, Reporter reporter)
                {
                    long bytes   = 0;
                    long records = 0;
                    int  xor     = 0;

                    while (values.HasNext())
                    {
                        SortValidator.RecordStatsChecker.RecordStatsWritable stats = values.Next();
                        bytes   += stats.GetBytes();
                        records += stats.GetRecords();
                        xor     ^= stats.GetChecksum();
                    }
                    output.Collect(key, new SortValidator.RecordStatsChecker.RecordStatsWritable(bytes
                                                                                                 , records, xor));
                }
コード例 #7
0
 /// <exception cref="System.IO.IOException"/>
 public virtual void Reduce(Text key, IEnumerator <Text> values, OutputCollector <Text
                                                                                  , Text> @out, Reporter reporter)
 {
     @out.Collect(new Text("Test"), new Text("Me"));
 }
コード例 #8
0
ファイル: MRBench.cs プロジェクト: orf53975/hadoop.net
			/// <exception cref="System.IO.IOException"/>
			public virtual void Map(WritableComparable key, Text value, OutputCollector<UTF8, 
				UTF8> output, Reporter reporter)
			{
				string line = value.ToString();
				output.Collect(new UTF8(Process(line)), new UTF8(string.Empty));
			}
コード例 #9
0
        /// <exception cref="System.IO.IOException"/>
        public virtual void TestFailAbort()
        {
            JobConf job = new JobConf();

            job.Set(FileSystem.FsDefaultNameKey, "faildel:///");
            job.SetClass("fs.faildel.impl", typeof(TestMRCJCFileOutputCommitter.FakeFileSystem
                                                   ), typeof(FileSystem));
            SetConfForFileOutputCommitter(job);
            JobContext          jContext  = new JobContextImpl(job, ((JobID)taskID.GetJobID()));
            TaskAttemptContext  tContext  = new TaskAttemptContextImpl(job, taskID);
            FileOutputCommitter committer = new FileOutputCommitter();

            FileOutputFormat.SetWorkOutputPath(job, committer.GetTaskAttemptPath(tContext));
            // do setup
            committer.SetupJob(jContext);
            committer.SetupTask(tContext);
            string   file      = "test.txt";
            FilePath jobTmpDir = new FilePath(committer.GetJobAttemptPath(jContext).ToUri().GetPath
                                                  ());
            FilePath taskTmpDir = new FilePath(committer.GetTaskAttemptPath(tContext).ToUri()
                                               .GetPath());
            FilePath expectedFile = new FilePath(taskTmpDir, file);
            // A reporter that does nothing
            Reporter reporter = Reporter.Null;
            // write output
            FileSystem       localFs         = new TestMRCJCFileOutputCommitter.FakeFileSystem();
            TextOutputFormat theOutputFormat = new TextOutputFormat();
            RecordWriter     theRecordWriter = theOutputFormat.GetRecordWriter(localFs, job, expectedFile
                                                                               .GetAbsolutePath(), reporter);

            WriteOutput(theRecordWriter, reporter);
            // do abort
            Exception th = null;

            try
            {
                committer.AbortTask(tContext);
            }
            catch (IOException ie)
            {
                th = ie;
            }
            NUnit.Framework.Assert.IsNotNull(th);
            NUnit.Framework.Assert.IsTrue(th is IOException);
            NUnit.Framework.Assert.IsTrue(th.Message.Contains("fake delete failed"));
            NUnit.Framework.Assert.IsTrue(expectedFile + " does not exists", expectedFile.Exists
                                              ());
            th = null;
            try
            {
                committer.AbortJob(jContext, JobStatus.State.Failed);
            }
            catch (IOException ie)
            {
                th = ie;
            }
            NUnit.Framework.Assert.IsNotNull(th);
            NUnit.Framework.Assert.IsTrue(th is IOException);
            NUnit.Framework.Assert.IsTrue(th.Message.Contains("fake delete failed"));
            NUnit.Framework.Assert.IsTrue("job temp dir does not exists", jobTmpDir.Exists());
        }
コード例 #10
0
 /// <exception cref="System.IO.IOException"/>
 public virtual void Map(WritableComparable key, Writable val, OutputCollector <Text
                                                                                , Text> @out, Reporter reporter)
 {
     @out.Collect(new Text("Hello"), new Text("World"));
 }
コード例 #11
0
 /// <exception cref="System.IO.IOException"/>
 public abstract RecordReader <K, V> GetRecordReader(InputSplit split, JobConf job,
                                                     Reporter reporter);
コード例 #12
0
ファイル: UtilsForTests.cs プロジェクト: orf53975/hadoop.net
 // Mapper that sleeps for a long time.
 // Used for running a job that will be killed
 /// <exception cref="System.IO.IOException"/>
 public virtual void Map(WritableComparable key, Writable value, OutputCollector <WritableComparable
                                                                                  , Writable> @out, Reporter reporter)
 {
     try
     {
         Sharpen.Thread.Sleep(1000000);
     }
     catch (Exception)
     {
     }
 }
コード例 #13
0
ファイル: UtilsForTests.cs プロジェクト: orf53975/hadoop.net
 // Mapper that fails
 /// <exception cref="System.IO.IOException"/>
 public virtual void Map(WritableComparable key, Writable value, OutputCollector <WritableComparable
                                                                                  , Writable> @out, Reporter reporter)
 {
     //NOTE- the next line is required for the TestDebugScript test to succeed
     System.Console.Error.WriteLine("failing map");
     throw new RuntimeException("failing map");
 }
コード例 #14
0
ファイル: UtilsForTests.cs プロジェクト: orf53975/hadoop.net
 GetRecordReader(InputSplit split, JobConf job, Reporter reporter)
 {
     return(new UtilsForTests.RandomInputFormat.RandomRecordReader(((FileSplit)split).
                                                                   GetPath()));
 }
コード例 #15
0
ファイル: UtilsForTests.cs プロジェクト: orf53975/hadoop.net
 /// <summary>The waiting function.</summary>
 /// <remarks>
 /// The waiting function.  The map exits once it gets a signal. Here the
 /// signal is the file existence.
 /// </remarks>
 /// <exception cref="System.IO.IOException"/>
 public virtual void Map(WritableComparable key, Writable val, OutputCollector <WritableComparable
                                                                                , Writable> output, Reporter reporter)
 {
     if (ShouldWait(id))
     {
         if (fs != null)
         {
             while (!fs.Exists(GetSignalFile(id)))
             {
                 try
                 {
                     reporter.Progress();
                     lock (this)
                     {
                         Sharpen.Runtime.Wait(this, 1000);
                     }
                 }
                 catch (Exception)
                 {
                     // wait for 1 sec
                     System.Console.Out.WriteLine("Interrupted while the map was waiting for " + " the signal."
                                                  );
                     break;
                 }
             }
         }
         else
         {
             throw new IOException("Could not get the DFS!!");
         }
     }
 }