Ejemplo n.º 1
0
            public override void InnerRun()
            {
                // Console.WriteLine("Reducer for " + FileID);

                // buffer input

                var tmp = new NonLockingMemoryKeyValueReaderWriter();
                using (var rdr = InputFolders.CreateStreamReader(FileID))
                {
                    tmp.Write(rdr);
                }

                // Console.WriteLine(tmp.Count + " items found in " + FileID);

                // reduce

                var wrt = new NonLockingMemoryKeyValueReaderWriter();
                var coll = new RecordWriterReducerCollector(wrt);
                foreach (var kk in tmp.GetKeys())
                {
                    // Console.WriteLine("Key "+kk+" has "+data[kk].Count+" values.");
                    try
                    {
                        reducer.Reduce(kk, tmp.GetValues(kk), coll, true);
                    }
                    catch (Exception z)
                    {
                        Console.WriteLine("Reducer crashed: " + z);
                    }
                }

                if (wrt.Count < 1)
                    return;

                // save output

                using (var output = OutputFolder.CreateWriter())
                {
                    output.Write(wrt);
                }
            }
Ejemplo n.º 2
0
                public override void InnerRun()
                {
                    var tmp = new NonLockingMemoryKeyValueReaderWriter();
                    try
                    {
                        mapper.Map(inputkey, value, new RecordWriterMapperCollector(tmp));
                    }
                    catch (Exception e)
                    {
                        Console.WriteLine("Mapper crashed: " + e);
                    }

                    if (tmp.Count == 0)
                        return;

                    if (reducer != null)
                    {
                        var tmp2 = new NonLockingMemoryKeyValueReaderWriter();

                        var rw = new RecordWriterReducerCollector(tmp2);
                        foreach (var k in tmp.GetKeys())
                        {
                            try
                            {
                                reducer.Reduce(k, tmp.GetValues(k), rw, false);
                            }
                            catch (Exception e)
                            {
                                Console.WriteLine("Reducer crashed: " + e);
                            }
                        }

                        using (var w = OutputFolder.CreateWriter())
                        {
                            w.Write(tmp2);
                        }
                    }
                    else
                    {
                        using (var w = OutputFolder.CreateWriter())
                        {
                            w.Write(tmp);
                        }
                    }
                }