Пример #1
0
            /// <summary>
            /// Combine InputSplits from child InputFormats into a
            /// <see cref="CompositeInputSplit"/>
            /// .
            /// </summary>
            /// <exception cref="System.IO.IOException"/>
            /// <exception cref="System.Exception"/>
            public override IList GetSplits(JobContext job)
            {
                IList <IList <InputSplit> > splits = new AList <IList <InputSplit> >(kids.Count);

                for (int i = 0; i < kids.Count; ++i)
                {
                    IList <InputSplit> tmp = kids[i].GetSplits(job);
                    if (null == tmp)
                    {
                        throw new IOException("Error gathering splits from child RReader");
                    }
                    if (i > 0 && splits[i - 1].Count != tmp.Count)
                    {
                        throw new IOException("Inconsistent split cardinality from child " + i + " (" + splits
                                              [i - 1].Count + "/" + tmp.Count + ")");
                    }
                    splits.Add(i, tmp);
                }
                int size = splits[0].Count;
                IList <InputSplit> ret = new AList <InputSplit>();

                for (int i_1 = 0; i_1 < size; ++i_1)
                {
                    CompositeInputSplit split = new CompositeInputSplit(splits.Count);
                    for (int j = 0; j < splits.Count; ++j)
                    {
                        split.Add(splits[j][i_1]);
                    }
                    ret.AddItem(split);
                }
                return(ret);
            }
Пример #2
0
            /// <exception cref="System.IO.IOException"/>
            /// <exception cref="System.Exception"/>
            public override RecordReader CreateRecordReader(InputSplit split, TaskAttemptContext
                                                            taskContext)
            {
                // child types unknowable
                if (!(split is CompositeInputSplit))
                {
                    throw new IOException("Invalid split type:" + split.GetType().FullName);
                }
                CompositeInputSplit spl   = (CompositeInputSplit)split;
                int capacity              = kids.Count;
                CompositeRecordReader ret = null;

                try
                {
                    if (!rrCstrMap.Contains(ident))
                    {
                        throw new IOException("No RecordReader for " + ident);
                    }
                    ret = (CompositeRecordReader)rrCstrMap[ident].NewInstance(id, taskContext.GetConfiguration
                                                                                  (), capacity, cmpcl);
                }
                catch (MemberAccessException e)
                {
                    throw new IOException(e);
                }
                catch (InstantiationException e)
                {
                    throw new IOException(e);
                }
                catch (TargetInvocationException e)
                {
                    throw new IOException(e);
                }
                for (int i = 0; i < capacity; ++i)
                {
                    ret.Add(((ComposableRecordReader)kids[i].CreateRecordReader(spl.Get(i), taskContext
                                                                                )));
                }
                return((ComposableRecordReader)ret);
            }