/// <summary> /// Combine InputSplits from child InputFormats into a /// <see cref="CompositeInputSplit"/> /// . /// </summary> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public override IList GetSplits(JobContext job) { IList <IList <InputSplit> > splits = new AList <IList <InputSplit> >(kids.Count); for (int i = 0; i < kids.Count; ++i) { IList <InputSplit> tmp = kids[i].GetSplits(job); if (null == tmp) { throw new IOException("Error gathering splits from child RReader"); } if (i > 0 && splits[i - 1].Count != tmp.Count) { throw new IOException("Inconsistent split cardinality from child " + i + " (" + splits [i - 1].Count + "/" + tmp.Count + ")"); } splits.Add(i, tmp); } int size = splits[0].Count; IList <InputSplit> ret = new AList <InputSplit>(); for (int i_1 = 0; i_1 < size; ++i_1) { CompositeInputSplit split = new CompositeInputSplit(splits.Count); for (int j = 0; j < splits.Count; ++j) { split.Add(splits[j][i_1]); } ret.AddItem(split); } return(ret); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public override RecordReader CreateRecordReader(InputSplit split, TaskAttemptContext taskContext) { // child types unknowable if (!(split is CompositeInputSplit)) { throw new IOException("Invalid split type:" + split.GetType().FullName); } CompositeInputSplit spl = (CompositeInputSplit)split; int capacity = kids.Count; CompositeRecordReader ret = null; try { if (!rrCstrMap.Contains(ident)) { throw new IOException("No RecordReader for " + ident); } ret = (CompositeRecordReader)rrCstrMap[ident].NewInstance(id, taskContext.GetConfiguration (), capacity, cmpcl); } catch (MemberAccessException e) { throw new IOException(e); } catch (InstantiationException e) { throw new IOException(e); } catch (TargetInvocationException e) { throw new IOException(e); } for (int i = 0; i < capacity; ++i) { ret.Add(((ComposableRecordReader)kids[i].CreateRecordReader(spl.Get(i), taskContext ))); } return((ComposableRecordReader)ret); }