/// <summary>
/// Expand globs in the given <code>filePattern</code> into a collection of
/// file patterns so that in the expanded set no file pattern has a
/// slash character ("/") in a curly bracket pair.
/// </summary>
/// <param name="filePattern"/>
/// <returns>expanded file patterns</returns>
/// <exception cref="System.IO.IOException"></exception>
public static IList <string> Expand(string filePattern)
{
IList <string> fullyExpanded = new AList <string>();
IList <GlobExpander.StringWithOffset> toExpand = new AList <GlobExpander.StringWithOffset
>();
toExpand.AddItem(new GlobExpander.StringWithOffset(filePattern, 0));
while (!toExpand.IsEmpty())
{
GlobExpander.StringWithOffset path = toExpand.Remove(0);
IList <GlobExpander.StringWithOffset> expanded = ExpandLeftmost(path);
if (expanded == null)
{
fullyExpanded.AddItem(path.@string);
}
else
{
toExpand.AddRange(0, expanded);
}
}
return(fullyExpanded);
}
/// <exception cref="System.IO.IOException"/>
private RawKeyValueIterator FinalMerge(JobConf job, FileSystem fs, IList <InMemoryMapOutput
<K, V> > inMemoryMapOutputs, IList <MergeManagerImpl.CompressAwarePath> onDiskMapOutputs
)
{
Log.Info("finalMerge called with " + inMemoryMapOutputs.Count + " in-memory map-outputs and "
+ onDiskMapOutputs.Count + " on-disk map-outputs");
long maxInMemReduce = GetMaxInMemReduceLimit();
// merge config params
Type keyClass = (Type)job.GetMapOutputKeyClass();
Type valueClass = (Type)job.GetMapOutputValueClass();
bool keepInputs = job.GetKeepFailedTaskFiles();
Path tmpDir = new Path(reduceId.ToString());
RawComparator <K> comparator = (RawComparator <K>)job.GetOutputKeyComparator();
// segments required to vacate memory
IList <Merger.Segment <K, V> > memDiskSegments = new AList <Merger.Segment <K, V> >();
long inMemToDiskBytes = 0;
bool mergePhaseFinished = false;
if (inMemoryMapOutputs.Count > 0)
{
TaskID mapId = inMemoryMapOutputs[0].GetMapId().GetTaskID();
inMemToDiskBytes = CreateInMemorySegments(inMemoryMapOutputs, memDiskSegments, maxInMemReduce
);
int numMemDiskSegments = memDiskSegments.Count;
if (numMemDiskSegments > 0 && ioSortFactor > onDiskMapOutputs.Count)
{
// If we reach here, it implies that we have less than io.sort.factor
// disk segments and this will be incremented by 1 (result of the
// memory segments merge). Since this total would still be
// <= io.sort.factor, we will not do any more intermediate merges,
// the merge of all these disk segments would be directly fed to the
// reduce method
mergePhaseFinished = true;
// must spill to disk, but can't retain in-mem for intermediate merge
Path outputPath = mapOutputFile.GetInputFileForWrite(mapId, inMemToDiskBytes).Suffix
(Org.Apache.Hadoop.Mapred.Task.MergedOutputPrefix);
RawKeyValueIterator rIter = Merger.Merge(job, fs, keyClass, valueClass, memDiskSegments
, numMemDiskSegments, tmpDir, comparator, reporter, spilledRecordsCounter, null,
mergePhase);
FSDataOutputStream @out = CryptoUtils.WrapIfNecessary(job, fs.Create(outputPath));
IFile.Writer <K, V> writer = new IFile.Writer <K, V>(job, @out, keyClass, valueClass
, codec, null, true);
try
{
Merger.WriteFile(rIter, writer, reporter, job);
writer.Close();
onDiskMapOutputs.AddItem(new MergeManagerImpl.CompressAwarePath(outputPath, writer
.GetRawLength(), writer.GetCompressedLength()));
writer = null;
}
catch (IOException e)
{
// add to list of final disk outputs.
if (null != outputPath)
{
try
{
fs.Delete(outputPath, true);
}
catch (IOException)
{
}
}
// NOTHING
throw;
}
finally
{
if (null != writer)
{
writer.Close();
}
}
Log.Info("Merged " + numMemDiskSegments + " segments, " + inMemToDiskBytes + " bytes to disk to satisfy "
+ "reduce memory limit");
inMemToDiskBytes = 0;
memDiskSegments.Clear();
}
else
{
if (inMemToDiskBytes != 0)
{
Log.Info("Keeping " + numMemDiskSegments + " segments, " + inMemToDiskBytes + " bytes in memory for "
+ "intermediate, on-disk merge");
}
}
}
// segments on disk
IList <Merger.Segment <K, V> > diskSegments = new AList <Merger.Segment <K, V> >();
long onDiskBytes = inMemToDiskBytes;
long rawBytes = inMemToDiskBytes;
MergeManagerImpl.CompressAwarePath[] onDisk = Sharpen.Collections.ToArray(onDiskMapOutputs
, new MergeManagerImpl.CompressAwarePath[onDiskMapOutputs.Count]);
foreach (MergeManagerImpl.CompressAwarePath file in onDisk)
{
long fileLength = fs.GetFileStatus(file).GetLen();
onDiskBytes += fileLength;
rawBytes += (file.GetRawDataLength() > 0) ? file.GetRawDataLength() : fileLength;
Log.Debug("Disk file: " + file + " Length is " + fileLength);
diskSegments.AddItem(new Merger.Segment <K, V>(job, fs, file, codec, keepInputs, (
file.ToString().EndsWith(Org.Apache.Hadoop.Mapred.Task.MergedOutputPrefix) ? null
: mergedMapOutputsCounter), file.GetRawDataLength()));
}
Log.Info("Merging " + onDisk.Length + " files, " + onDiskBytes + " bytes from disk"
);
diskSegments.Sort(new _IComparer_786());
// build final list of segments from merged backed by disk + in-mem
IList <Merger.Segment <K, V> > finalSegments = new AList <Merger.Segment <K, V> >();
long inMemBytes = CreateInMemorySegments(inMemoryMapOutputs, finalSegments, 0);
Log.Info("Merging " + finalSegments.Count + " segments, " + inMemBytes + " bytes from memory into reduce"
);
if (0 != onDiskBytes)
{
int numInMemSegments = memDiskSegments.Count;
diskSegments.AddRange(0, memDiskSegments);
memDiskSegments.Clear();
// Pass mergePhase only if there is a going to be intermediate
// merges. See comment where mergePhaseFinished is being set
Progress thisPhase = (mergePhaseFinished) ? null : mergePhase;
RawKeyValueIterator diskMerge = Merger.Merge(job, fs, keyClass, valueClass, codec
, diskSegments, ioSortFactor, numInMemSegments, tmpDir, comparator, reporter, false
, spilledRecordsCounter, null, thisPhase);
diskSegments.Clear();
if (0 == finalSegments.Count)
{
return(diskMerge);
}
finalSegments.AddItem(new Merger.Segment <K, V>(new MergeManagerImpl.RawKVIteratorReader
(this, diskMerge, onDiskBytes), true, rawBytes));
}
return(Merger.Merge(job, fs, keyClass, valueClass, finalSegments, finalSegments.Count
, tmpDir, comparator, reporter, spilledRecordsCounter, null, null));
}
