/// <exception cref="System.IO.IOException"/>
private void CopyPartitions(Path mapOutputPath, Path indexPath)
{
FileSystem localFs = FileSystem.GetLocal(jobConf);
FileSystem rfs = ((LocalFileSystem)localFs).GetRaw();
FSDataOutputStream rawOutput = rfs.Create(mapOutputPath, true, BufSize);
SpillRecord spillRecord = new SpillRecord(numberOfPartitions);
IndexRecord indexRecord = new IndexRecord();
for (int i = 0; i < numberOfPartitions; i++)
{
indexRecord.startOffset = rawOutput.GetPos();
byte[] buffer = outStreams[i].ToByteArray();
IFileOutputStream checksumOutput = new IFileOutputStream(rawOutput);
checksumOutput.Write(buffer);
// Write checksum.
checksumOutput.Finish();
// Write index record
indexRecord.rawLength = (long)buffer.Length;
indexRecord.partLength = rawOutput.GetPos() - indexRecord.startOffset;
spillRecord.PutIndex(indexRecord, i);
reporter.Progress();
}
rawOutput.Close();
spillRecord.WriteToFile(indexPath, jobConf);
}
/// <exception cref="System.IO.IOException"/>
private IndexCache.IndexInformation ReadIndexFileToCache(Path indexFileName, string
mapId, string expectedIndexOwner)
{
IndexCache.IndexInformation info;
IndexCache.IndexInformation newInd = new IndexCache.IndexInformation();
if ((info = cache.PutIfAbsent(mapId, newInd)) != null)
{
lock (info)
{
while (IsUnderConstruction(info))
{
try
{
Sharpen.Runtime.Wait(info);
}
catch (Exception e)
{
throw new IOException("Interrupted waiting for construction", e);
}
}
}
Log.Debug("IndexCache HIT: MapId " + mapId + " found");
return(info);
}
Log.Debug("IndexCache MISS: MapId " + mapId + " not found");
SpillRecord tmp = null;
try
{
tmp = new SpillRecord(indexFileName, conf, expectedIndexOwner);
}
catch (Exception e)
{
tmp = new SpillRecord(0);
Sharpen.Collections.Remove(cache, mapId);
throw new IOException("Error Reading IndexFile", e);
}
finally
{
lock (newInd)
{
newInd.mapSpillRecord = tmp;
Sharpen.Runtime.NotifyAll(newInd);
}
}
queue.AddItem(mapId);
if (totalMemoryUsed.AddAndGet(newInd.GetSize()) > totalMemoryAllowed)
{
FreeIndexInformation();
}
return(newInd);
}