/// <exception cref="System.IO.IOException"/> private void CopyPartitions(Path mapOutputPath, Path indexPath) { FileSystem localFs = FileSystem.GetLocal(jobConf); FileSystem rfs = ((LocalFileSystem)localFs).GetRaw(); FSDataOutputStream rawOutput = rfs.Create(mapOutputPath, true, BufSize); SpillRecord spillRecord = new SpillRecord(numberOfPartitions); IndexRecord indexRecord = new IndexRecord(); for (int i = 0; i < numberOfPartitions; i++) { indexRecord.startOffset = rawOutput.GetPos(); byte[] buffer = outStreams[i].ToByteArray(); IFileOutputStream checksumOutput = new IFileOutputStream(rawOutput); checksumOutput.Write(buffer); // Write checksum. checksumOutput.Finish(); // Write index record indexRecord.rawLength = (long)buffer.Length; indexRecord.partLength = rawOutput.GetPos() - indexRecord.startOffset; spillRecord.PutIndex(indexRecord, i); reporter.Progress(); } rawOutput.Close(); spillRecord.WriteToFile(indexPath, jobConf); }
/// <exception cref="System.IO.IOException"/> private IndexCache.IndexInformation ReadIndexFileToCache(Path indexFileName, string mapId, string expectedIndexOwner) { IndexCache.IndexInformation info; IndexCache.IndexInformation newInd = new IndexCache.IndexInformation(); if ((info = cache.PutIfAbsent(mapId, newInd)) != null) { lock (info) { while (IsUnderConstruction(info)) { try { Sharpen.Runtime.Wait(info); } catch (Exception e) { throw new IOException("Interrupted waiting for construction", e); } } } Log.Debug("IndexCache HIT: MapId " + mapId + " found"); return(info); } Log.Debug("IndexCache MISS: MapId " + mapId + " not found"); SpillRecord tmp = null; try { tmp = new SpillRecord(indexFileName, conf, expectedIndexOwner); } catch (Exception e) { tmp = new SpillRecord(0); Sharpen.Collections.Remove(cache, mapId); throw new IOException("Error Reading IndexFile", e); } finally { lock (newInd) { newInd.mapSpillRecord = tmp; Sharpen.Runtime.NotifyAll(newInd); } } queue.AddItem(mapId); if (totalMemoryUsed.AddAndGet(newInd.GetSize()) > totalMemoryAllowed) { FreeIndexInformation(); } return(newInd); }