Ejemplo n.º 1
0
            /// <exception cref="System.IO.IOException"/>
            private void CopyPartitions(Path mapOutputPath, Path indexPath)
            {
                FileSystem         localFs     = FileSystem.GetLocal(jobConf);
                FileSystem         rfs         = ((LocalFileSystem)localFs).GetRaw();
                FSDataOutputStream rawOutput   = rfs.Create(mapOutputPath, true, BufSize);
                SpillRecord        spillRecord = new SpillRecord(numberOfPartitions);
                IndexRecord        indexRecord = new IndexRecord();

                for (int i = 0; i < numberOfPartitions; i++)
                {
                    indexRecord.startOffset = rawOutput.GetPos();
                    byte[]            buffer         = outStreams[i].ToByteArray();
                    IFileOutputStream checksumOutput = new IFileOutputStream(rawOutput);
                    checksumOutput.Write(buffer);
                    // Write checksum.
                    checksumOutput.Finish();
                    // Write index record
                    indexRecord.rawLength  = (long)buffer.Length;
                    indexRecord.partLength = rawOutput.GetPos() - indexRecord.startOffset;
                    spillRecord.PutIndex(indexRecord, i);
                    reporter.Progress();
                }
                rawOutput.Close();
                spillRecord.WriteToFile(indexPath, jobConf);
            }
Ejemplo n.º 2
0
        /// <exception cref="System.IO.IOException"/>
        private IndexCache.IndexInformation ReadIndexFileToCache(Path indexFileName, string
                                                                 mapId, string expectedIndexOwner)
        {
            IndexCache.IndexInformation info;
            IndexCache.IndexInformation newInd = new IndexCache.IndexInformation();
            if ((info = cache.PutIfAbsent(mapId, newInd)) != null)
            {
                lock (info)
                {
                    while (IsUnderConstruction(info))
                    {
                        try
                        {
                            Sharpen.Runtime.Wait(info);
                        }
                        catch (Exception e)
                        {
                            throw new IOException("Interrupted waiting for construction", e);
                        }
                    }
                }
                Log.Debug("IndexCache HIT: MapId " + mapId + " found");
                return(info);
            }
            Log.Debug("IndexCache MISS: MapId " + mapId + " not found");
            SpillRecord tmp = null;

            try
            {
                tmp = new SpillRecord(indexFileName, conf, expectedIndexOwner);
            }
            catch (Exception e)
            {
                tmp = new SpillRecord(0);
                Sharpen.Collections.Remove(cache, mapId);
                throw new IOException("Error Reading IndexFile", e);
            }
            finally
            {
                lock (newInd)
                {
                    newInd.mapSpillRecord = tmp;
                    Sharpen.Runtime.NotifyAll(newInd);
                }
            }
            queue.AddItem(mapId);
            if (totalMemoryUsed.AddAndGet(newInd.GetSize()) > totalMemoryAllowed)
            {
                FreeIndexInformation();
            }
            return(newInd);
        }