示例#1
0
 /// <summary>There is no persistent storage.</summary>
 /// <remarks>There is no persistent storage. Just clear the buffers.</remarks>
 /// <exception cref="System.IO.IOException"/>
 public override void Create(int layoutVersion)
 {
     // EditLogOutputStream
     System.Diagnostics.Debug.Assert(doubleBuf.IsFlushed(), "previous data is not flushed yet"
                                     );
     this.doubleBuf = new EditsDoubleBuffer(DefaultBufferSize);
 }
        public virtual void TestDoubleBuffer()
        {
            EditsDoubleBuffer buf = new EditsDoubleBuffer(1024);

            NUnit.Framework.Assert.IsTrue(buf.IsFlushed());
            byte[] data = new byte[100];
            buf.WriteRaw(data, 0, data.Length);
            NUnit.Framework.Assert.AreEqual("Should count new data correctly", data.Length, buf
                                            .CountBufferedBytes());
            NUnit.Framework.Assert.IsTrue("Writing to current buffer should not affect flush state"
                                          , buf.IsFlushed());
            // Swap the buffers
            buf.SetReadyToFlush();
            NUnit.Framework.Assert.AreEqual("Swapping buffers should still count buffered bytes"
                                            , data.Length, buf.CountBufferedBytes());
            NUnit.Framework.Assert.IsFalse(buf.IsFlushed());
            // Flush to a stream
            DataOutputBuffer outBuf = new DataOutputBuffer();

            buf.FlushTo(outBuf);
            NUnit.Framework.Assert.AreEqual(data.Length, outBuf.GetLength());
            NUnit.Framework.Assert.IsTrue(buf.IsFlushed());
            NUnit.Framework.Assert.AreEqual(0, buf.CountBufferedBytes());
            // Write some more
            buf.WriteRaw(data, 0, data.Length);
            NUnit.Framework.Assert.AreEqual("Should count new data correctly", data.Length, buf
                                            .CountBufferedBytes());
            buf.SetReadyToFlush();
            buf.FlushTo(outBuf);
            NUnit.Framework.Assert.AreEqual(data.Length * 2, outBuf.GetLength());
            NUnit.Framework.Assert.AreEqual(0, buf.CountBufferedBytes());
            outBuf.Close();
        }
示例#3
0
        /// <exception cref="System.IO.IOException"/>
        internal EditLogBackupOutputStream(NamenodeRegistration bnReg, JournalInfo journalInfo
                                           )
            : base()
        {
            // RPC proxy to backup node
            // backup node registration
            // active node registration
            // serialized output sent to backup node
            // backup node
            // active name-node
            this.bnRegistration = bnReg;
            this.journalInfo    = journalInfo;
            IPEndPoint bnAddress = NetUtils.CreateSocketAddr(bnRegistration.GetAddress());

            try
            {
                this.backupNode = NameNodeProxies.CreateNonHAProxy <JournalProtocol>(new HdfsConfiguration
                                                                                         (), bnAddress, UserGroupInformation.GetCurrentUser(), true).GetProxy();
            }
            catch (IOException e)
            {
                Storage.Log.Error("Error connecting to: " + bnAddress, e);
                throw;
            }
            this.doubleBuf = new EditsDoubleBuffer(DefaultBufferSize);
            this.@out      = new DataOutputBuffer(DefaultBufferSize);
        }
示例#4
0
 /// <exception cref="System.IO.IOException"/>
 public override void Close()
 {
     if (fp == null)
     {
         throw new IOException("Trying to use aborted output stream");
     }
     try
     {
         // close should have been called after all pending transactions
         // have been flushed & synced.
         // if already closed, just skip
         if (doubleBuf != null)
         {
             doubleBuf.Close();
             doubleBuf = null;
         }
         // remove any preallocated padding bytes from the transaction log.
         if (fc != null && fc.IsOpen())
         {
             fc.Truncate(fc.Position());
             fc.Close();
             fc = null;
         }
         fp.Close();
         fp = null;
     }
     finally
     {
         IOUtils.Cleanup(Log, fc, fp);
         doubleBuf = null;
         fc        = null;
         fp        = null;
     }
     fp = null;
 }
示例#5
0
        /// <exception cref="System.IO.IOException"/>
        public override void Close()
        {
            // EditLogOutputStream
            // close should have been called after all pending transactions
            // have been flushed & synced.
            int size = doubleBuf.CountBufferedBytes();

            if (size != 0)
            {
                throw new IOException("BackupEditStream has " + size + " records still to be flushed and cannot be closed."
                                      );
            }
            RPC.StopProxy(backupNode);
            // stop the RPC threads
            doubleBuf.Close();
            doubleBuf = null;
        }
        public virtual void ShouldFailToCloseWhenUnflushed()
        {
            EditsDoubleBuffer buf = new EditsDoubleBuffer(1024);

            buf.WriteRaw(new byte[1], 0, 1);
            try
            {
                buf.Close();
                NUnit.Framework.Assert.Fail("Did not fail to close with unflushed data");
            }
            catch (IOException ioe)
            {
                if (!ioe.ToString().Contains("still to be flushed"))
                {
                    throw;
                }
            }
        }
示例#7
0
        /// <summary>Creates output buffers and file object.</summary>
        /// <param name="conf">Configuration object</param>
        /// <param name="name">File name to store edit log</param>
        /// <param name="size">Size of flush buffer</param>
        /// <exception cref="System.IO.IOException"/>
        public EditLogFileOutputStream(Configuration conf, FilePath name, int size)
            : base()
        {
            shouldSyncWritesAndSkipFsync = conf.GetBoolean(DFSConfigKeys.DfsNamenodeEditsNoeditlogchannelflush
                                                           , DFSConfigKeys.DfsNamenodeEditsNoeditlogchannelflushDefault);
            file      = name;
            doubleBuf = new EditsDoubleBuffer(size);
            RandomAccessFile rp;

            if (shouldSyncWritesAndSkipFsync)
            {
                rp = new RandomAccessFile(name, "rws");
            }
            else
            {
                rp = new RandomAccessFile(name, "rw");
            }
            fp = new FileOutputStream(rp.GetFD());
            // open for append
            fc = rp.GetChannel();
            fc.Position(fc.Size());
        }
示例#8
0
 /// <exception cref="System.IO.IOException"/>
 public override void Abort()
 {
     RPC.StopProxy(backupNode);
     doubleBuf = null;
 }