Ejemplo n.º 1
0
            /// <exception cref="System.IO.IOException"/>
            public ChecksumFSInputChecker(ChecksumFileSystem fs, Path file, int bufferSize)
                : base(file, fs.GetFileStatus(file).GetReplication())
            {
                this.datas = fs.GetRawFileSystem().Open(file, bufferSize);
                this.fs    = fs;
                Path sumFile = fs.GetChecksumFile(file);

                try
                {
                    int sumBufferSize = fs.GetSumBufferSize(fs.GetBytesPerSum(), bufferSize);
                    sums = fs.GetRawFileSystem().Open(sumFile, sumBufferSize);
                    byte[] version = new byte[ChecksumVersion.Length];
                    sums.ReadFully(version);
                    if (!Arrays.Equals(version, ChecksumVersion))
                    {
                        throw new IOException("Not a checksum file: " + sumFile);
                    }
                    this.bytesPerSum = sums.ReadInt();
                    Set(fs.verifyChecksum, DataChecksum.NewCrc32(), bytesPerSum, 4);
                }
                catch (FileNotFoundException)
                {
                    // quietly ignore
                    Set(fs.verifyChecksum, null, 1, 0);
                }
                catch (IOException e)
                {
                    // loudly ignore
                    Log.Warn("Problem opening checksum file: " + file + ".  Ignoring exception: ", e);
                    Set(fs.verifyChecksum, null, 1, 0);
                }
            }
Ejemplo n.º 2
0
        /// <exception cref="System.IO.IOException"/>
        public override void Recover()
        {
            /*
             * Steps of recover
             * 1) Read from last mirror (from mirror or mirror.old)
             * 2) Read from last edit log, and apply such edit log
             * 3) Write new mirror to mirror.writing
             * 4) Rename mirror to mirror.old
             * 5) Move mirror.writing to mirror
             * 6) Remove mirror.old
             * 7) Remove edit log and create a new empty edit log
             */
            // Open mirror from serialized file
            Path mirrorPath       = new Path(fsWorkingPath, MirrorFilename);
            Path oldMirrorPath    = new Path(fsWorkingPath, MirrorFilename + ".old");
            FSDataInputStream @is = null;

            if (fs.Exists(mirrorPath))
            {
                @is = fs.Open(mirrorPath);
            }
            else
            {
                if (fs.Exists(oldMirrorPath))
                {
                    @is = fs.Open(oldMirrorPath);
                }
            }
            if (null != @is)
            {
                ICollection <string> labels = new AddToClusterNodeLabelsRequestPBImpl(YarnServerResourceManagerServiceProtos.AddToClusterNodeLabelsRequestProto
                                                                                      .ParseDelimitedFrom(@is)).GetNodeLabels();
                IDictionary <NodeId, ICollection <string> > nodeToLabels = new ReplaceLabelsOnNodeRequestPBImpl
                                                                               (YarnServerResourceManagerServiceProtos.ReplaceLabelsOnNodeRequestProto.ParseDelimitedFrom
                                                                                   (@is)).GetNodeToLabels();
                mgr.AddToCluserNodeLabels(labels);
                mgr.ReplaceLabelsOnNode(nodeToLabels);
                @is.Close();
            }
            // Open and process editlog
            editLogPath = new Path(fsWorkingPath, EditlogFilename);
            if (fs.Exists(editLogPath))
            {
                @is = fs.Open(editLogPath);
                while (true)
                {
                    try
                    {
                        // read edit log one by one
                        FileSystemNodeLabelsStore.SerializedLogType type = FileSystemNodeLabelsStore.SerializedLogType
                                                                           .Values()[@is.ReadInt()];
                        switch (type)
                        {
                        case FileSystemNodeLabelsStore.SerializedLogType.AddLabels:
                        {
                            ICollection <string> labels = YarnServerResourceManagerServiceProtos.AddToClusterNodeLabelsRequestProto
                                                          .ParseDelimitedFrom(@is).GetNodeLabelsList();
                            mgr.AddToCluserNodeLabels(Sets.NewHashSet(labels.GetEnumerator()));
                            break;
                        }

                        case FileSystemNodeLabelsStore.SerializedLogType.RemoveLabels:
                        {
                            ICollection <string> labels = YarnServerResourceManagerServiceProtos.RemoveFromClusterNodeLabelsRequestProto
                                                          .ParseDelimitedFrom(@is).GetNodeLabelsList();
                            mgr.RemoveFromClusterNodeLabels(labels);
                            break;
                        }

                        case FileSystemNodeLabelsStore.SerializedLogType.NodeToLabels:
                        {
                            IDictionary <NodeId, ICollection <string> > map = new ReplaceLabelsOnNodeRequestPBImpl
                                                                                  (YarnServerResourceManagerServiceProtos.ReplaceLabelsOnNodeRequestProto.ParseDelimitedFrom
                                                                                      (@is)).GetNodeToLabels();
                            mgr.ReplaceLabelsOnNode(map);
                            break;
                        }
                        }
                    }
                    catch (EOFException)
                    {
                        // EOF hit, break
                        break;
                    }
                }
            }
            // Serialize current mirror to mirror.writing
            Path writingMirrorPath = new Path(fsWorkingPath, MirrorFilename + ".writing");
            FSDataOutputStream os  = fs.Create(writingMirrorPath, true);

            ((AddToClusterNodeLabelsRequestPBImpl)AddToClusterNodeLabelsRequestPBImpl.NewInstance
                 (mgr.GetClusterNodeLabels())).GetProto().WriteDelimitedTo(os);
            ((ReplaceLabelsOnNodeRequestPBImpl)ReplaceLabelsOnNodeRequest.NewInstance(mgr.GetNodeLabels
                                                                                          ())).GetProto().WriteDelimitedTo(os);
            os.Close();
            // Move mirror to mirror.old
            if (fs.Exists(mirrorPath))
            {
                fs.Delete(oldMirrorPath, false);
                fs.Rename(mirrorPath, oldMirrorPath);
            }
            // move mirror.writing to mirror
            fs.Rename(writingMirrorPath, mirrorPath);
            fs.Delete(writingMirrorPath, false);
            // remove mirror.old
            fs.Delete(oldMirrorPath, false);
            // create a new editlog file
            editlogOs = fs.Create(editLogPath, true);
            editlogOs.Close();
            Log.Info("Finished write mirror at:" + mirrorPath.ToString());
            Log.Info("Finished create editlog file at:" + editLogPath.ToString());
        }