/// <exception cref="System.IO.IOException"/> public ChecksumFSInputChecker(ChecksumFileSystem fs, Path file, int bufferSize) : base(file, fs.GetFileStatus(file).GetReplication()) { this.datas = fs.GetRawFileSystem().Open(file, bufferSize); this.fs = fs; Path sumFile = fs.GetChecksumFile(file); try { int sumBufferSize = fs.GetSumBufferSize(fs.GetBytesPerSum(), bufferSize); sums = fs.GetRawFileSystem().Open(sumFile, sumBufferSize); byte[] version = new byte[ChecksumVersion.Length]; sums.ReadFully(version); if (!Arrays.Equals(version, ChecksumVersion)) { throw new IOException("Not a checksum file: " + sumFile); } this.bytesPerSum = sums.ReadInt(); Set(fs.verifyChecksum, DataChecksum.NewCrc32(), bytesPerSum, 4); } catch (FileNotFoundException) { // quietly ignore Set(fs.verifyChecksum, null, 1, 0); } catch (IOException e) { // loudly ignore Log.Warn("Problem opening checksum file: " + file + ". Ignoring exception: ", e); Set(fs.verifyChecksum, null, 1, 0); } }
/// <exception cref="System.IO.IOException"/> public override void Recover() { /* * Steps of recover * 1) Read from last mirror (from mirror or mirror.old) * 2) Read from last edit log, and apply such edit log * 3) Write new mirror to mirror.writing * 4) Rename mirror to mirror.old * 5) Move mirror.writing to mirror * 6) Remove mirror.old * 7) Remove edit log and create a new empty edit log */ // Open mirror from serialized file Path mirrorPath = new Path(fsWorkingPath, MirrorFilename); Path oldMirrorPath = new Path(fsWorkingPath, MirrorFilename + ".old"); FSDataInputStream @is = null; if (fs.Exists(mirrorPath)) { @is = fs.Open(mirrorPath); } else { if (fs.Exists(oldMirrorPath)) { @is = fs.Open(oldMirrorPath); } } if (null != @is) { ICollection <string> labels = new AddToClusterNodeLabelsRequestPBImpl(YarnServerResourceManagerServiceProtos.AddToClusterNodeLabelsRequestProto .ParseDelimitedFrom(@is)).GetNodeLabels(); IDictionary <NodeId, ICollection <string> > nodeToLabels = new ReplaceLabelsOnNodeRequestPBImpl (YarnServerResourceManagerServiceProtos.ReplaceLabelsOnNodeRequestProto.ParseDelimitedFrom (@is)).GetNodeToLabels(); mgr.AddToCluserNodeLabels(labels); mgr.ReplaceLabelsOnNode(nodeToLabels); @is.Close(); } // Open and process editlog editLogPath = new Path(fsWorkingPath, EditlogFilename); if (fs.Exists(editLogPath)) { @is = fs.Open(editLogPath); while (true) { try { // read edit log one by one FileSystemNodeLabelsStore.SerializedLogType type = FileSystemNodeLabelsStore.SerializedLogType .Values()[@is.ReadInt()]; switch (type) { case FileSystemNodeLabelsStore.SerializedLogType.AddLabels: { ICollection <string> labels = YarnServerResourceManagerServiceProtos.AddToClusterNodeLabelsRequestProto .ParseDelimitedFrom(@is).GetNodeLabelsList(); mgr.AddToCluserNodeLabels(Sets.NewHashSet(labels.GetEnumerator())); break; } case FileSystemNodeLabelsStore.SerializedLogType.RemoveLabels: { ICollection <string> labels = YarnServerResourceManagerServiceProtos.RemoveFromClusterNodeLabelsRequestProto .ParseDelimitedFrom(@is).GetNodeLabelsList(); mgr.RemoveFromClusterNodeLabels(labels); break; } case FileSystemNodeLabelsStore.SerializedLogType.NodeToLabels: { IDictionary <NodeId, ICollection <string> > map = new ReplaceLabelsOnNodeRequestPBImpl (YarnServerResourceManagerServiceProtos.ReplaceLabelsOnNodeRequestProto.ParseDelimitedFrom (@is)).GetNodeToLabels(); mgr.ReplaceLabelsOnNode(map); break; } } } catch (EOFException) { // EOF hit, break break; } } } // Serialize current mirror to mirror.writing Path writingMirrorPath = new Path(fsWorkingPath, MirrorFilename + ".writing"); FSDataOutputStream os = fs.Create(writingMirrorPath, true); ((AddToClusterNodeLabelsRequestPBImpl)AddToClusterNodeLabelsRequestPBImpl.NewInstance (mgr.GetClusterNodeLabels())).GetProto().WriteDelimitedTo(os); ((ReplaceLabelsOnNodeRequestPBImpl)ReplaceLabelsOnNodeRequest.NewInstance(mgr.GetNodeLabels ())).GetProto().WriteDelimitedTo(os); os.Close(); // Move mirror to mirror.old if (fs.Exists(mirrorPath)) { fs.Delete(oldMirrorPath, false); fs.Rename(mirrorPath, oldMirrorPath); } // move mirror.writing to mirror fs.Rename(writingMirrorPath, mirrorPath); fs.Delete(writingMirrorPath, false); // remove mirror.old fs.Delete(oldMirrorPath, false); // create a new editlog file editlogOs = fs.Create(editLogPath, true); editlogOs.Close(); Log.Info("Finished write mirror at:" + mirrorPath.ToString()); Log.Info("Finished create editlog file at:" + editLogPath.ToString()); }