/// <exception cref="System.IO.IOException"/>
public ChecksumFSInputChecker(ChecksumFileSystem fs, Path file, int bufferSize)
: base(file, fs.GetFileStatus(file).GetReplication())
{
this.datas = fs.GetRawFileSystem().Open(file, bufferSize);
this.fs = fs;
Path sumFile = fs.GetChecksumFile(file);
try
{
int sumBufferSize = fs.GetSumBufferSize(fs.GetBytesPerSum(), bufferSize);
sums = fs.GetRawFileSystem().Open(sumFile, sumBufferSize);
byte[] version = new byte[ChecksumVersion.Length];
sums.ReadFully(version);
if (!Arrays.Equals(version, ChecksumVersion))
{
throw new IOException("Not a checksum file: " + sumFile);
}
this.bytesPerSum = sums.ReadInt();
Set(fs.verifyChecksum, DataChecksum.NewCrc32(), bytesPerSum, 4);
}
catch (FileNotFoundException)
{
// quietly ignore
Set(fs.verifyChecksum, null, 1, 0);
}
catch (IOException e)
{
// loudly ignore
Log.Warn("Problem opening checksum file: " + file + ". Ignoring exception: ", e);
Set(fs.verifyChecksum, null, 1, 0);
}
}
/// <exception cref="System.IO.IOException"/>
public override void Recover()
{
/*
* Steps of recover
* 1) Read from last mirror (from mirror or mirror.old)
* 2) Read from last edit log, and apply such edit log
* 3) Write new mirror to mirror.writing
* 4) Rename mirror to mirror.old
* 5) Move mirror.writing to mirror
* 6) Remove mirror.old
* 7) Remove edit log and create a new empty edit log
*/
// Open mirror from serialized file
Path mirrorPath = new Path(fsWorkingPath, MirrorFilename);
Path oldMirrorPath = new Path(fsWorkingPath, MirrorFilename + ".old");
FSDataInputStream @is = null;
if (fs.Exists(mirrorPath))
{
@is = fs.Open(mirrorPath);
}
else
{
if (fs.Exists(oldMirrorPath))
{
@is = fs.Open(oldMirrorPath);
}
}
if (null != @is)
{
ICollection <string> labels = new AddToClusterNodeLabelsRequestPBImpl(YarnServerResourceManagerServiceProtos.AddToClusterNodeLabelsRequestProto
.ParseDelimitedFrom(@is)).GetNodeLabels();
IDictionary <NodeId, ICollection <string> > nodeToLabels = new ReplaceLabelsOnNodeRequestPBImpl
(YarnServerResourceManagerServiceProtos.ReplaceLabelsOnNodeRequestProto.ParseDelimitedFrom
(@is)).GetNodeToLabels();
mgr.AddToCluserNodeLabels(labels);
mgr.ReplaceLabelsOnNode(nodeToLabels);
@is.Close();
}
// Open and process editlog
editLogPath = new Path(fsWorkingPath, EditlogFilename);
if (fs.Exists(editLogPath))
{
@is = fs.Open(editLogPath);
while (true)
{
try
{
// read edit log one by one
FileSystemNodeLabelsStore.SerializedLogType type = FileSystemNodeLabelsStore.SerializedLogType
.Values()[@is.ReadInt()];
switch (type)
{
case FileSystemNodeLabelsStore.SerializedLogType.AddLabels:
{
ICollection <string> labels = YarnServerResourceManagerServiceProtos.AddToClusterNodeLabelsRequestProto
.ParseDelimitedFrom(@is).GetNodeLabelsList();
mgr.AddToCluserNodeLabels(Sets.NewHashSet(labels.GetEnumerator()));
break;
}
case FileSystemNodeLabelsStore.SerializedLogType.RemoveLabels:
{
ICollection <string> labels = YarnServerResourceManagerServiceProtos.RemoveFromClusterNodeLabelsRequestProto
.ParseDelimitedFrom(@is).GetNodeLabelsList();
mgr.RemoveFromClusterNodeLabels(labels);
break;
}
case FileSystemNodeLabelsStore.SerializedLogType.NodeToLabels:
{
IDictionary <NodeId, ICollection <string> > map = new ReplaceLabelsOnNodeRequestPBImpl
(YarnServerResourceManagerServiceProtos.ReplaceLabelsOnNodeRequestProto.ParseDelimitedFrom
(@is)).GetNodeToLabels();
mgr.ReplaceLabelsOnNode(map);
break;
}
}
}
catch (EOFException)
{
// EOF hit, break
break;
}
}
}
// Serialize current mirror to mirror.writing
Path writingMirrorPath = new Path(fsWorkingPath, MirrorFilename + ".writing");
FSDataOutputStream os = fs.Create(writingMirrorPath, true);
((AddToClusterNodeLabelsRequestPBImpl)AddToClusterNodeLabelsRequestPBImpl.NewInstance
(mgr.GetClusterNodeLabels())).GetProto().WriteDelimitedTo(os);
((ReplaceLabelsOnNodeRequestPBImpl)ReplaceLabelsOnNodeRequest.NewInstance(mgr.GetNodeLabels
())).GetProto().WriteDelimitedTo(os);
os.Close();
// Move mirror to mirror.old
if (fs.Exists(mirrorPath))
{
fs.Delete(oldMirrorPath, false);
fs.Rename(mirrorPath, oldMirrorPath);
}
// move mirror.writing to mirror
fs.Rename(writingMirrorPath, mirrorPath);
fs.Delete(writingMirrorPath, false);
// remove mirror.old
fs.Delete(oldMirrorPath, false);
// create a new editlog file
editlogOs = fs.Create(editLogPath, true);
editlogOs.Close();
Log.Info("Finished write mirror at:" + mirrorPath.ToString());
Log.Info("Finished create editlog file at:" + editLogPath.ToString());
}