public virtual void TestDefaultPolicy()
{
Configuration conf = new HdfsConfiguration();
ReplaceDatanodeOnFailure p = ReplaceDatanodeOnFailure.Get(conf);
DatanodeInfo[] infos = new DatanodeInfo[5];
DatanodeInfo[][] datanodes = new DatanodeInfo[infos.Length + 1][];
datanodes[0] = new DatanodeInfo[0];
for (int i = 0; i < infos.Length;)
{
infos[i] = DFSTestUtil.GetLocalDatanodeInfo(50020 + i);
i++;
datanodes[i] = new DatanodeInfo[i];
System.Array.Copy(infos, 0, datanodes[i], 0, datanodes[i].Length);
}
bool[] isAppend = new bool[] { true, true, false, false };
bool[] isHflushed = new bool[] { true, false, true, false };
for (short replication = 1; replication <= infos.Length; replication++)
{
for (int nExistings = 0; nExistings < datanodes.Length; nExistings++)
{
DatanodeInfo[] existings = datanodes[nExistings];
NUnit.Framework.Assert.AreEqual(nExistings, existings.Length);
for (int i_1 = 0; i_1 < isAppend.Length; i_1++)
{
for (int j = 0; j < isHflushed.Length; j++)
{
int half = replication / 2;
bool enoughReplica = replication <= nExistings;
bool noReplica = nExistings == 0;
bool replicationL3 = replication < 3;
bool existingsLEhalf = nExistings <= half;
bool isAH = isAppend[i_1] || isHflushed[j];
bool expected;
if (enoughReplica || noReplica || replicationL3)
{
expected = false;
}
else
{
expected = isAH || existingsLEhalf;
}
bool computed = p.Satisfy(replication, existings, isAppend[i_1], isHflushed[j]);
try
{
NUnit.Framework.Assert.AreEqual(expected, computed);
}
catch (Exception e)
{
string s = "replication=" + replication + "\nnExistings =" + nExistings + "\nisAppend ="
+ isAppend[i_1] + "\nisHflushed =" + isHflushed[j];
throw new RuntimeException(s, e);
}
}
}
}
}
}
public virtual void TestBestEffort()
{
Configuration conf = new HdfsConfiguration();
//always replace a datanode but do not throw exception
ReplaceDatanodeOnFailure.Write(ReplaceDatanodeOnFailure.Policy.Always, true, conf
);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
try
{
DistributedFileSystem fs = cluster.GetFileSystem();
Path f = new Path(Dir, "testIgnoreReplaceFailure");
byte[] bytes = new byte[1000];
{
Log.Info("write " + bytes.Length + " bytes to " + f);
FSDataOutputStream @out = fs.Create(f, Replication);
@out.Write(bytes);
@out.Close();
FileStatus status = fs.GetFileStatus(f);
NUnit.Framework.Assert.AreEqual(Replication, status.GetReplication());
NUnit.Framework.Assert.AreEqual(bytes.Length, status.GetLen());
}
{
Log.Info("append another " + bytes.Length + " bytes to " + f);
FSDataOutputStream @out = fs.Append(f);
@out.Write(bytes);
@out.Close();
}
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestReplaceDatanodeOnFailure()
{
Configuration conf = new HdfsConfiguration();
//always replace a datanode
ReplaceDatanodeOnFailure.Write(ReplaceDatanodeOnFailure.Policy.Always, true, conf
);
string[] racks = new string[Replication];
Arrays.Fill(racks, Rack0);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Racks(racks).NumDataNodes
(Replication).Build();
try
{
DistributedFileSystem fs = cluster.GetFileSystem();
Path dir = new Path(Dir);
TestReplaceDatanodeOnFailure.SlowWriter[] slowwriters = new TestReplaceDatanodeOnFailure.SlowWriter
[10];
for (int i = 1; i <= slowwriters.Length; i++)
{
//create slow writers in different speed
slowwriters[i - 1] = new TestReplaceDatanodeOnFailure.SlowWriter(fs, new Path(dir
, "file" + i), i * 200L);
}
foreach (TestReplaceDatanodeOnFailure.SlowWriter s in slowwriters)
{
s.Start();
}
// Let slow writers write something.
// Some of them are too slow and will be not yet started.
SleepSeconds(1);
//start new datanodes
cluster.StartDataNodes(conf, 2, true, null, new string[] { Rack1, Rack1 });
//stop an old datanode
cluster.StopDataNode(AppendTestUtil.NextInt(Replication));
//Let the slow writer writes a few more seconds
//Everyone should have written something.
SleepSeconds(5);
//check replication and interrupt.
foreach (TestReplaceDatanodeOnFailure.SlowWriter s_1 in slowwriters)
{
s_1.CheckReplication();
s_1.InterruptRunning();
}
//close files
foreach (TestReplaceDatanodeOnFailure.SlowWriter s_2 in slowwriters)
{
s_2.JoinAndClose();
}
//Verify the file
Log.Info("Verify the file");
for (int i_1 = 0; i_1 < slowwriters.Length; i_1++)
{
Log.Info(slowwriters[i_1].filepath + ": length=" + fs.GetFileStatus(slowwriters[i_1
].filepath).GetLen());
FSDataInputStream @in = null;
try
{
@in = fs.Open(slowwriters[i_1].filepath);
for (int j = 0; (x = @in.Read()) != -1; j++)
{
NUnit.Framework.Assert.AreEqual(j, x);
}
}
finally
{
IOUtils.CloseStream(@in);
}
}
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
