public void GivenATreeBuiltWithNameGroupingOnly()
{
var tb = new TreeBuilder<INode>();
tb.Group<NameNode>().As(nn => new PartNode { Part = PartNode.Strip(nn.Text) });
var nn1 = new NameNode { Text = "X.Y.U" };
var nn2 = new NameNode { Text = "X.Y.V" };
_result = tb.Build(new[] { nn1, nn2 });
}
public void TestThatIncompleteTreeResultsInProperError()
{
var tb = new TreeBuilder<INode>();
tb.Group<NameNode>().As(nn => new PartNode { Part = PartNode.Strip(nn.Text) });
var nn1 = new NameNode { Text = "X.Y" };
var nn2 = new NameNode { Text = "A.B" };
Assert.Throws<ArgumentException>(() => tb.Build(new[] {nn1, nn2}));
}
static VBParameter ExtractLambdaParameter(NameNode name)
{
if (name == null)
return null;
if (!(name.Parent is DeclaratorNode) || !(name.Parent.Parent is ParameterNode))
return null;
LambdaNode lambdaNode = name.Parent.Parent.Parent as LambdaNode;
if (lambdaNode == null)
return null;
LambdaExpression lambdaExpression = name.Tree.SourceFile.Binder.CompileExpression(lambdaNode) as LambdaExpression;
if (lambdaExpression == null)
return null;
return lambdaExpression.Parameters
.FirstOrDefault(p => p.Name.EqualsNoCase(((IdentifierNode)name).Name));
}
public void CreateSchemaError_AddSyntaxNode()
{
// arrange
var message = "FooBar";
var node = new NameNode("foo");
// act
ISchemaError schemaError = SchemaErrorBuilder.New()
.SetMessage(message)
.AddSyntaxNode(node)
.Build();
// assert
Assert.Equal(message, schemaError.Message);
Assert.Collection(schemaError.SyntaxNodes,
t => Assert.Equal(node, t));
Assert.Empty(schemaError.Extensions);
Assert.Null(schemaError.Exception);
Assert.Null(schemaError.TypeSystemObject);
Assert.Null(schemaError.Path);
Assert.Null(schemaError.Code);
}
public virtual void TestFormatShouldBeIgnoredForNonFileBasedDirs()
{
Configuration conf = new HdfsConfiguration();
string logicalName = "mycluster";
// DFS_NAMENODE_RPC_ADDRESS_KEY are required to identify the NameNode
// is configured in HA, then only DFS_NAMENODE_SHARED_EDITS_DIR_KEY
// is considered.
string localhost = "127.0.0.1";
IPEndPoint nnAddr1 = new IPEndPoint(localhost, 8020);
IPEndPoint nnAddr2 = new IPEndPoint(localhost, 9020);
HATestUtil.SetFailoverConfigurations(conf, logicalName, nnAddr1, nnAddr2);
conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, new FilePath(DfsBaseDir, "name").GetAbsolutePath
());
conf.SetBoolean(DFSConfigKeys.DfsNamenodeSupportAllowFormatKey, true);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeEditsPluginPrefix, "dummy"
), typeof(TestGenericJournalConf.DummyJournalManager).FullName);
conf.Set(DFSConfigKeys.DfsNamenodeSharedEditsDirKey, "dummy://" + localhost + ":2181/ledgers"
);
conf.Set(DFSConfigKeys.DfsHaNamenodeIdKey, "nn1");
// An internal assert is added to verify the working of test
NameNode.Format(conf);
}
private NameNode GoDeeperIntoGenericArgumentList(TypeReference reference, NameNode nameNode)
{
var deepestNameNode = nameNode;
if (reference.IsGenericInstance)
{
var startOfTypeNode = deepestNameNode.StartArgumentList();
var referenceAsGenericInstance = reference as GenericInstanceType;
if (referenceAsGenericInstance.HasGenericArguments)
{
for (int i = 0; i < referenceAsGenericInstance.GenericArguments.Count; i++)
{
deepestNameNode = TraverseThroughReferenceName(referenceAsGenericInstance.GenericArguments[i], startOfTypeNode);
if (i < referenceAsGenericInstance.GenericArguments.Count - 1)
{
startOfTypeNode = deepestNameNode.AddAnotherArgument();
}
}
}
deepestNameNode = deepestNameNode.EndArgumentList();
}
return(deepestNameNode);
}
/// <summary>
/// Test that if splitThreshold is zero, then we always get a separate
/// call per storage.
/// </summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void TestAlwaysSplit()
{
StartUpCluster(0);
NameNode nn = cluster.GetNameNode();
DataNode dn = cluster.GetDataNodes()[0];
// Create a file with a few blocks.
CreateFile(GenericTestUtils.GetMethodName(), BlocksInFile);
// Insert a spy object for the NN RPC.
DatanodeProtocolClientSideTranslatorPB nnSpy = DataNodeTestUtils.SpyOnBposToNN(dn
, nn);
// Trigger a block report so there is an interaction with the spy
// object.
DataNodeTestUtils.TriggerBlockReport(dn);
ArgumentCaptor <StorageBlockReport[]> captor = ArgumentCaptor.ForClass <StorageBlockReport
[]>();
Org.Mockito.Mockito.Verify(nnSpy, Org.Mockito.Mockito.Times(cluster.GetStoragesPerDatanode
())).BlockReport(Matchers.Any <DatanodeRegistration>(), Matchers.AnyString(), captor
.Capture(), Org.Mockito.Mockito.AnyObject <BlockReportContext>());
VerifyCapturedArguments(captor, 1, BlocksInFile);
}
/// <summary>
/// Tests setting the rpc port to a different as the web port that an
/// exception is NOT thrown
/// </summary>
/// <exception cref="System.IO.IOException"/>
public virtual void TestThatDifferentRPCandHttpPortsAreOK()
{
Configuration conf = new HdfsConfiguration();
FilePath nameDir = new FilePath(MiniDFSCluster.GetBaseDirectory(), "name");
conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, nameDir.GetAbsolutePath());
Random rand = new Random();
// A few retries in case the ports we choose are in use.
for (int i = 0; i < 5; ++i)
{
int port1 = 30000 + rand.Next(10000);
int port2 = port1 + 1 + rand.Next(10000);
FileSystem.SetDefaultUri(conf, "hdfs://localhost:" + port1);
conf.Set(DFSConfigKeys.DfsNamenodeHttpAddressKey, "127.0.0.1:" + port2);
DFSTestUtil.FormatNameNode(conf);
NameNode nameNode = null;
try
{
nameNode = new NameNode(conf);
// should be OK!
break;
}
catch (BindException)
{
continue;
}
finally
{
// Port in use? Try another.
if (nameNode != null)
{
nameNode.Stop();
}
}
}
}
public void Test_InternalNodeCanBeSharedBetweenDifferentTrees()
{
NameNode firstName = new NameNode("Kallyn");
MockSyntaxTree tree = new MockSyntaxTree(
new FullNameNode(
firstName,
new NameNode("Gowdy")
)
);
Assert.Equal("Kallyn", tree.FullName.FirstName.ToString());
MockSyntaxTree newTree = tree.SetRoot(
new FullNameNode(
firstName,
new NameNode("G")
)
) as MockSyntaxTree;
Assert.NotNull(newTree);
Assert.NotSame(tree, newTree);
Assert.NotSame(tree.Root, newTree.Root);
Assert.NotSame(tree.Root.FirstName, firstName);
Assert.NotSame(newTree.Root.FirstName, firstName);
Assert.NotSame(tree.Root.FirstName, newTree.Root.FirstName);
Assert.NotEqual(tree, newTree);
Assert.NotEqual(tree.Root, newTree.Root);
Assert.Equal(tree.Root.FirstName, firstName);
Assert.Equal(newTree.Root.FirstName, firstName);
Assert.Equal(tree.Root.FirstName, newTree.Root.FirstName);
Assert.Same(tree.Root.FirstName.InternalNode, firstName.InternalNode);
}
public virtual void SetUp()
{
conf = new Configuration();
// Stall the standby checkpointer in two ways
conf.SetLong(DFSConfigKeys.DfsNamenodeCheckpointPeriodKey, long.MaxValue);
conf.SetLong(DFSConfigKeys.DfsNamenodeCheckpointTxnsKey, 20);
// Make it autoroll after 10 edits
conf.SetFloat(DFSConfigKeys.DfsNamenodeEditLogAutorollMultiplierThreshold, 0.5f);
conf.SetInt(DFSConfigKeys.DfsNamenodeEditLogAutorollCheckIntervalMs, 100);
int retryCount = 0;
while (true)
{
try
{
int basePort = 10060 + random.Next(100) * 2;
MiniDFSNNTopology topology = new MiniDFSNNTopology().AddNameservice(new MiniDFSNNTopology.NSConf
("ns1").AddNN(new MiniDFSNNTopology.NNConf("nn1").SetHttpPort(basePort)).AddNN(new
MiniDFSNNTopology.NNConf("nn2").SetHttpPort(basePort + 1)));
cluster = new MiniDFSCluster.Builder(conf).NnTopology(topology).NumDataNodes(0).Build
();
cluster.WaitActive();
nn0 = cluster.GetNameNode(0);
fs = HATestUtil.ConfigureFailoverFs(cluster, conf);
cluster.TransitionToActive(0);
fs = cluster.GetFileSystem(0);
editLog = nn0.GetNamesystem().GetEditLog();
++retryCount;
break;
}
catch (BindException)
{
Log.Info("Set up MiniDFSCluster failed due to port conflicts, retry " + retryCount
+ " times");
}
}
}
private static void IntegrateFields(
ObjectTypeDefinitionNode rootType,
ITypeInfo typeInfo,
ISet <string> names,
ICollection <FieldDefinitionNode> fields)
{
string schemaName = typeInfo.Schema.Name;
foreach (FieldDefinitionNode field in rootType.Fields)
{
FieldDefinitionNode current = field;
if (names.Add(current.Name.Value))
{
current = current.AddDelegationPath(schemaName);
}
else
{
var path = new SelectionPathComponent(
field.Name,
field.Arguments.Select(t => new ArgumentNode(
t.Name,
new ScopedVariableNode(
null,
new NameNode(ScopeNames.Arguments),
t.Name))).ToList());
var newName = new NameNode(
typeInfo.CreateUniqueName(current));
current = current.WithName(newName)
.AddDelegationPath(schemaName, path);
}
fields.Add(current);
}
}
public virtual void TestNameNodeMultipleSwitchesUsingBKJM()
{
MiniDFSCluster cluster = null;
try
{
Configuration conf = new Configuration();
conf.SetInt(DFSConfigKeys.DfsHaTaileditsPeriodKey, 1);
conf.Set(DFSConfigKeys.DfsNamenodeSharedEditsDirKey, BKJMUtil.CreateJournalURI("/correctEditLogSelection"
).ToString());
BKJMUtil.AddJournalManagerDefinition(conf);
cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology.SimpleHATopology
()).NumDataNodes(0).ManageNameDfsSharedDirs(false).Build();
NameNode nn1 = cluster.GetNameNode(0);
NameNode nn2 = cluster.GetNameNode(1);
cluster.WaitActive();
cluster.TransitionToActive(0);
nn1.GetRpcServer().RollEditLog();
// Roll Edits from current Active.
// Transition to standby current active gracefully.
cluster.TransitionToStandby(0);
// Make the other Active and Roll edits multiple times
cluster.TransitionToActive(1);
nn2.GetRpcServer().RollEditLog();
nn2.GetRpcServer().RollEditLog();
// Now One more failover. So NN1 should be able to failover successfully.
cluster.TransitionToStandby(1);
cluster.TransitionToActive(0);
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <summary>
/// Test cancellation of ongoing checkpoints when failover happens
/// mid-checkpoint.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestCheckpointCancellation()
{
cluster.TransitionToStandby(0);
// Create an edit log in the shared edits dir with a lot
// of mkdirs operations. This is solely so that the image is
// large enough to take a non-trivial amount of time to load.
// (only ~15MB)
URI sharedUri = cluster.GetSharedEditsDir(0, 1);
FilePath sharedDir = new FilePath(sharedUri.GetPath(), "current");
FilePath tmpDir = new FilePath(MiniDFSCluster.GetBaseDirectory(), "testCheckpointCancellation-tmp"
);
FSNamesystem fsn = cluster.GetNamesystem(0);
FSImageTestUtil.CreateAbortedLogWithMkdirs(tmpDir, NumDirsInLog, 3, fsn.GetFSDirectory
().GetLastInodeId() + 1);
string fname = NNStorage.GetInProgressEditsFileName(3);
new FilePath(tmpDir, fname).RenameTo(new FilePath(sharedDir, fname));
// Checkpoint as fast as we can, in a tight loop.
cluster.GetConfiguration(1).SetInt(DFSConfigKeys.DfsNamenodeCheckpointPeriodKey,
0);
cluster.RestartNameNode(1);
nn1 = cluster.GetNameNode(1);
cluster.TransitionToActive(0);
bool canceledOne = false;
for (int i = 0; i < 10 && !canceledOne; i++)
{
DoEdits(i * 10, i * 10 + 10);
cluster.TransitionToStandby(0);
cluster.TransitionToActive(1);
cluster.TransitionToStandby(1);
cluster.TransitionToActive(0);
canceledOne = StandbyCheckpointer.GetCanceledCount() > 0;
}
NUnit.Framework.Assert.IsTrue(canceledOne);
}
public static void SetupCluster()
{
Configuration conf = new HdfsConfiguration();
string[] racks = new string[] { "/rack1", "/rack1", "/rack1", "/rack2", "/rack2",
"/rack2" };
storages = DFSTestUtil.CreateDatanodeStorageInfos(racks);
dataNodes = DFSTestUtil.ToDatanodeDescriptor(storages);
FileSystem.SetDefaultUri(conf, "hdfs://localhost:0");
conf.Set(DFSConfigKeys.DfsNamenodeHttpAddressKey, "0.0.0.0:0");
FilePath baseDir = PathUtils.GetTestDir(typeof(TestReplicationPolicy));
conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, new FilePath(baseDir, "name").GetPath
());
conf.SetBoolean(DFSConfigKeys.DfsNamenodeAvoidStaleDatanodeForReadKey, true);
conf.SetBoolean(DFSConfigKeys.DfsNamenodeAvoidStaleDatanodeForWriteKey, true);
conf.SetBoolean(DFSConfigKeys.DfsNamenodeReplicationConsiderloadKey, true);
DFSTestUtil.FormatNameNode(conf);
namenode = new NameNode(conf);
int blockSize = 1024;
dnrList = new AList <DatanodeRegistration>();
dnManager = namenode.GetNamesystem().GetBlockManager().GetDatanodeManager();
// Register DNs
for (int i = 0; i < 6; i++)
{
DatanodeRegistration dnr = new DatanodeRegistration(dataNodes[i], new StorageInfo
(HdfsServerConstants.NodeType.DataNode), new ExportedBlockKeys(), VersionInfo.GetVersion
());
dnrList.AddItem(dnr);
dnManager.RegisterDatanode(dnr);
dataNodes[i].GetStorageInfos()[0].SetUtilizationForTesting(2 * HdfsConstants.MinBlocksForWrite
* blockSize, 0L, 2 * HdfsConstants.MinBlocksForWrite * blockSize, 0L);
dataNodes[i].UpdateHeartbeat(BlockManagerTestUtil.GetStorageReportsForDatanode(dataNodes
[i]), 0L, 0L, 0, 0, null);
}
}
public void Test_AddNodeCreatesNewFacadeTree()
{
MockSyntaxTree tree = new MockSyntaxTree(
new FullNameNode(
new NameNode("Kallyn"),
new NameNode("Gowdy")
)
);
Assert.Equal(2, tree.FullName.Children.Count);
NameNode otherLastName = new NameNode("Other");
FullNameNode fullName = (FullNameNode)tree.FullName.AddNode(otherLastName);
Assert.Collection(
fullName.Children,
n => Assert.Equal(new NameNode("Kallyn"), n),
n => Assert.Equal(new NameNode("Gowdy"), n),
n => Assert.Equal(new NameNode("Other"), n)
);
Assert.Collection(
tree.FullName.Children,
n => Assert.Same(n.InternalNode, fullName.FirstName.InternalNode),
n => Assert.Same(n.InternalNode, fullName.Children[1].InternalNode) // Last name gets changed by AddNode
);
Assert.Collection(
tree.FullName.Children,
n => Assert.NotSame(n, fullName.FirstName),
n => Assert.NotSame(n, fullName.Children[1])
);
Assert.Equal(16, fullName.Width);
Assert.Equal("{Kallyn Gowdy Other}", fullName.ToString());
}
/// <summary>Verify datanode port usage.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestDataNodePorts()
{
NameNode nn = null;
try
{
nn = StartNameNode();
// start data-node on the same port as name-node
Configuration conf2 = new HdfsConfiguration(config);
conf2.Set(DFSConfigKeys.DfsDatanodeDataDirKey, new FilePath(hdfsDir, "data").GetPath
());
conf2.Set(DFSConfigKeys.DfsDatanodeAddressKey, FileSystem.GetDefaultUri(config).GetAuthority
());
conf2.Set(DFSConfigKeys.DfsDatanodeHttpAddressKey, ThisHost);
bool started = CanStartDataNode(conf2);
NUnit.Framework.Assert.IsFalse(started);
// should fail
// bind http server to the same port as name-node
conf2.Set(DFSConfigKeys.DfsDatanodeAddressKey, ThisHost);
conf2.Set(DFSConfigKeys.DfsDatanodeHttpAddressKey, config.Get(DFSConfigKeys.DfsNamenodeHttpAddressKey
));
started = CanStartDataNode(conf2);
NUnit.Framework.Assert.IsFalse(started);
// should fail
// both ports are different from the name-node ones
conf2.Set(DFSConfigKeys.DfsDatanodeAddressKey, ThisHost);
conf2.Set(DFSConfigKeys.DfsDatanodeHttpAddressKey, ThisHost);
conf2.Set(DFSConfigKeys.DfsDatanodeIpcAddressKey, ThisHost);
started = CanStartDataNode(conf2);
NUnit.Framework.Assert.IsTrue(started);
}
finally
{
// should start now
StopNameNode(nn);
}
}
/// <exception cref="System.Exception"/>
private static void TestFailoverAfterCrashDuringLogRoll(bool writeHeader)
{
Configuration conf = new Configuration();
conf.SetInt(DFSConfigKeys.DfsHaTaileditsPeriodKey, int.MaxValue);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology
.SimpleHATopology()).NumDataNodes(0).Build();
FileSystem fs = HATestUtil.ConfigureFailoverFs(cluster, conf);
try
{
cluster.TransitionToActive(0);
NameNode nn0 = cluster.GetNameNode(0);
nn0.GetRpcServer().RollEditLog();
cluster.ShutdownNameNode(0);
CreateEmptyInProgressEditLog(cluster, nn0, writeHeader);
cluster.TransitionToActive(1);
}
finally
{
IOUtils.Cleanup(Log, fs);
cluster.Shutdown();
}
}
/// <exception cref="System.Exception"/>
public virtual void TestSaveWhileEditsRolled()
{
Configuration conf = GetConf();
NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode);
DFSTestUtil.FormatNameNode(conf);
FSNamesystem fsn = FSNamesystem.LoadFromDisk(conf);
try
{
DoAnEdit(fsn, 1);
CheckpointSignature sig = fsn.RollEditLog();
Log.Warn("Checkpoint signature: " + sig);
// Do another edit
DoAnEdit(fsn, 2);
// Save namespace
fsn.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
fsn.SaveNamespace();
// Now shut down and restart the NN
fsn.Close();
fsn = null;
// Start a new namesystem, which should be able to recover
// the namespace from the previous incarnation.
fsn = FSNamesystem.LoadFromDisk(conf);
// Make sure the image loaded including our edits.
CheckEditExists(fsn, 1);
CheckEditExists(fsn, 2);
}
finally
{
if (fsn != null)
{
fsn.Close();
}
}
}
/// <exception cref="System.IO.IOException"/>
private NamespaceInfo Handshake(Configuration conf)
{
// connect to name node
IPEndPoint nnAddress = NameNode.GetServiceAddress(conf, true);
this.namenode = NameNodeProxies.CreateNonHAProxy <NamenodeProtocol>(conf, nnAddress
, UserGroupInformation.GetCurrentUser(), true).GetProxy();
this.nnRpcAddress = NetUtils.GetHostPortString(nnAddress);
this.nnHttpAddress = DFSUtil.GetInfoServer(nnAddress, conf, DFSUtil.GetHttpClientScheme
(conf)).ToURL();
// get version and id info from the name-node
NamespaceInfo nsInfo = null;
while (!IsStopRequested())
{
try
{
nsInfo = Handshake(namenode);
break;
}
catch (SocketTimeoutException e)
{
// name-node is busy
Log.Info("Problem connecting to server: " + nnAddress);
try
{
Sharpen.Thread.Sleep(1000);
}
catch (Exception)
{
Log.Warn("Encountered exception ", e);
}
}
}
return(nsInfo);
}
/*
* Try to read the files inside snapshot but deleted in original place after
* restarting post checkpoint. refer HDFS-5427
*/
/// <exception cref="System.Exception"/>
public virtual void TestReadSnapshotFileWithCheckpoint()
{
Path foo = new Path("/foo");
hdfs.Mkdirs(foo);
hdfs.AllowSnapshot(foo);
Path bar = new Path("/foo/bar");
DFSTestUtil.CreateFile(hdfs, bar, 100, (short)2, 100024L);
hdfs.CreateSnapshot(foo, "s1");
NUnit.Framework.Assert.IsTrue(hdfs.Delete(bar, true));
// checkpoint
NameNode nameNode = cluster.GetNameNode();
NameNodeAdapter.EnterSafeMode(nameNode, false);
NameNodeAdapter.SaveNamespace(nameNode);
NameNodeAdapter.LeaveSafeMode(nameNode);
// restart namenode to load snapshot files from fsimage
cluster.RestartNameNode(true);
string snapshotPath = Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.GetSnapshotPath
(foo.ToString(), "s1/bar");
DFSTestUtil.ReadFile(hdfs, new Path(snapshotPath));
}
public virtual void TestNoPopulatingReplQueuesWhenExitingSafemode()
{
DFSTestUtil.CreateFile(fs, new Path("/test"), 15 * BlockSize, (short)3, 1L);
HATestUtil.WaitForStandbyToCatchUp(nn0, nn1);
// get some blocks in the SBN's image
nn1.GetRpcServer().SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter, false);
NameNodeAdapter.SaveNamespace(nn1);
nn1.GetRpcServer().SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave, false);
// and some blocks in the edit logs
DFSTestUtil.CreateFile(fs, new Path("/test2"), 15 * BlockSize, (short)3, 1L);
nn0.GetRpcServer().RollEditLog();
cluster.StopDataNode(1);
cluster.ShutdownNameNode(1);
//Configuration sbConf = cluster.getConfiguration(1);
//sbConf.setInt(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, 1);
cluster.RestartNameNode(1, false);
nn1 = cluster.GetNameNode(1);
GenericTestUtils.WaitFor(new _Supplier_708(this), 100, 10000);
BlockManagerTestUtil.UpdateState(nn1.GetNamesystem().GetBlockManager());
NUnit.Framework.Assert.AreEqual(0L, nn1.GetNamesystem().GetUnderReplicatedBlocks(
));
NUnit.Framework.Assert.AreEqual(0L, nn1.GetNamesystem().GetPendingReplicationBlocks
());
}
protected override ICollection <Object> GetFilteredTargets()
{
string[] paths = mSearchName.Split('|');
List <NameNode> nameNodeList = new List <NameNode>();
foreach (string path in paths)
{
if (!string.IsNullOrEmpty(path))
{
nameNodeList.Add(NameNode.Parse(path));
}
}
List <Object> list = new List <Object>();
foreach (GameObject go in Selection.GetFiltered(typeof(GameObject), SelectionMode.Deep))
{
if (Contains(nameNodeList, go))
{
list.Add(go);
}
}
return(list);
}
/// <summary>
/// Make sure the WebHdfsFileSystem will retry based on RetriableException when
/// rpcServer is null in NamenodeWebHdfsMethods while NameNode starts up.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestRetryWhileNNStartup()
{
Configuration conf = DFSTestUtil.NewHAConfiguration(LogicalName);
MiniDFSCluster cluster = null;
IDictionary <string, bool> resultMap = new Dictionary <string, bool>();
try
{
cluster = new MiniDFSCluster.Builder(conf).NnTopology(topo).NumDataNodes(0).Build
();
HATestUtil.SetFailoverConfigurations(cluster, conf, LogicalName);
cluster.WaitActive();
cluster.TransitionToActive(0);
NameNode namenode = cluster.GetNameNode(0);
NamenodeProtocols rpcServer = namenode.GetRpcServer();
Whitebox.SetInternalState(namenode, "rpcServer", null);
new _Thread_212(this, conf, resultMap).Start();
Sharpen.Thread.Sleep(1000);
Whitebox.SetInternalState(namenode, "rpcServer", rpcServer);
lock (this)
{
while (!resultMap.Contains("mkdirs"))
{
Sharpen.Runtime.Wait(this);
}
NUnit.Framework.Assert.IsTrue(resultMap["mkdirs"]);
}
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestConfModificationFederationAndHa()
{
HdfsConfiguration conf = new HdfsConfiguration();
string nsId = "ns1";
string nnId = "nn1";
conf.Set(DFSConfigKeys.DfsNameservices, nsId);
conf.Set(DFSConfigKeys.DfsNameserviceId, nsId);
conf.Set(DFSConfigKeys.DfsHaNamenodesKeyPrefix + "." + nsId, nnId);
// Set the nameservice specific keys with nameserviceId in the config key
foreach (string key in NameNode.NamenodeSpecificKeys)
{
// Note: value is same as the key
conf.Set(DFSUtil.AddKeySuffixes(key, nsId, nnId), key);
}
// Initialize generic keys from specific keys
NameNode.InitializeGenericKeys(conf, nsId, nnId);
// Retrieve the keys without nameserviceId and Ensure generic keys are set
// to the correct value
foreach (string key_1 in NameNode.NamenodeSpecificKeys)
{
NUnit.Framework.Assert.AreEqual(key_1, conf.Get(key_1));
}
}
public virtual void SetupCluster()
{
conf = new Configuration();
conf.SetInt(DFSConfigKeys.DfsBlockSizeKey, SmallBlock);
// Bump up replication interval so that we only run replication
// checks explicitly.
conf.SetInt(DFSConfigKeys.DfsNamenodeReplicationIntervalKey, 600);
// Increase max streams so that we re-replicate quickly.
conf.SetInt(DFSConfigKeys.DfsNamenodeReplicationMaxStreamsKey, 1000);
// See RandomDeleterPolicy javadoc.
conf.SetClass(DFSConfigKeys.DfsBlockReplicatorClassnameKey, typeof(TestDNFencing.RandomDeleterPolicy
), typeof(BlockPlacementPolicy));
conf.SetInt(DFSConfigKeys.DfsHaTaileditsPeriodKey, 1);
cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology.SimpleHATopology
()).NumDataNodes(3).Build();
nn1 = cluster.GetNameNode(0);
nn2 = cluster.GetNameNode(1);
cluster.WaitActive();
cluster.TransitionToActive(0);
// Trigger block reports so that the first NN trusts all
// of the DNs, and will issue deletions
cluster.TriggerBlockReports();
fs = HATestUtil.ConfigureFailoverFs(cluster, conf);
}
/// <exception cref="System.Exception"/>
public virtual void TestTxIdPersistence()
{
Configuration conf = GetConf();
NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode);
DFSTestUtil.FormatNameNode(conf);
FSNamesystem fsn = FSNamesystem.LoadFromDisk(conf);
try
{
// We have a BEGIN_LOG_SEGMENT txn to start
NUnit.Framework.Assert.AreEqual(1, fsn.GetEditLog().GetLastWrittenTxId());
DoAnEdit(fsn, 1);
NUnit.Framework.Assert.AreEqual(2, fsn.GetEditLog().GetLastWrittenTxId());
fsn.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
fsn.SaveNamespace();
// 2 more txns: END the first segment, BEGIN a new one
NUnit.Framework.Assert.AreEqual(4, fsn.GetEditLog().GetLastWrittenTxId());
// Shut down and restart
fsn.GetFSImage().Close();
fsn.Close();
// 1 more txn to END that segment
NUnit.Framework.Assert.AreEqual(5, fsn.GetEditLog().GetLastWrittenTxId());
fsn = null;
fsn = FSNamesystem.LoadFromDisk(conf);
// 1 more txn to start new segment on restart
NUnit.Framework.Assert.AreEqual(6, fsn.GetEditLog().GetLastWrittenTxId());
}
finally
{
if (fsn != null)
{
fsn.Close();
}
}
}
/// <exception cref="System.IO.IOException"/>
private static INodesInPath CreateSingleDirectory(FSDirectory fsd, INodesInPath existing
, string localName, PermissionStatus perm)
{
System.Diagnostics.Debug.Assert(fsd.HasWriteLock());
existing = UnprotectedMkdir(fsd, fsd.AllocateNewInodeId(), existing, Sharpen.Runtime.GetBytesForString
(localName, Charsets.Utf8), perm, null, Time.Now());
if (existing == null)
{
return(null);
}
INode newNode = existing.GetLastINode();
// Directory creation also count towards FilesCreated
// to match count of FilesDeleted metric.
NameNode.GetNameNodeMetrics().IncrFilesCreated();
string cur = existing.GetPath();
fsd.GetEditLog().LogMkDir(cur, newNode);
if (NameNode.stateChangeLog.IsDebugEnabled())
{
NameNode.stateChangeLog.Debug("mkdirs: created directory " + cur);
}
return(existing);
}
/// <exception cref="Javax.Servlet.ServletException"/>
/// <exception cref="System.IO.IOException"/>
protected override void DoGet(HttpServletRequest request, HttpServletResponse response
)
{
try
{
ServletContext context = GetServletContext();
FSImage nnImage = NameNodeHttpServer.GetFsImageFromContext(context);
ImageServlet.GetImageParams parsedParams = new ImageServlet.GetImageParams(request
, response);
Configuration conf = (Configuration)context.GetAttribute(JspHelper.CurrentConf);
NameNodeMetrics metrics = NameNode.GetNameNodeMetrics();
ValidateRequest(context, conf, request, response, nnImage, parsedParams.GetStorageInfoString
());
UserGroupInformation.GetCurrentUser().DoAs(new _PrivilegedExceptionAction_98(parsedParams
, nnImage, metrics, response, conf));
}
catch (Exception t)
{
// Metrics non-null only when used inside name node
// Metrics non-null only when used inside name node
// Potential race where the file was deleted while we were in the
// process of setting headers!
// It's possible the file could be deleted after this point, but
// we've already opened the 'fis' stream.
// It's also possible length could change, but this would be
// detected by the client side as an inaccurate length header.
// send file
string errMsg = "GetImage failed. " + StringUtils.StringifyException(t);
response.SendError(HttpServletResponse.ScGone, errMsg);
throw new IOException(errMsg);
}
finally
{
response.GetOutputStream().Close();
}
}
public static Org.Apache.Hadoop.Hdfs.Tools.DFSZKFailoverController Create(Configuration
conf)
{
Configuration localNNConf = DFSHAAdmin.AddSecurityConfiguration(conf);
string nsId = DFSUtil.GetNamenodeNameServiceId(conf);
if (!HAUtil.IsHAEnabled(localNNConf, nsId))
{
throw new HadoopIllegalArgumentException("HA is not enabled for this namenode.");
}
string nnId = HAUtil.GetNameNodeId(localNNConf, nsId);
if (nnId == null)
{
string msg = "Could not get the namenode ID of this node. " + "You may run zkfc on the node other than namenode.";
throw new HadoopIllegalArgumentException(msg);
}
NameNode.InitializeGenericKeys(localNNConf, nsId, nnId);
DFSUtil.SetGenericConf(localNNConf, nsId, nnId, ZkfcConfKeys);
NNHAServiceTarget localTarget = new NNHAServiceTarget(localNNConf, nsId, nnId);
return(new Org.Apache.Hadoop.Hdfs.Tools.DFSZKFailoverController(localNNConf, localTarget
));
}
protected override FieldNode RewriteField(FieldNode node, bool first)
{
FieldNode current = node;
if (first)
{
NameNode responseName = node.Alias ?? node.Name;
NameNode alias = responseName.CreateNewName(_requestPrefix);
_aliases[alias.Value] = responseName.Value;
current = current.WithAlias(alias);
}
current = Rewrite(current, node.Arguments, first,
(p, c) => RewriteMany(p, c, RewriteArgument),
current.WithArguments);
if (node.SelectionSet != null)
{
current = Rewrite(current, node.SelectionSet, false,
RewriteSelectionSet, current.WithSelectionSet);
}
return(current);
}
public static List <GridViewColumn> GetColumnsForExpression(
GridViewTemplate template,
ExpressionNode expression,
out bool containsAggregate)
{
containsAggregate = false;
List <GridViewColumn> gridViewColumnList = new List <GridViewColumn>();
Stack <ExpressionNode> expressionNodeStack = new Stack <ExpressionNode>();
expressionNodeStack.Push(expression);
while (expressionNodeStack.Count > 0)
{
ExpressionNode expressionNode = expressionNodeStack.Pop();
if (expressionNode != null)
{
NameNode nameNode = expressionNode as NameNode;
if (nameNode != null && !nameNode.IsConst)
{
GridViewColumn column = (GridViewColumn)template.Columns[nameNode.Name];
if (column != null)
{
gridViewColumnList.Add(column);
}
}
else if (expressionNode is AggregateNode)
{
containsAggregate = true;
}
foreach (ExpressionNode childNode in expressionNode.GetChildNodes())
{
expressionNodeStack.Push(childNode);
}
}
}
return(gridViewColumnList);
}
/// <summary>Private helper methods to save delegation keys and tokens in fsimage</summary>
/// <exception cref="System.IO.IOException"/>
private void SaveCurrentTokens(DataOutputStream @out, string sdPath)
{
lock (this)
{
StartupProgress prog = NameNode.GetStartupProgress();
Step step = new Step(StepType.DelegationTokens, sdPath);
prog.BeginStep(Phase.SavingCheckpoint, step);
prog.SetTotal(Phase.SavingCheckpoint, step, this._enclosing.currentTokens.Count);
StartupProgress.Counter counter = prog.GetCounter(Phase.SavingCheckpoint, step);
@out.WriteInt(this._enclosing.currentTokens.Count);
IEnumerator <DelegationTokenIdentifier> iter = this._enclosing.currentTokens.Keys.
GetEnumerator();
while (iter.HasNext())
{
DelegationTokenIdentifier id = iter.Next();
id.Write(@out);
AbstractDelegationTokenSecretManager.DelegationTokenInformation info = this._enclosing
.currentTokens[id];
@out.WriteLong(info.GetRenewDate());
counter.Increment();
}
prog.EndStep(Phase.SavingCheckpoint, step);
}
}
public NameNode AddTypeSpecification(TypeSpecification typeSpecification)
{
typeSpecifications = typeSpecifications ?? new Dictionary <int, NameNode>();
int specificationNumber = (int)typeSpecification.MetadataType;
if (typeSpecification.IsArray)
{
var arrayType = typeSpecification as ArrayType;
if (arrayType.Rank > 1)
{
specificationNumber = 666;
}
}
NameNode result;
if (!typeSpecifications.TryGetValue(specificationNumber, out result))
{
result = new NameNode();
typeSpecifications.Add(specificationNumber, result);
}
return(result);
}
protected override void ValidateName(NameNode node, string tokenString)
{
}
public NamedType(NameNode node)
: this(node.Name)
{
NameNode = node;
}
public Alias(NameNode alias, NameNode name)
{
Al = alias;
Name = name;
}
