/// <summary>
/// Check if the path represents a directory as determined by the basename
/// being "." or "..", or the path ending with a directory separator
/// </summary>
/// <returns>boolean if this represents a directory</returns>
public virtual bool RepresentsDirectory()
{
string uriPath = uri.GetPath();
string name = Runtime.Substring(uriPath, uriPath.LastIndexOf("/") + 1);
// Path will munch off the chars that indicate a dir, so there's no way
// to perform this test except by examining the raw basename we maintain
return(name.IsEmpty() || name.Equals(".") || name.Equals(".."));
}
public virtual void TestURI()
{
Log.Info("Testing correct Unix URI: " + UriUnix);
URI u = Util.StringAsURI(UriUnix);
Log.Info("Uri: " + u);
NUnit.Framework.Assert.IsNotNull("Uri should not be null at this point", u);
NUnit.Framework.Assert.AreEqual(UriFileSchema, u.GetScheme());
NUnit.Framework.Assert.AreEqual(UriPathUnix, u.GetPath());
Log.Info("Testing correct windows URI: " + UriWindows);
u = Util.StringAsURI(UriWindows);
Log.Info("Uri: " + u);
NUnit.Framework.Assert.IsNotNull("Uri should not be null at this point", u);
NUnit.Framework.Assert.AreEqual(UriFileSchema, u.GetScheme());
NUnit.Framework.Assert.AreEqual(UriPathWindows.Replace("%20", " "), u.GetPath());
}
public virtual void TestPositiveHarFileSystemBasics()
{
// check Har version:
Assert.Equal(HarFileSystem.Version, harFileSystem.GetHarVersion
());
// check Har URI:
URI harUri = harFileSystem.GetUri();
Assert.Equal(harPath.ToUri().GetPath(), harUri.GetPath());
Assert.Equal("har", harUri.GetScheme());
// check Har home path:
Path homePath = harFileSystem.GetHomeDirectory();
Assert.Equal(harPath.ToUri().GetPath(), homePath.ToUri().GetPath
());
// check working directory:
Path workDirPath0 = harFileSystem.GetWorkingDirectory();
Assert.Equal(homePath, workDirPath0);
// check that its impossible to reset the working directory
// (#setWorkingDirectory should have no effect):
harFileSystem.SetWorkingDirectory(new Path("/foo/bar"));
Assert.Equal(workDirPath0, harFileSystem.GetWorkingDirectory()
);
}
private string AppendOrReplaceParamter(string uri, string newQuery)
{
if (uri.Contains(YarnWebParams.NextRefreshInterval + "="))
{
return(uri.ReplaceAll(YarnWebParams.NextRefreshInterval + "=[^&]+", newQuery));
}
try
{
URI oldUri = new URI(uri);
string appendQuery = oldUri.GetQuery();
if (appendQuery == null)
{
appendQuery = newQuery;
}
else
{
appendQuery += "&" + newQuery;
}
URI newUri = new URI(oldUri.GetScheme(), oldUri.GetAuthority(), oldUri.GetPath(),
appendQuery, oldUri.GetFragment());
return(newUri.ToString());
}
catch (URISyntaxException)
{
return(null);
}
}
private static string Relativize(URI cwdUri, URI srcUri, bool isDir)
{
string uriPath = srcUri.GetPath();
string cwdPath = cwdUri.GetPath();
if (cwdPath.Equals(uriPath))
{
return(Path.CurDir);
}
// find common ancestor
int lastSep = FindLongestDirPrefix(cwdPath, uriPath, isDir);
StringBuilder relPath = new StringBuilder();
// take the remaining path fragment after the ancestor
if (lastSep < uriPath.Length)
{
relPath.Append(Runtime.Substring(uriPath, lastSep + 1));
}
// if cwd has a path fragment after the ancestor, convert them to ".."
if (lastSep < cwdPath.Length)
{
while (lastSep != -1)
{
if (relPath.Length != 0)
{
relPath.Insert(0, Path.Separator);
}
relPath.Insert(0, "..");
lastSep = cwdPath.IndexOf(Path.Separator, lastSep + 1);
}
}
return(relPath.ToString());
}
/// <summary>Resolve the uri's hostname and add the default port if not in the uri</summary>
/// <param name="uri">to resolve</param>
/// <param name="defaultPort">if none is given</param>
/// <returns>URI</returns>
public static URI GetCanonicalUri(URI uri, int defaultPort)
{
// skip if there is no authority, ie. "file" scheme or relative uri
string host = uri.GetHost();
if (host == null)
{
return(uri);
}
string fqHost = CanonicalizeHost(host);
int port = uri.GetPort();
// short out if already canonical with a port
if (host.Equals(fqHost) && port != -1)
{
return(uri);
}
// reconstruct the uri with the canonical host and port
try
{
uri = new URI(uri.GetScheme(), uri.GetUserInfo(), fqHost, (port == -1) ? defaultPort
: port, uri.GetPath(), uri.GetQuery(), uri.GetFragment());
}
catch (URISyntaxException e)
{
throw new ArgumentException(e);
}
return(uri);
}
public static string[] ValidatePaths(string[] paths)
{
AList <string> validPaths = new AList <string>();
for (int i = 0; i < paths.Length; ++i)
{
try
{
URI uriPath = (new Path(paths[i])).ToUri();
if (uriPath.GetScheme() == null || uriPath.GetScheme().Equals(FileScheme))
{
validPaths.AddItem(new Path(uriPath.GetPath()).ToString());
}
else
{
Log.Warn(paths[i] + " is not a valid path. Path should be with " + FileScheme + " scheme or without scheme"
);
throw new YarnRuntimeException(paths[i] + " is not a valid path. Path should be with "
+ FileScheme + " scheme or without scheme");
}
}
catch (ArgumentException e)
{
Log.Warn(e.Message);
throw new YarnRuntimeException(paths[i] + " is not a valid path. Path should be with "
+ FileScheme + " scheme or without scheme");
}
}
string[] arrValidPaths = new string[validPaths.Count];
Sharpen.Collections.ToArray(validPaths, arrValidPaths);
return(arrValidPaths);
}
public virtual void TestFileUrls()
{
// URLStreamHandler is already set in JVM by testDfsUrls()
Configuration conf = new HdfsConfiguration();
// Locate the test temporary directory.
if (!TestRootDir.Exists())
{
if (!TestRootDir.Mkdirs())
{
throw new IOException("Cannot create temporary directory: " + TestRootDir);
}
}
FilePath tmpFile = new FilePath(TestRootDir, "thefile");
URI uri = tmpFile.ToURI();
FileSystem fs = FileSystem.Get(uri, conf);
try
{
byte[] fileContent = new byte[1024];
for (int i = 0; i < fileContent.Length; ++i)
{
fileContent[i] = unchecked ((byte)i);
}
// First create the file through the FileSystem API
OutputStream os = fs.Create(new Path(uri.GetPath()));
os.Write(fileContent);
os.Close();
// Second, open and read the file content through the URL API.
Uri fileURL = uri.ToURL();
InputStream @is = fileURL.OpenStream();
NUnit.Framework.Assert.IsNotNull(@is);
byte[] bytes = new byte[4096];
NUnit.Framework.Assert.AreEqual(1024, @is.Read(bytes));
@is.Close();
for (int i_1 = 0; i_1 < fileContent.Length; ++i_1)
{
NUnit.Framework.Assert.AreEqual(fileContent[i_1], bytes[i_1]);
}
// Cleanup: delete the file
fs.Delete(new Path(uri.GetPath()), false);
}
finally
{
fs.Close();
}
}
/// <summary>Construct a Bookkeeper journal manager.</summary>
/// <exception cref="System.IO.IOException"/>
public BookKeeperJournalManager(Configuration conf, URI uri, NamespaceInfo nsInfo
)
{
this.conf = conf;
this.nsInfo = nsInfo;
string zkConnect = uri.GetAuthority().Replace(";", ",");
basePath = uri.GetPath();
ensembleSize = conf.GetInt(BkjmBookkeeperEnsembleSize, BkjmBookkeeperEnsembleSizeDefault
);
quorumSize = conf.GetInt(BkjmBookkeeperQuorumSize, BkjmBookkeeperQuorumSizeDefault
);
ackQuorumSize = conf.GetInt(BkjmBookkeeperAckQuorumSize, quorumSize);
addEntryTimeout = conf.GetInt(BkjmBookkeeperAddEntryTimeoutSec, BkjmBookkeeperAddEntryTimeoutDefault
);
speculativeReadTimeout = conf.GetInt(BkjmBookkeeperSpeculativeReadTimeoutMs, BkjmBookkeeperSpeculativeReadTimeoutDefault
);
readEntryTimeout = conf.GetInt(BkjmBookkeeperReadEntryTimeoutSec, BkjmBookkeeperReadEntryTimeoutDefault
);
ledgerPath = basePath + "/ledgers";
string maxTxIdPath = basePath + "/maxtxid";
string currentInprogressNodePath = basePath + "/CurrentInprogress";
versionPath = basePath + "/version";
digestpw = conf.Get(BkjmBookkeeperDigestPw, BkjmBookkeeperDigestPwDefault);
try
{
zkConnectLatch = new CountDownLatch(1);
int bkjmZKSessionTimeout = conf.GetInt(BkjmZkSessionTimeout, BkjmZkSessionTimeoutDefault
);
zkc = new ZooKeeper(zkConnect, bkjmZKSessionTimeout, new BookKeeperJournalManager.ZkConnectionWatcher
(this));
// Configured zk session timeout + some extra grace period (here
// BKJM_ZK_SESSION_TIMEOUT_DEFAULT used as grace period)
int zkConnectionLatchTimeout = bkjmZKSessionTimeout + BkjmZkSessionTimeoutDefault;
if (!zkConnectLatch.Await(zkConnectionLatchTimeout, TimeUnit.Milliseconds))
{
throw new IOException("Error connecting to zookeeper");
}
PrepareBookKeeperEnv();
ClientConfiguration clientConf = new ClientConfiguration();
clientConf.SetSpeculativeReadTimeout(speculativeReadTimeout);
clientConf.SetReadEntryTimeout(readEntryTimeout);
clientConf.SetAddEntryTimeout(addEntryTimeout);
bkc = new BookKeeper(clientConf, zkc);
}
catch (KeeperException e)
{
throw new IOException("Error initializing zk", e);
}
catch (Exception ie)
{
Sharpen.Thread.CurrentThread().Interrupt();
throw new IOException("Interrupted while initializing bk journal manager", ie);
}
ci = new CurrentInprogress(zkc, currentInprogressNodePath);
maxTxId = new MaxTxId(zkc, maxTxIdPath);
}
/// <exception cref="System.Exception"/>
private static void WaitForLogRollInSharedDir(MiniDFSCluster cluster, long startTxId
)
{
URI sharedUri = cluster.GetSharedEditsDir(0, 1);
FilePath sharedDir = new FilePath(sharedUri.GetPath(), "current");
FilePath expectedLog = new FilePath(sharedDir, NNStorage.GetInProgressEditsFileName
(startTxId));
GenericTestUtils.WaitFor(new _Supplier_153(expectedLog), 100, 10000);
}
internal static string ParseJournalId(URI uri)
{
string path = uri.GetPath();
Preconditions.CheckArgument(path != null && !path.IsEmpty(), "Bad URI '%s': must identify journal in path component"
, uri);
string journalId = Sharpen.Runtime.Substring(path, 1);
CheckJournalId(journalId);
return(journalId);
}
/// <summary>
/// Returns a boolean to denote whether a cache file is visible to all(public)
/// or not
/// </summary>
/// <param name="conf"/>
/// <param name="uri"/>
/// <returns>true if the path in the uri is visible to all, false otherwise</returns>
/// <exception cref="System.IO.IOException"/>
internal static bool IsPublic(Configuration conf, URI uri, IDictionary <URI, FileStatus
> statCache)
{
FileSystem fs = FileSystem.Get(uri, conf);
Path current = new Path(uri.GetPath());
//the leaf level file should be readable by others
if (!CheckPermissionOfOther(fs, current, FsAction.Read, statCache))
{
return(false);
}
return(AncestorsHaveExecutePermissions(fs, current.GetParent(), statCache));
}
private static URI RemoveAuthority(URI uri)
{
try
{
uri = new URI(uri.GetScheme(), string.Empty, uri.GetPath(), uri.GetQuery(), uri.GetFragment
());
}
catch (URISyntaxException e)
{
throw new ArgumentException(e.GetLocalizedMessage());
}
return(uri);
}
/// <exception cref="System.IO.IOException"/>
private static void AddMRFrameworkToDistributedCache(Configuration conf)
{
string framework = conf.Get(MRJobConfig.MapreduceApplicationFrameworkPath, string.Empty
);
if (!framework.IsEmpty())
{
URI uri;
try
{
uri = new URI(framework);
}
catch (URISyntaxException e)
{
throw new ArgumentException("Unable to parse '" + framework + "' as a URI, check the setting for "
+ MRJobConfig.MapreduceApplicationFrameworkPath, e);
}
string linkedName = uri.GetFragment();
// resolve any symlinks in the URI path so using a "current" symlink
// to point to a specific version shows the specific version
// in the distributed cache configuration
FileSystem fs = FileSystem.Get(conf);
Path frameworkPath = fs.MakeQualified(new Path(uri.GetScheme(), uri.GetAuthority(
), uri.GetPath()));
FileContext fc = FileContext.GetFileContext(frameworkPath.ToUri(), conf);
frameworkPath = fc.ResolvePath(frameworkPath);
uri = frameworkPath.ToUri();
try
{
uri = new URI(uri.GetScheme(), uri.GetAuthority(), uri.GetPath(), null, linkedName
);
}
catch (URISyntaxException e)
{
throw new ArgumentException(e);
}
DistributedCache.AddCacheArchive(uri, conf);
}
}
/// <summary>Probe for a path being a parent of another</summary>
/// <param name="parent">parent path</param>
/// <param name="child">possible child path</param>
/// <returns>true if the parent's path matches the start of the child's</returns>
private bool IsParentOf(Path parent, Path child)
{
URI parentURI = parent.ToUri();
string parentPath = parentURI.GetPath();
if (!parentPath.EndsWith("/"))
{
parentPath += "/";
}
URI childURI = child.ToUri();
string childPath = childURI.GetPath();
return(childPath.StartsWith(parentPath));
}
/// <summary>Get a proxied URI for the original URI.</summary>
/// <param name="originalUri">
/// the original URI to go through the proxy, or null if
/// a default path "/" can be used.
/// </param>
/// <param name="proxyUri">the URI of the proxy itself, scheme, host and port are used.
/// </param>
/// <param name="id">the id of the application</param>
/// <returns>the proxied URI</returns>
public static URI GetProxyUri(URI originalUri, URI proxyUri, ApplicationId id)
{
try
{
string path = GetPath(id, originalUri == null ? "/" : originalUri.GetPath());
return(new URI(proxyUri.GetScheme(), proxyUri.GetAuthority(), path, originalUri ==
null ? null : originalUri.GetQuery(), originalUri == null ? null : originalUri.
GetFragment()));
}
catch (URISyntaxException e)
{
throw new RuntimeException("Could not proxify " + originalUri, e);
}
}
private StorageLocation(StorageType storageType, URI uri)
{
this.storageType = storageType;
if (uri.GetScheme() == null || Sharpen.Runtime.EqualsIgnoreCase("file", uri.GetScheme
()))
{
// drop any (illegal) authority in the URI for backwards compatibility
this.file = new FilePath(uri.GetPath());
}
else
{
throw new ArgumentException("Unsupported URI schema in " + uri);
}
}
/// <summary>Constructor</summary>
/// <param name="uri">base file system</param>
/// <param name="conf">configuration</param>
/// <exception cref="System.IO.IOException"></exception>
public ChRootedFileSystem(URI uri, Configuration conf)
: base(FileSystem.Get(uri, conf))
{
string pathString = uri.GetPath();
if (pathString.IsEmpty())
{
pathString = "/";
}
chRootPathPart = new Path(pathString);
chRootPathPartString = chRootPathPart.ToUri().GetPath();
myUri = uri;
workingDir = GetHomeDirectory();
}
/// <exception cref="System.Exception"/>
private void TestFailoverFinalizesAndReadsInProgress(bool partialTxAtEnd)
{
Configuration conf = new Configuration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology
.SimpleHATopology()).NumDataNodes(0).Build();
try
{
// Create a fake in-progress edit-log in the shared directory
URI sharedUri = cluster.GetSharedEditsDir(0, 1);
FilePath sharedDir = new FilePath(sharedUri.GetPath(), "current");
FSNamesystem fsn = cluster.GetNamesystem(0);
FSImageTestUtil.CreateAbortedLogWithMkdirs(sharedDir, NumDirsInLog, 1, fsn.GetFSDirectory
().GetLastInodeId() + 1);
AssertEditFiles(Sharpen.Collections.SingletonList(sharedUri), NNStorage.GetInProgressEditsFileName
(1));
if (partialTxAtEnd)
{
FileOutputStream outs = null;
try
{
FilePath editLogFile = new FilePath(sharedDir, NNStorage.GetInProgressEditsFileName
(1));
outs = new FileOutputStream(editLogFile, true);
outs.Write(new byte[] { unchecked ((int)(0x18)), unchecked ((int)(0x00)), unchecked (
(int)(0x00)), unchecked ((int)(0x00)) });
Log.Error("editLogFile = " + editLogFile);
}
finally
{
IOUtils.Cleanup(Log, outs);
}
}
// Transition one of the NNs to active
cluster.TransitionToActive(0);
// In the transition to active, it should have read the log -- and
// hence see one of the dirs we made in the fake log.
string testPath = "/dir" + NumDirsInLog;
NUnit.Framework.Assert.IsNotNull(cluster.GetNameNode(0).GetRpcServer().GetFileInfo
(testPath));
// It also should have finalized that log in the shared directory and started
// writing to a new one at the next txid.
AssertEditFiles(Sharpen.Collections.SingletonList(sharedUri), NNStorage.GetFinalizedEditsFileName
(1, NumDirsInLog + 1), NNStorage.GetInProgressEditsFileName(NumDirsInLog + 2));
}
finally
{
cluster.Shutdown();
}
}
private void DigestURI(URI uri)
{
this.scheme = uri.GetScheme();
this.encodedSchemeSpecificPart = uri.GetRawSchemeSpecificPart();
this.encodedAuthority = uri.GetRawAuthority();
this.host = uri.GetHost();
this.port = uri.GetPort();
this.encodedUserInfo = uri.GetRawUserInfo();
this.userInfo = uri.GetUserInfo();
this.encodedPath = uri.GetRawPath();
this.path = uri.GetPath();
this.encodedQuery = uri.GetRawQuery();
this.queryParams = ParseQuery(uri.GetRawQuery(), Consts.Utf8);
this.encodedFragment = uri.GetRawFragment();
this.fragment = uri.GetFragment();
}
/// <summary>Add the volume of the passed-in directory to the list of volumes to check.
/// </summary>
/// <remarks>
/// Add the volume of the passed-in directory to the list of volumes to check.
/// If <code>required</code> is true, and this volume is already present, but
/// is marked redundant, it will be marked required. If the volume is already
/// present but marked required then this method is a no-op.
/// </remarks>
/// <param name="directoryToCheck">The directory whose volume will be checked for available space.
/// </param>
/// <exception cref="System.IO.IOException"/>
private void AddDirToCheck(URI directoryToCheck, bool required)
{
FilePath dir = new FilePath(directoryToCheck.GetPath());
if (!dir.Exists())
{
throw new IOException("Missing directory " + dir.GetAbsolutePath());
}
NameNodeResourceChecker.CheckedVolume newVolume = new NameNodeResourceChecker.CheckedVolume
(this, dir, required);
NameNodeResourceChecker.CheckedVolume volume = volumes[newVolume.GetVolume()];
if (volume == null || !volume.IsRequired())
{
volumes[newVolume.GetVolume()] = newVolume;
}
}
public static URL GetYarnUrlFromURI(URI uri)
{
URL url = RecordFactoryProvider.GetRecordFactory(null).NewRecordInstance <URL>();
if (uri.GetHost() != null)
{
url.SetHost(uri.GetHost());
}
if (uri.GetUserInfo() != null)
{
url.SetUserInfo(uri.GetUserInfo());
}
url.SetPort(uri.GetPort());
url.SetScheme(uri.GetScheme());
url.SetFile(uri.GetPath());
return(url);
}
/// <summary>Converts the passed File to a URI.</summary>
/// <remarks>
/// Converts the passed File to a URI. This method trims the trailing slash if
/// one is appended because the underlying file is in fact a directory that
/// exists.
/// </remarks>
/// <param name="f">the file to convert</param>
/// <returns>the resulting URI</returns>
/// <exception cref="System.IO.IOException"/>
public static URI FileAsURI(FilePath f)
{
URI u = f.GetCanonicalFile().ToURI();
// trim the trailing slash, if it's present
if (u.GetPath().EndsWith("/"))
{
string uriAsString = u.ToString();
try
{
u = new URI(Sharpen.Runtime.Substring(uriAsString, 0, uriAsString.Length - 1));
}
catch (URISyntaxException e)
{
throw new IOException(e);
}
}
return(u);
}
/// <summary>
/// Create an InetSocketAddress from the given target string and
/// default port.
/// </summary>
/// <remarks>
/// Create an InetSocketAddress from the given target string and
/// default port. If the string cannot be parsed correctly, the
/// <code>configName</code> parameter is used as part of the
/// exception message, allowing the user to better diagnose
/// the misconfiguration.
/// </remarks>
/// <param name="target">a string of either "host" or "host:port"</param>
/// <param name="defaultPort">
/// the default port if <code>target</code> does not
/// include a port number
/// </param>
/// <param name="configName">
/// the name of the configuration from which
/// <code>target</code> was loaded. This is used in the
/// exception message in the case that parsing fails.
/// </param>
public static IPEndPoint CreateSocketAddr(string target, int defaultPort, string
configName)
{
string helpText = string.Empty;
if (configName != null)
{
helpText = " (configuration property '" + configName + "')";
}
if (target == null)
{
throw new ArgumentException("Target address cannot be null." + helpText);
}
target = target.Trim();
bool hasScheme = target.Contains("://");
URI uri = null;
try
{
uri = hasScheme ? URI.Create(target) : URI.Create("dummyscheme://" + target);
}
catch (ArgumentException)
{
throw new ArgumentException("Does not contain a valid host:port authority: " + target
+ helpText);
}
string host = uri.GetHost();
int port = uri.GetPort();
if (port == -1)
{
port = defaultPort;
}
string path = uri.GetPath();
if ((host == null) || (port < 0) || (!hasScheme && path != null && !path.IsEmpty(
)))
{
throw new ArgumentException("Does not contain a valid host:port authority: " + target
+ helpText);
}
return(CreateSocketAddrForHost(host, port));
}
/// <summary>Returns a qualified path object.</summary>
public virtual Org.Apache.Hadoop.FS.Path MakeQualified(URI defaultUri, Org.Apache.Hadoop.FS.Path
workingDir)
{
Org.Apache.Hadoop.FS.Path path = this;
if (!IsAbsolute())
{
path = new Org.Apache.Hadoop.FS.Path(workingDir, this);
}
URI pathUri = path.ToUri();
string scheme = pathUri.GetScheme();
string authority = pathUri.GetAuthority();
string fragment = pathUri.GetFragment();
if (scheme != null && (authority != null || defaultUri.GetAuthority() == null))
{
return(path);
}
if (scheme == null)
{
scheme = defaultUri.GetScheme();
}
if (authority == null)
{
authority = defaultUri.GetAuthority();
if (authority == null)
{
authority = string.Empty;
}
}
URI newUri = null;
try
{
newUri = new URI(scheme, authority, NormalizePath(scheme, pathUri.GetPath()), null
, fragment);
}
catch (URISyntaxException e)
{
throw new ArgumentException(e);
}
return(new Org.Apache.Hadoop.FS.Path(newUri));
}
/// <exception cref="System.Exception"/>
internal static void RunTestCache(int port)
{
Configuration conf = new Configuration();
MiniDFSCluster cluster = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NameNodePort(port).NumDataNodes(2).Build
();
URI uri = cluster.GetFileSystem().GetUri();
Log.Info("uri=" + uri);
{
FileSystem fs = FileSystem.Get(uri, new Configuration());
CheckPath(cluster, fs);
for (int i = 0; i < 100; i++)
{
NUnit.Framework.Assert.IsTrue(fs == FileSystem.Get(uri, new Configuration()));
}
}
if (port == NameNode.DefaultPort)
{
//test explicit default port
URI uri2 = new URI(uri.GetScheme(), uri.GetUserInfo(), uri.GetHost(), NameNode.DefaultPort
, uri.GetPath(), uri.GetQuery(), uri.GetFragment());
Log.Info("uri2=" + uri2);
FileSystem fs = FileSystem.Get(uri2, conf);
CheckPath(cluster, fs);
for (int i = 0; i < 100; i++)
{
NUnit.Framework.Assert.IsTrue(fs == FileSystem.Get(uri2, new Configuration()));
}
}
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <summary>Convert a nested URI to decode the underlying path.</summary>
/// <remarks>
/// Convert a nested URI to decode the underlying path. The translation takes
/// the authority and parses it into the underlying scheme and authority.
/// For example, "myscheme://hdfs@nn/my/path" is converted to
/// "hdfs://nn/my/path".
/// </remarks>
/// <param name="nestedUri">the URI from the nested URI</param>
/// <returns>the unnested path</returns>
public static Path UnnestUri(URI nestedUri)
{
string[] parts = nestedUri.GetAuthority().Split("@", 2);
StringBuilder result = new StringBuilder(parts[0]);
result.Append("://");
if (parts.Length == 2)
{
result.Append(parts[1]);
}
result.Append(nestedUri.GetPath());
if (nestedUri.GetQuery() != null)
{
result.Append("?");
result.Append(nestedUri.GetQuery());
}
if (nestedUri.GetFragment() != null)
{
result.Append("#");
result.Append(nestedUri.GetFragment());
}
return(new Path(result.ToString()));
}
/// <summary>Resolve a child path against a parent path.</summary>
public Path(Org.Apache.Hadoop.FS.Path parent, Org.Apache.Hadoop.FS.Path child)
{
// Add a slash to parent's path so resolution is compatible with URI's
URI parentUri = parent.uri;
string parentPath = parentUri.GetPath();
if (!(parentPath.Equals("/") || parentPath.IsEmpty()))
{
try
{
parentUri = new URI(parentUri.GetScheme(), parentUri.GetAuthority(), parentUri.GetPath
() + "/", null, parentUri.GetFragment());
}
catch (URISyntaxException e)
{
throw new ArgumentException(e);
}
}
URI resolved = parentUri.Resolve(child.uri);
Initialize(resolved.GetScheme(), resolved.GetAuthority(), resolved.GetPath(), resolved
.GetFragment());
}
/// <exception cref="System.IO.IOException"/>
private static void CreateEmptyInProgressEditLog(MiniDFSCluster cluster, NameNode
nn, bool writeHeader)
{
long txid = nn.GetNamesystem().GetEditLog().GetLastWrittenTxId();
URI sharedEditsUri = cluster.GetSharedEditsDir(0, 1);
FilePath sharedEditsDir = new FilePath(sharedEditsUri.GetPath());
Storage.StorageDirectory storageDir = new Storage.StorageDirectory(sharedEditsDir
);
FilePath inProgressFile = NameNodeAdapter.GetInProgressEditsFile(storageDir, txid
+ 1);
NUnit.Framework.Assert.IsTrue("Failed to create in-progress edits file", inProgressFile
.CreateNewFile());
if (writeHeader)
{
DataOutputStream @out = new DataOutputStream(new FileOutputStream(inProgressFile)
);
EditLogFileOutputStream.WriteHeader(NameNodeLayoutVersion.CurrentLayoutVersion, @out
);
@out.Close();
}
}
/// <summary>
/// Test cancellation of ongoing checkpoints when failover happens
/// mid-checkpoint.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestCheckpointCancellation()
{
cluster.TransitionToStandby(0);
// Create an edit log in the shared edits dir with a lot
// of mkdirs operations. This is solely so that the image is
// large enough to take a non-trivial amount of time to load.
// (only ~15MB)
URI sharedUri = cluster.GetSharedEditsDir(0, 1);
FilePath sharedDir = new FilePath(sharedUri.GetPath(), "current");
FilePath tmpDir = new FilePath(MiniDFSCluster.GetBaseDirectory(), "testCheckpointCancellation-tmp"
);
FSNamesystem fsn = cluster.GetNamesystem(0);
FSImageTestUtil.CreateAbortedLogWithMkdirs(tmpDir, NumDirsInLog, 3, fsn.GetFSDirectory
().GetLastInodeId() + 1);
string fname = NNStorage.GetInProgressEditsFileName(3);
new FilePath(tmpDir, fname).RenameTo(new FilePath(sharedDir, fname));
// Checkpoint as fast as we can, in a tight loop.
cluster.GetConfiguration(1).SetInt(DFSConfigKeys.DfsNamenodeCheckpointPeriodKey,
0);
cluster.RestartNameNode(1);
nn1 = cluster.GetNameNode(1);
cluster.TransitionToActive(0);
bool canceledOne = false;
for (int i = 0; i < 10 && !canceledOne; i++)
{
DoEdits(i * 10, i * 10 + 10);
cluster.TransitionToStandby(0);
cluster.TransitionToActive(1);
cluster.TransitionToStandby(1);
cluster.TransitionToActive(0);
canceledOne = StandbyCheckpointer.GetCanceledCount() > 0;
}
NUnit.Framework.Assert.IsTrue(canceledOne);
}