public static async Task moveZnode(SolrZkClient zkClient, string src, string dst, CancellationToken token)
{
String destName = normalizeDest(src, dst, true, true);
// Special handling if the source has no children, i.e. copying just a single file.
if (!(await zkClient.getChildren(src, null, true)).Any())
{
await zkClient.makePath(destName, false, true);
await zkClient.setData(destName, await zkClient.getData(src, null, null, true), true);
}
else
{
await traverseZkTree(zkClient, src, VISIT_ORDER.VISIT_PRE, async path =>
{
var finalDestination = dst;
if (path.Equals(src) == false)
{
finalDestination += "/" + path.Substring(src.Length + 1);
}
await zkClient.makePath(finalDestination, false, true);
await zkClient.setData(finalDestination, await zkClient.getData(path, null, null, true), true);
});
}
// Insure all source znodes are present in dest before deleting the source.
// throws error if not all there so the source is left intact. Throws error if source and dest don't match.
await checkAllZnodesThere(zkClient, src, destName);
await clean(zkClient, src, token);
}
public static async Task downloadFromZK(SolrZkClient zkClient, string zkPath, string filePath, CancellationToken token)
{
try
{
token.ThrowIfCancellationRequested();
var children = await zkClient.getChildren(zkPath, null, true);
// If it has no children, it's a leaf node, write the associated data from the ZNode.
// Otherwise, continue recursing, but write the associated data to a special file if any
if (!children.Any())
{
// If we didn't copy data down, then we also didn't create the file. But we still need a marker on the local
// disk so create an empty file.
if (await copyDataDown(zkClient, zkPath, filePath) == 0)
{
Debug.WriteLine("Download: " + filePath);
File.Create(filePath);
}
}
else
{
Directory.CreateDirectory(filePath); // Make parent dir.
// ZK nodes, whether leaf or not can have data. If it's a non-leaf node and
// has associated data write it into the special file.
token.ThrowIfCancellationRequested();
Debug.WriteLine("Download: " + filePath);
await copyDataDown(zkClient, zkPath, Path.Combine(filePath, ZKNODE_DATA_FILE));
foreach (var child in children)
{
var zkChild = zkPath;
if (zkChild.EndsWith("/") == false)
{
zkChild += "/";
}
zkChild += child;
if (await isEphemeral(zkClient, zkChild))
{
// Don't copy ephemeral nodes
continue;
}
// Go deeper into the tree now
await downloadFromZK(zkClient, zkChild, Path.Combine(filePath, child), token);
}
}
}
catch (Exception ex)
{
if (ex is KeeperException || ex is ThreadInterruptedException)
{
throw new IOException("Error downloading files from zookeeper path " + zkPath + " to " + filePath,
SolrZkClient.checkInterrupted(ex));
}
throw;
}
}
public async Task <List <string> > listConfigs()
{
try
{
return(await zkClient.getChildren(ConfigsZKnode, null, true));
}
catch (KeeperException.NoNodeException)
{
return(new List <string>());
}
catch (Exception ex)
{
if (ex is KeeperException || ex is ThreadInterruptedException)
{
throw new IOException("Error listing configs", SolrZkClient.checkInterrupted(ex));
}
throw;
}
}
// Insure that all the nodes in one path match the nodes in the other as a safety check before removing
// the source in a 'mv' command.
private static async Task checkAllZnodesThere(SolrZkClient zkClient, string src, string dst)
{
foreach (var node in await zkClient.getChildren(src, null, true))
{
if (await zkClient.exists(dst + "/" + node, true) == false)
{
throw new Exception("mv command did not move node " + dst + "/" + node + " source left intact");
}
await checkAllZnodesThere(zkClient, src + "/" + node, dst + "/" + node);
}
}
//TODO: Check if works
#region CHECK IF WORKS
/// <summary>
/// Lists a ZNode child and (optionally) the znodes of all the children. No data is dumped
/// </summary>
/// <param name="zkClient"></param>
/// <param name="path">The node to remove on Zookeeper</param>
/// <param name="recurse">Whether to remove children</param>
/// <returns>An indented list of the znodes suitable for display</returns>
/// <exception cref="KeeperException"> Could not perform the Zookeeper operation</exception>
public static async Task <string> listZnode(SolrZkClient zkClient, string path, bool recurse)
{
var root = path;
if (path.ToLower(CultureInfo.InvariantCulture).StartsWith("zk:"))
{
root = path.Substring(3);
}
if (root.Equals("/") == false && root.EndsWith("/"))
{
root = root.Substring(0, root.Length - 1);
}
var sb = new StringBuilder();
if (recurse == false)
{
foreach (var node in await zkClient.getChildren(root, null, true))
{
if (node.Equals("zookeeper") == false)
{
sb.Append(node).Append(Environment.NewLine);
}
}
return(sb.ToString());
}
await traverseZkTree(zkClient, root, VISIT_ORDER.VISIT_PRE, znode =>
{
if (znode.StartsWith("/zookeeper"))
{
return; // can't do anything with this node!
}
int iPos = znode.LastIndexOf("/");
if (iPos > 0)
{
for (int idx = 0; idx < iPos; ++idx)
{
sb.Append(" ");
}
sb.Append(znode.Substring(iPos + 1)).Append(Environment.NewLine);
}
else
{
sb.Append(znode).Append(Environment.NewLine);
}
});
return(sb.ToString());
}
public static async Task <IEnumerable <string> > GetTree(SolrZkClient zkCnxn, string zooPath = "/")
{
var children = (await zkCnxn.getChildren(zooPath, null, false));
var nodes = new List <string>();
if (!children.Any())
{
return(nodes);
}
if (zooPath.Last() != '/')
{
zooPath += "/";
}
foreach (var child in children)
{
nodes.Add(zooPath + child);
nodes.AddRange(await GetTree(zkCnxn, zooPath + child));
}
return(nodes);
}
/// <summary>
/// Recursively visit a zk tree rooted at path and apply the given visitor to each path. Exists as a separate method
/// because some of the logic can get nuanced.
/// </summary>
/// <param name="zkClient"></param>
/// <param name="path">The path to start from</param>
/// <param name="visitOrder">Whether to call the visitor at the at the ending or beginning of the run.</param>
/// <param name="visitor">The operation to perform on each path</param>
public static async Task traverseZkTree(SolrZkClient zkClient, string path, VISIT_ORDER visitOrder, Action <string> visitor)
{
if (visitOrder == VISIT_ORDER.VISIT_PRE)
{
visitor.Invoke(path);
}
List <string> children;
try
{
children = await zkClient.getChildren(path, null, true);
}
catch (KeeperException.NoNodeException)
{
return;
}
foreach (var child in children)
{
// we can't do anything to the built-in zookeeper node
if (path.Equals("/") && child.Equals("zookeeper"))
{
continue;
}
if (path.StartsWith("/zookeeper"))
{
continue;
}
if (path.Equals("/"))
{
await traverseZkTree(zkClient, path + child, visitOrder, visitor);
}
else
{
await traverseZkTree(zkClient, path + "/" + child, visitOrder, visitor);
}
}
if (visitOrder == VISIT_ORDER.VISIT_POST)
{
visitor.Invoke(path);
}
}
/// <summary>
/// Copy between local file system and Zookeeper, or from one Zookeeper node to another, optionally copying recursively.
/// </summary>
/// <param name="zkClient"></param>
/// <param name="src">Source to copy from. Both src and dst may be Znodes. However, both may NOT be local</param>
/// <param name="srcIsZk"></param>
/// <param name="dst">The place to copy the files too. Both src and dst may be Znodes. However both may NOT be local</param>
/// <param name="dstIsZk"></param>
/// <param name="recurse">If the source is a directory, reccursively copy the contents iff this is true.</param>
/// <exception cref="ArgumentException">Explanatory exception due to bad params, failed operation, etc.</exception>
public static async Task zkTransfer(SolrZkClient zkClient, string src, bool srcIsZk, string dst, bool dstIsZk, bool recurse, CancellationToken token)
{
if (srcIsZk == false && dstIsZk == false)
{
throw new Exception("One or both of source or destination must specify ZK nodes.");
}
// Make sure -recurse is specified if the source has children.
if (recurse == false)
{
if (srcIsZk)
{
if ((await zkClient.getChildren(src, null, true)).Any())
{
throw new ArgumentException("Zookeeper node " + src + " has children and recurse is false");
}
}
else if (IsDirectory(src))
{
throw new ArgumentException("Local path " + src + " is a directory and recurse is false");
}
}
if (dstIsZk && dst.Length == 0)
{
dst = "/"; // for consistency, one can copy from zk: and send to zk:/
}
dst = normalizeDest(src, dst, srcIsZk, dstIsZk);
// ZK -> ZK copy.
if (srcIsZk && dstIsZk)
{
await traverseZkTree(zkClient, src, VISIT_ORDER.VISIT_PRE, async path =>
{
var finalDestination = dst;
if (path.Equals(src) == false)
{
finalDestination += "/" + path.Substring(src.Length + 1);
}
await zkClient.makePath(finalDestination, false, true);
await zkClient.setData(finalDestination, await zkClient.getData(path, null, null, true), true);
});
return;
}
//local -> ZK copy
if (dstIsZk)
{
await uploadToZK(zkClient, src, dst, null, token);
return;
}
// Copying individual files from ZK requires special handling since downloadFromZK assumes the node has children.
// This is kind of a weak test for the notion of "directory" on Zookeeper.
// ZK -> local copy where ZK is a parent node
if ((await zkClient.getChildren(src, null, true)).Any())
{
await downloadFromZK(zkClient, src, dst, token);
return;
}
// Single file ZK -> local copy where ZK is a leaf node
if (IsDirectory(dst))
{
if (dst.EndsWith(Path.DirectorySeparatorChar.ToString()) == false)
{
dst += Path.DirectorySeparatorChar;
}
dst = normalizeDest(src, dst, srcIsZk, dstIsZk);
}
byte[] data = await zkClient.getData(src, null, null, true);
Directory.CreateDirectory(Path.GetDirectoryName(dst));
//TODO: Log here
//log.info("Writing file {}", filename);
File.WriteAllBytes(dst, data);
}
