/// <summary> /// Creates a canonical blob path from a canonical tree path and a blob name. /// </summary> /// <param name="tree"></param> /// <param name="name"></param> public CanonicalBlobPath(CanonicalTreePath tree, string name) { this.Tree = tree; this.Name = name; this._asString = String.Concat(Tree.ToString(), Name); }
public async Task <Errorable <TreeTree> > PersistTreeNodesByBlobPaths(Maybe <TreeID> rootID, IEnumerable <CanonicalBlobIDPath> paths) { TreeNode root; if (rootID.HasValue) { var eroot = await getTree(rootID.Value); if (eroot.HasErrors) { return(eroot.Errors); } root = eroot.Value; } else { root = null; } var nodeByPath = new Dictionary <string, Tuple <CanonicalTreePath, TreeNode.Builder> >(); int depthCapacity = 5; var depthGroups = new List <Tuple <CanonicalTreePath, TreeNode.Builder> > [depthCapacity]; Tuple <CanonicalTreePath, TreeNode.Builder> tpl; // Add the root node by default: CanonicalTreePath rootPath = (CanonicalTreePath)"/"; if (rootID.HasValue) { tpl = new Tuple <CanonicalTreePath, TreeNode.Builder>(rootPath, new TreeNode.Builder(root)); } else { tpl = new Tuple <CanonicalTreePath, TreeNode.Builder>(rootPath, new TreeNode.Builder(new List <TreeTreeReference>(), new List <TreeBlobReference>())); } nodeByPath.Add(rootPath.ToString(), tpl); // Initialize the depthGroups array: int deepest = 0; depthGroups[0] = new List <Tuple <CanonicalTreePath, TreeNode.Builder> >(1) { tpl }; for (int i = 1; i < depthCapacity; ++i) { depthGroups[i] = new List <Tuple <CanonicalTreePath, TreeNode.Builder> >(); } Debug.WriteLine(String.Empty); foreach (CanonicalBlobIDPath path in paths) { TreeNode.Builder tnb; CanonicalTreePath treePath; string treePathStr; // Create node builders for each subfolder in the path below the root: for (int depth = 1; depth <= path.Path.Tree.Parts.Count; ++depth) { treePath = path.Path.Tree.GetPartialTree(depth); treePathStr = treePath.ToString(); // Get the node builder for the current path or create it: if (!nodeByPath.ContainsKey(treePathStr)) { tnb = null; if (rootID.HasValue) { // Get the TreeNode if available, otherwise construct a new builder: var ecurr = await getTreeIDByPath(root, new TreeTreePath(rootID.Value, treePath)); if (ecurr.HasErrors) { return(ecurr.Errors); } var curr = ecurr.Value; if (curr.TreeID.HasValue) { // Get the TreeNode: var etr = await getTree(curr.TreeID.Value); if (etr.HasErrors) { return(etr.Errors); } // Create the builder from that node: tnb = new TreeNode.Builder(etr.Value); } } if (tnb == null) { // New builder: tnb = new TreeNode.Builder(new List <TreeTreeReference>(), new List <TreeBlobReference>()); } // Keep track of this node builder for the given path: Debug.WriteLine(treePathStr); tpl = new Tuple <CanonicalTreePath, TreeNode.Builder>(treePath, tnb); nodeByPath.Add(treePathStr, tpl); // Maintain the depthGroups array: if (depth >= depthCapacity) { // TODO: tune this resize policy: int newCapacity = depth + 2; // Extend the array: Array.Resize(ref depthGroups, newCapacity); // Initialize all the new elements: for (int i = depthCapacity; i < newCapacity; ++i) { depthGroups[i] = new List <Tuple <CanonicalTreePath, TreeNode.Builder> >(); } depthCapacity = newCapacity; } if (depth > deepest) { deepest = depth; } depthGroups[depth].Add(tpl); } } treePath = path.Path.Tree; treePathStr = treePath.ToString(); // Get the node builder for the current path or create it: Tuple <CanonicalTreePath, TreeNode.Builder> ptnb; bool test = nodeByPath.TryGetValue(treePathStr, out ptnb); Debug.Assert(test); // Add or update the TreeBlobReference for this blob path: string blobName = path.Path.Name; var trblb = new TreeBlobReference.Builder(blobName, path.BlobID); int blidx = ptnb.Item2.Blobs.FindIndex(trbl => trbl.Name == blobName); if (blidx == -1) { ptnb.Item2.Blobs.Add(trblb); } else { ptnb.Item2.Blobs[blidx] = trblb; } } #if DEBUG Debug.WriteLine(String.Empty); foreach (var mtpl in nodeByPath.Values) { Debug.WriteLine(String.Format( "{0}: {1}", mtpl.Item1.ToString(), String.Join(", ", mtpl.Item2.Blobs.Select(trbl => trbl.Name + ":" + trbl.BlobID.ToString(firstLength: 7))) )); } #endif for (int i = deepest; i >= 0; --i) { Debug.WriteLine(String.Empty); Debug.WriteLine("Depth #{0}", i); var nodes = depthGroups[i]; foreach (var node in nodes) { Debug.WriteLine(node.Item1.ToString()); foreach (var bl in node.Item2.Blobs) { Debug.WriteLine(new string('_', i * 2) + bl.Name + ":" + bl.BlobID.ToString(firstLength: 7)); } } } // Persist each tree depth level: var awaiting = new List <Task <Errorable <TreeNode> > >(depthGroups[deepest].Count); var lastNodes = new List <Tuple <CanonicalTreePath, TreeNode> >(depthGroups[deepest].Count); var result = new Dictionary <TreeID, TreeNode>(); Debug.WriteLine(String.Format("Starting depth group #{0}", deepest)); foreach (var dnode in depthGroups[deepest]) { // Finalize the TreeNode: TreeNode tn = dnode.Item2; lastNodes.Add(new Tuple <CanonicalTreePath, TreeNode>(dnode.Item1, tn)); if (!result.ContainsKey(tn.ID)) { Debug.WriteLine(String.Format("{0}: Persisting TreeID {1}", dnode.Item1.ToString(), tn.ID.ToString(firstLength: 7))); result.Add(tn.ID, tn); // Start persistence task and add to `awaiting` var tsk = Task.Run(() => persistTree(tn)); awaiting.Add(tsk); } else { Debug.WriteLine(String.Format("{0}: Already persisted TreeID {1}", dnode.Item1.ToString(), tn.ID.ToString(firstLength: 7))); } } for (int i = deepest - 1; i >= 0; --i) { // Await last depth group if non-empty: if (awaiting.Count > 0) { Debug.WriteLine(String.Format("Awaiting previous depth group's persistence")); var errs = await Task.WhenAll(awaiting); if (errs.Any(err => err.HasErrors)) { return(errs.Aggregate(new ErrorContainer(), (acc, err) => acc + err.Errors)); } } Debug.WriteLine(String.Format("Starting depth group #{0}", i)); awaiting = new List <Task <Errorable <TreeNode> > >(depthGroups[i].Count); var currNodes = new List <Tuple <CanonicalTreePath, TreeNode> >(depthGroups[i].Count); // Update each tree node in this depth group: foreach (var node in depthGroups[i]) { // Create TreeTreeReferences to point to child TreeIDs: var childNodes = lastNodes.Where(t => t.Item1.GetParent() == node.Item1); foreach (var childNode in childNodes) { // Add or update the TreeTreeReference to the child tree: int tridx = node.Item2.Trees.FindIndex(trtr => trtr.Name == childNode.Item1.Name); var trtrb = new TreeTreeReference.Builder(childNode.Item1.Name, childNode.Item2.ID); if (tridx == -1) { Debug.WriteLine(String.Format("{0}: Adding (name: {1}, treeid: {2})", node.Item1.ToString(), trtrb.Name, trtrb.TreeID.ToString(firstLength: 7))); node.Item2.Trees.Add(trtrb); } else { Debug.WriteLine(String.Format("{0}: Updating (name: {1}, old-treeid: {2}, new-treeid: {3})", node.Item1.ToString(), trtrb.Name, node.Item2.Trees[tridx].TreeID, trtrb.TreeID.ToString(firstLength: 7))); node.Item2.Trees[tridx] = trtrb; } } // Finalize the TreeNode: TreeNode tn = (TreeNode)node.Item2; currNodes.Add(new Tuple <CanonicalTreePath, TreeNode>(node.Item1, tn)); if (!result.ContainsKey(tn.ID)) { Debug.WriteLine(String.Format("{0}: Persisting TreeID {1}", node.Item1.ToString(), tn.ID.ToString(firstLength: 7))); result.Add(tn.ID, tn); // Start persistence task and add to `awaiting` var tsk = Task.Run(() => persistTree(tn)); awaiting.Add(tsk); } else { Debug.WriteLine(String.Format("{0}: Already persisted TreeID {1}", node.Item1.ToString(), tn.ID.ToString(firstLength: 7))); } } lastNodes = currNodes; } if (awaiting.Count > 0) { Debug.WriteLine(String.Format("Awaiting previous depth group's persistence")); var errs = await Task.WhenAll(awaiting); if (errs.Any(err => err.HasErrors)) { return(errs.Aggregate(new ErrorContainer(), (acc, err) => acc + err.Errors)); } } // Last depth group should be count of 1, the new root TreeNode: Debug.Assert(lastNodes.Count == 1); return(new TreeTree(lastNodes[0].Item2.ID, new ImmutableContainer <TreeID, TreeNode>(result))); }