internal async Task DeleteTagByNameTest() { TreeNode tr = new TreeNode.Builder(new List <TreeTreeReference>(0), new List <TreeBlobReference>(0)); await trrepo.PersistTree(tr.ID, new IVO.Definition.Containers.ImmutableContainer <TreeID, TreeNode>(trx => trx.ID, tr)); Commit cm = new Commit.Builder(new List <CommitID>(0), tr.ID, "James S. Dunne", DateTimeOffset.Now, "Initial commit."); await cmrepo.PersistCommit(cm); Tag tg = new Tag.Builder((TagName)"v1.0", cm.ID, "James S. Dunne", DateTimeOffset.Now, "Testing tags"); await tgrepo.PersistTag(tg); var retgPre = await tgrepo.GetTag(tg.ID); Assert.IsFalse(retgPre.HasErrors); Tag rtgPre = retgPre.Value; Assert.IsNotNull(rtgPre); Assert.AreEqual(tg.ID, rtgPre.ID); Assert.AreEqual(tg.Name.ToString(), rtgPre.Name.ToString()); Assert.AreEqual(tg.CommitID, rtgPre.CommitID); Assert.AreEqual(tg.Tagger, rtgPre.Tagger); Assert.AreEqual(tg.DateTagged.ToString(), rtgPre.DateTagged.ToString()); await tgrepo.DeleteTagByName((TagName)"v1.0"); var retgPost = await tgrepo.GetTag(tg.ID); Assert.IsTrue(retgPost.HasErrors); Assert.AreEqual(1, retgPost.Errors.Count); }
internal async Task DeleteRefByNameTest() { TreeNode tr = new TreeNode.Builder(new List <TreeTreeReference>(0), new List <TreeBlobReference>(0)); await trrepo.PersistTree(tr.ID, new IVO.Definition.Containers.ImmutableContainer <TreeID, TreeNode>(trx => trx.ID, tr)); Commit cm = new Commit.Builder(new List <CommitID>(0), tr.ID, "James S. Dunne", DateTimeOffset.Now, "Initial commit."); await cmrepo.PersistCommit(cm); Ref rf = new Ref.Builder((RefName)"v1.0", cm.ID); await rfrepo.PersistRef(rf); var edrf = await rfrepo.DeleteRefByName((RefName)"v1.0"); Assert.IsFalse(edrf.HasErrors); Ref drf = edrf.Value; Assert.IsNotNull(drf); Assert.AreEqual(rf.Name.ToString(), drf.Name.ToString()); Assert.AreEqual(rf.CommitID, drf.CommitID); var errf = await rfrepo.GetRefByName((RefName)"v1.0"); Assert.IsTrue(errf.HasErrors); Assert.AreEqual(1, errf.Errors.Count); }
internal async Task PersistRefTest() { TreeNode tr = new TreeNode.Builder(new List <TreeTreeReference>(0), new List <TreeBlobReference>(0)); await trrepo.PersistTree(tr.ID, new IVO.Definition.Containers.ImmutableContainer <TreeID, TreeNode>(trx => trx.ID, tr)); Commit cm = new Commit.Builder(new List <CommitID>(0), tr.ID, "James S. Dunne", DateTimeOffset.Now, "Initial commit."); await cmrepo.PersistCommit(cm); Ref rf = new Ref.Builder((RefName)"v1.0", cm.ID); await rfrepo.PersistRef(rf); }
internal async Task PersistTagTest() { TreeNode tr = new TreeNode.Builder(new List <TreeTreeReference>(0), new List <TreeBlobReference>(0)); await trrepo.PersistTree(tr.ID, new IVO.Definition.Containers.ImmutableContainer <TreeID, TreeNode>(trx => trx.ID, tr)); Commit cm = new Commit.Builder(new List <CommitID>(0), tr.ID, "James S. Dunne", DateTimeOffset.Now, "Initial commit."); await cmrepo.PersistCommit(cm); Tag tg = new Tag.Builder((TagName)"v1.0", cm.ID, "James S. Dunne", DateTimeOffset.Now, "Testing tags"); await tgrepo.PersistTag(tg); }
private async Task createTrees() { PersistingBlob pbHeader = new PersistingBlob("<div>Header</div>".ToStream()); sblobs = await blrepo.PersistBlobs(pbHeader); trSection1 = new TreeNode.Builder(); trSection2 = new TreeNode.Builder(); trSection3 = new TreeNode.Builder(); trImages = new TreeNode.Builder(); trCSS = new TreeNode.Builder(); trJS = new TreeNode.Builder(); trPages = new TreeNode.Builder( new List <TreeTreeReference> { new TreeTreeReference.Builder("section1", trSection1.ID), new TreeTreeReference.Builder("section2", trSection2.ID), new TreeTreeReference.Builder("section3", trSection3.ID) }, new List <TreeBlobReference>(0) ); trContent = new TreeNode.Builder( new List <TreeTreeReference> { new TreeTreeReference.Builder("images", trImages.ID), new TreeTreeReference.Builder("css", trCSS.ID), new TreeTreeReference.Builder("js", trJS.ID) }, new List <TreeBlobReference>(0) ); trTemplate = new TreeNode.Builder( new List <TreeTreeReference>(0), new List <TreeBlobReference> { new TreeBlobReference.Builder("header", sblobs[0].Value.ID) } ); trRoot = new TreeNode.Builder( new List <TreeTreeReference> { new TreeTreeReference.Builder("template", trTemplate.ID), new TreeTreeReference.Builder("content", trContent.ID), new TreeTreeReference.Builder("pages", trPages.ID) }, new List <TreeBlobReference>(0) ); rootId = trRoot.ID; trees = new ImmutableContainer <TreeID, TreeNode>(tr => tr.ID, trSection1, trSection2, trSection3, trPages, trImages, trCSS, trJS, trContent, trTemplate, trRoot); RecursivePrint(trRoot.ID, trees); }
public Errorable <TreeNode> retrieve(SqlCommand cmd, SqlDataReader dr, int expectedCapacity = 10) { TreeNode.Builder tb = new TreeNode.Builder(new List <TreeTreeReference>(), new List <TreeBlobReference>()); // Read the TreeTreeReferences: while (dr.Read()) { var name = dr.GetSqlString(0).Value; var linked_treeid = (TreeID)dr.GetSqlBinary(1).Value; tb.Trees.Add(new TreeTreeReference.Builder(name, linked_treeid)); } if (!dr.NextResult()) { return(new TreeTreePathDoesNotExistError(this._path)); } // Read the TreeBlobReferences: while (dr.Read()) { var name = dr.GetSqlString(0).Value; var linked_blobid = (BlobID)dr.GetSqlBinary(1).Value.ToArray(20); tb.Blobs.Add(new TreeBlobReference.Builder(name, linked_blobid)); } TreeNode tr = tb; TreeID retrievedId = (TreeID)((SqlBinary)cmd.Parameters["@treeid"].SqlValue).Value; if (tr.ID != retrievedId) { return(new ComputedTreeIDMismatchError(tr.ID, retrievedId)); } return(tr); }
public Errorable <TreeNode> retrieve(SqlCommand cmd, SqlDataReader dr) { TreeNode.Builder tb = new TreeNode.Builder(new List <TreeTreeReference>(), new List <TreeBlobReference>()); // Read the TreeTreeReferences: while (dr.Read()) { var name = dr.GetSqlString(0).Value; var linked_treeid = (TreeID)dr.GetSqlBinary(1).Value; tb.Trees.Add(new TreeTreeReference.Builder(name, linked_treeid)); } if (!dr.NextResult()) { return(new TreeIDRecordDoesNotExistError(_id)); } // Read the TreeBlobReferences: while (dr.Read()) { var name = dr.GetSqlString(0).Value; var linked_blobid = (BlobID)dr.GetSqlBinary(1).Value.ToArray(20); tb.Blobs.Add(new TreeBlobReference.Builder(name, linked_blobid)); } TreeNode tr = tb; if (tr.ID != _id) { return(new ComputedTreeIDMismatchError(tr.ID, _id)); } return(tr); }
public Errorable <TreeTree> retrieve(SqlCommand cmd, SqlDataReader dr, int expectedCapacity = 10) { Dictionary <TreeID, TreeNode.Builder> trees = new Dictionary <TreeID, TreeNode.Builder>(expectedCapacity); // Iterate through rows of the recursive query, assuming ordering of rows guarantees tree depth locality. while (dr.Read()) { SqlBinary btreeid = dr.GetSqlBinary(0); SqlBinary blinked_treeid = dr.GetSqlBinary(1); SqlString treename = dr.GetSqlString(2); SqlBinary linked_blobid = dr.GetSqlBinary(3); SqlString blobname = dr.GetSqlString(4); TreeID?treeid = btreeid.IsNull ? (TreeID?)null : (TreeID)btreeid.Value; TreeID?linked_treeid = blinked_treeid.IsNull ? (TreeID?)null : (TreeID)blinked_treeid.Value; TreeID pullFor = treeid.HasValue ? treeid.Value : linked_treeid.Value; TreeNode.Builder curr; if (!trees.TryGetValue(pullFor, out curr)) { curr = new TreeNode.Builder(new List <TreeTreeReference>(), new List <TreeBlobReference>()); trees.Add(pullFor, curr); } // The tree to add the blob link to: TreeNode.Builder blobTree = curr; // Add a tree link: if (treeid.HasValue && linked_treeid.HasValue) { // Create the Tree.Builder for the linked_treeid if it does not exist: if (!trees.TryGetValue(linked_treeid.Value, out blobTree)) { blobTree = new TreeNode.Builder(new List <TreeTreeReference>(), new List <TreeBlobReference>()); trees.Add(linked_treeid.Value, blobTree); } List <TreeTreeReference> treeRefs = curr.Trees; bool isDupe = false; if (treeRefs.Count > 0) { // Only check the previous ref record for dupe: // TODO: verify that SQL Server will *always* return rows in an order that supports depth locality from a recursive CTE. isDupe = ( (treeRefs[treeRefs.Count - 1].TreeID == linked_treeid.Value) && (treeRefs[treeRefs.Count - 1].Name == treename.Value) ); } // Don't re-add the same tree link: if (!isDupe) { treeRefs.Add(new TreeTreeReference.Builder(treename.Value, linked_treeid.Value)); } } // Add a blob link to the child or parent tree: if (!linked_blobid.IsNull) { blobTree.Blobs.Add(new TreeBlobReference.Builder(blobname.Value, (BlobID)linked_blobid.Value.ToArray(20))); } } List <TreeNode> finals = new List <TreeNode>(trees.Count); foreach (KeyValuePair <TreeID, TreeNode.Builder> pair in trees) { TreeNode tr = pair.Value; if (tr.ID != pair.Key) { return(new ComputedTreeIDMismatchError(tr.ID, pair.Key)); } finals.Add(tr); } // Return the final result with immutable objects: return(new TreeTree(this._id, new ImmutableContainer <TreeID, TreeNode>(tr => tr.ID, finals))); }
public async Task <Errorable <TreeTree> > PersistTreeNodesByBlobPaths(Maybe <TreeID> rootID, IEnumerable <CanonicalBlobIDPath> paths) { TreeNode root; if (rootID.HasValue) { var eroot = await getTree(rootID.Value); if (eroot.HasErrors) { return(eroot.Errors); } root = eroot.Value; } else { root = null; } var nodeByPath = new Dictionary <string, Tuple <CanonicalTreePath, TreeNode.Builder> >(); int depthCapacity = 5; var depthGroups = new List <Tuple <CanonicalTreePath, TreeNode.Builder> > [depthCapacity]; Tuple <CanonicalTreePath, TreeNode.Builder> tpl; // Add the root node by default: CanonicalTreePath rootPath = (CanonicalTreePath)"/"; if (rootID.HasValue) { tpl = new Tuple <CanonicalTreePath, TreeNode.Builder>(rootPath, new TreeNode.Builder(root)); } else { tpl = new Tuple <CanonicalTreePath, TreeNode.Builder>(rootPath, new TreeNode.Builder(new List <TreeTreeReference>(), new List <TreeBlobReference>())); } nodeByPath.Add(rootPath.ToString(), tpl); // Initialize the depthGroups array: int deepest = 0; depthGroups[0] = new List <Tuple <CanonicalTreePath, TreeNode.Builder> >(1) { tpl }; for (int i = 1; i < depthCapacity; ++i) { depthGroups[i] = new List <Tuple <CanonicalTreePath, TreeNode.Builder> >(); } Debug.WriteLine(String.Empty); foreach (CanonicalBlobIDPath path in paths) { TreeNode.Builder tnb; CanonicalTreePath treePath; string treePathStr; // Create node builders for each subfolder in the path below the root: for (int depth = 1; depth <= path.Path.Tree.Parts.Count; ++depth) { treePath = path.Path.Tree.GetPartialTree(depth); treePathStr = treePath.ToString(); // Get the node builder for the current path or create it: if (!nodeByPath.ContainsKey(treePathStr)) { tnb = null; if (rootID.HasValue) { // Get the TreeNode if available, otherwise construct a new builder: var ecurr = await getTreeIDByPath(root, new TreeTreePath(rootID.Value, treePath)); if (ecurr.HasErrors) { return(ecurr.Errors); } var curr = ecurr.Value; if (curr.TreeID.HasValue) { // Get the TreeNode: var etr = await getTree(curr.TreeID.Value); if (etr.HasErrors) { return(etr.Errors); } // Create the builder from that node: tnb = new TreeNode.Builder(etr.Value); } } if (tnb == null) { // New builder: tnb = new TreeNode.Builder(new List <TreeTreeReference>(), new List <TreeBlobReference>()); } // Keep track of this node builder for the given path: Debug.WriteLine(treePathStr); tpl = new Tuple <CanonicalTreePath, TreeNode.Builder>(treePath, tnb); nodeByPath.Add(treePathStr, tpl); // Maintain the depthGroups array: if (depth >= depthCapacity) { // TODO: tune this resize policy: int newCapacity = depth + 2; // Extend the array: Array.Resize(ref depthGroups, newCapacity); // Initialize all the new elements: for (int i = depthCapacity; i < newCapacity; ++i) { depthGroups[i] = new List <Tuple <CanonicalTreePath, TreeNode.Builder> >(); } depthCapacity = newCapacity; } if (depth > deepest) { deepest = depth; } depthGroups[depth].Add(tpl); } } treePath = path.Path.Tree; treePathStr = treePath.ToString(); // Get the node builder for the current path or create it: Tuple <CanonicalTreePath, TreeNode.Builder> ptnb; bool test = nodeByPath.TryGetValue(treePathStr, out ptnb); Debug.Assert(test); // Add or update the TreeBlobReference for this blob path: string blobName = path.Path.Name; var trblb = new TreeBlobReference.Builder(blobName, path.BlobID); int blidx = ptnb.Item2.Blobs.FindIndex(trbl => trbl.Name == blobName); if (blidx == -1) { ptnb.Item2.Blobs.Add(trblb); } else { ptnb.Item2.Blobs[blidx] = trblb; } } #if DEBUG Debug.WriteLine(String.Empty); foreach (var mtpl in nodeByPath.Values) { Debug.WriteLine(String.Format( "{0}: {1}", mtpl.Item1.ToString(), String.Join(", ", mtpl.Item2.Blobs.Select(trbl => trbl.Name + ":" + trbl.BlobID.ToString(firstLength: 7))) )); } #endif for (int i = deepest; i >= 0; --i) { Debug.WriteLine(String.Empty); Debug.WriteLine("Depth #{0}", i); var nodes = depthGroups[i]; foreach (var node in nodes) { Debug.WriteLine(node.Item1.ToString()); foreach (var bl in node.Item2.Blobs) { Debug.WriteLine(new string('_', i * 2) + bl.Name + ":" + bl.BlobID.ToString(firstLength: 7)); } } } // Persist each tree depth level: var awaiting = new List <Task <Errorable <TreeNode> > >(depthGroups[deepest].Count); var lastNodes = new List <Tuple <CanonicalTreePath, TreeNode> >(depthGroups[deepest].Count); var result = new Dictionary <TreeID, TreeNode>(); Debug.WriteLine(String.Format("Starting depth group #{0}", deepest)); foreach (var dnode in depthGroups[deepest]) { // Finalize the TreeNode: TreeNode tn = dnode.Item2; lastNodes.Add(new Tuple <CanonicalTreePath, TreeNode>(dnode.Item1, tn)); if (!result.ContainsKey(tn.ID)) { Debug.WriteLine(String.Format("{0}: Persisting TreeID {1}", dnode.Item1.ToString(), tn.ID.ToString(firstLength: 7))); result.Add(tn.ID, tn); // Start persistence task and add to `awaiting` var tsk = Task.Run(() => persistTree(tn)); awaiting.Add(tsk); } else { Debug.WriteLine(String.Format("{0}: Already persisted TreeID {1}", dnode.Item1.ToString(), tn.ID.ToString(firstLength: 7))); } } for (int i = deepest - 1; i >= 0; --i) { // Await last depth group if non-empty: if (awaiting.Count > 0) { Debug.WriteLine(String.Format("Awaiting previous depth group's persistence")); var errs = await Task.WhenAll(awaiting); if (errs.Any(err => err.HasErrors)) { return(errs.Aggregate(new ErrorContainer(), (acc, err) => acc + err.Errors)); } } Debug.WriteLine(String.Format("Starting depth group #{0}", i)); awaiting = new List <Task <Errorable <TreeNode> > >(depthGroups[i].Count); var currNodes = new List <Tuple <CanonicalTreePath, TreeNode> >(depthGroups[i].Count); // Update each tree node in this depth group: foreach (var node in depthGroups[i]) { // Create TreeTreeReferences to point to child TreeIDs: var childNodes = lastNodes.Where(t => t.Item1.GetParent() == node.Item1); foreach (var childNode in childNodes) { // Add or update the TreeTreeReference to the child tree: int tridx = node.Item2.Trees.FindIndex(trtr => trtr.Name == childNode.Item1.Name); var trtrb = new TreeTreeReference.Builder(childNode.Item1.Name, childNode.Item2.ID); if (tridx == -1) { Debug.WriteLine(String.Format("{0}: Adding (name: {1}, treeid: {2})", node.Item1.ToString(), trtrb.Name, trtrb.TreeID.ToString(firstLength: 7))); node.Item2.Trees.Add(trtrb); } else { Debug.WriteLine(String.Format("{0}: Updating (name: {1}, old-treeid: {2}, new-treeid: {3})", node.Item1.ToString(), trtrb.Name, node.Item2.Trees[tridx].TreeID, trtrb.TreeID.ToString(firstLength: 7))); node.Item2.Trees[tridx] = trtrb; } } // Finalize the TreeNode: TreeNode tn = (TreeNode)node.Item2; currNodes.Add(new Tuple <CanonicalTreePath, TreeNode>(node.Item1, tn)); if (!result.ContainsKey(tn.ID)) { Debug.WriteLine(String.Format("{0}: Persisting TreeID {1}", node.Item1.ToString(), tn.ID.ToString(firstLength: 7))); result.Add(tn.ID, tn); // Start persistence task and add to `awaiting` var tsk = Task.Run(() => persistTree(tn)); awaiting.Add(tsk); } else { Debug.WriteLine(String.Format("{0}: Already persisted TreeID {1}", node.Item1.ToString(), tn.ID.ToString(firstLength: 7))); } } lastNodes = currNodes; } if (awaiting.Count > 0) { Debug.WriteLine(String.Format("Awaiting previous depth group's persistence")); var errs = await Task.WhenAll(awaiting); if (errs.Any(err => err.HasErrors)) { return(errs.Aggregate(new ErrorContainer(), (acc, err) => acc + err.Errors)); } } // Last depth group should be count of 1, the new root TreeNode: Debug.Assert(lastNodes.Count == 1); return(new TreeTree(lastNodes[0].Item2.ID, new ImmutableContainer <TreeID, TreeNode>(result))); }
private async Task <Errorable <TreeNode> > getTree(TreeID id) { FileInfo fi = system.getPathByID(id); if (!fi.Exists) { return(new TreeIDRecordDoesNotExistError(id)); } byte[] buf; int nr = 0; using (var fs = new FileStream(fi.FullName, FileMode.Open, FileAccess.Read, FileShare.Read, 16384, true)) { // TODO: implement an async buffered Stream: buf = new byte[16384]; nr = await fs.ReadAsync(buf, 0, 16384).ConfigureAwait(continueOnCapturedContext: false); if (nr >= 16384) { // My, what a large tree you have! throw new NotSupportedException(); } } TreeNode.Builder tb = new TreeNode.Builder(new List <TreeTreeReference>(), new List <TreeBlobReference>()); // Parse the Tree: using (var ms = new MemoryStream(buf, 0, nr, false)) using (var sr = new StreamReader(ms, Encoding.UTF8)) { string line; while ((line = sr.ReadLine()) != null) { if (line.StartsWith("tree ")) { string linked_treeid = line.Substring(5, (TreeID.ByteArrayLength * 2)); string name = line.Substring(6 + (TreeID.ByteArrayLength * 2)); // Attempt to parse the TreeID and verify its existence: Errorable <TreeID> trid = TreeID.TryParse(linked_treeid); if (trid.HasErrors) { return(trid.Errors); } if (!system.getPathByID(trid.Value).Exists) { return(new TreeIDRecordDoesNotExistError(trid.Value)); } tb.Trees.Add(new TreeTreeReference.Builder(name, trid.Value)); } else if (line.StartsWith("blob ")) { string linked_blobid = line.Substring(5, (TreeID.ByteArrayLength * 2)); string name = line.Substring(6 + (TreeID.ByteArrayLength * 2)); // Attempt to parse the BlobID and verify its existence: Errorable <BlobID> blid = BlobID.TryParse(linked_blobid); if (blid.HasErrors) { return(blid.Errors); } if (!system.getPathByID(blid.Value).Exists) { return(new BlobIDRecordDoesNotExistError(blid.Value)); } tb.Blobs.Add(new TreeBlobReference.Builder(name, blid.Value)); } } } // Create the immutable Tree from the Builder: TreeNode tr = tb; // Validate the computed TreeID: if (tr.ID != id) { return(new ComputedTreeIDMismatchError(tr.ID, id)); } return(tr); }