internal async Task DeleteTagByNameTest()
{
TreeNode tr = new TreeNode.Builder(new List <TreeTreeReference>(0), new List <TreeBlobReference>(0));
await trrepo.PersistTree(tr.ID, new IVO.Definition.Containers.ImmutableContainer <TreeID, TreeNode>(trx => trx.ID, tr));
Commit cm = new Commit.Builder(new List <CommitID>(0), tr.ID, "James S. Dunne", DateTimeOffset.Now, "Initial commit.");
await cmrepo.PersistCommit(cm);
Tag tg = new Tag.Builder((TagName)"v1.0", cm.ID, "James S. Dunne", DateTimeOffset.Now, "Testing tags");
await tgrepo.PersistTag(tg);
var retgPre = await tgrepo.GetTag(tg.ID);
Assert.IsFalse(retgPre.HasErrors);
Tag rtgPre = retgPre.Value;
Assert.IsNotNull(rtgPre);
Assert.AreEqual(tg.ID, rtgPre.ID);
Assert.AreEqual(tg.Name.ToString(), rtgPre.Name.ToString());
Assert.AreEqual(tg.CommitID, rtgPre.CommitID);
Assert.AreEqual(tg.Tagger, rtgPre.Tagger);
Assert.AreEqual(tg.DateTagged.ToString(), rtgPre.DateTagged.ToString());
await tgrepo.DeleteTagByName((TagName)"v1.0");
var retgPost = await tgrepo.GetTag(tg.ID);
Assert.IsTrue(retgPost.HasErrors);
Assert.AreEqual(1, retgPost.Errors.Count);
}
internal async Task DeleteRefByNameTest()
{
TreeNode tr = new TreeNode.Builder(new List <TreeTreeReference>(0), new List <TreeBlobReference>(0));
await trrepo.PersistTree(tr.ID, new IVO.Definition.Containers.ImmutableContainer <TreeID, TreeNode>(trx => trx.ID, tr));
Commit cm = new Commit.Builder(new List <CommitID>(0), tr.ID, "James S. Dunne", DateTimeOffset.Now, "Initial commit.");
await cmrepo.PersistCommit(cm);
Ref rf = new Ref.Builder((RefName)"v1.0", cm.ID);
await rfrepo.PersistRef(rf);
var edrf = await rfrepo.DeleteRefByName((RefName)"v1.0");
Assert.IsFalse(edrf.HasErrors);
Ref drf = edrf.Value;
Assert.IsNotNull(drf);
Assert.AreEqual(rf.Name.ToString(), drf.Name.ToString());
Assert.AreEqual(rf.CommitID, drf.CommitID);
var errf = await rfrepo.GetRefByName((RefName)"v1.0");
Assert.IsTrue(errf.HasErrors);
Assert.AreEqual(1, errf.Errors.Count);
}
internal async Task PersistRefTest()
{
TreeNode tr = new TreeNode.Builder(new List <TreeTreeReference>(0), new List <TreeBlobReference>(0));
await trrepo.PersistTree(tr.ID, new IVO.Definition.Containers.ImmutableContainer <TreeID, TreeNode>(trx => trx.ID, tr));
Commit cm = new Commit.Builder(new List <CommitID>(0), tr.ID, "James S. Dunne", DateTimeOffset.Now, "Initial commit.");
await cmrepo.PersistCommit(cm);
Ref rf = new Ref.Builder((RefName)"v1.0", cm.ID);
await rfrepo.PersistRef(rf);
}
internal async Task PersistTagTest()
{
TreeNode tr = new TreeNode.Builder(new List <TreeTreeReference>(0), new List <TreeBlobReference>(0));
await trrepo.PersistTree(tr.ID, new IVO.Definition.Containers.ImmutableContainer <TreeID, TreeNode>(trx => trx.ID, tr));
Commit cm = new Commit.Builder(new List <CommitID>(0), tr.ID, "James S. Dunne", DateTimeOffset.Now, "Initial commit.");
await cmrepo.PersistCommit(cm);
Tag tg = new Tag.Builder((TagName)"v1.0", cm.ID, "James S. Dunne", DateTimeOffset.Now, "Testing tags");
await tgrepo.PersistTag(tg);
}
private async Task createTrees()
{
PersistingBlob pbHeader = new PersistingBlob("<div>Header</div>".ToStream());
sblobs = await blrepo.PersistBlobs(pbHeader);
trSection1 = new TreeNode.Builder();
trSection2 = new TreeNode.Builder();
trSection3 = new TreeNode.Builder();
trImages = new TreeNode.Builder();
trCSS = new TreeNode.Builder();
trJS = new TreeNode.Builder();
trPages = new TreeNode.Builder(
new List <TreeTreeReference> {
new TreeTreeReference.Builder("section1", trSection1.ID),
new TreeTreeReference.Builder("section2", trSection2.ID),
new TreeTreeReference.Builder("section3", trSection3.ID)
},
new List <TreeBlobReference>(0)
);
trContent = new TreeNode.Builder(
new List <TreeTreeReference> {
new TreeTreeReference.Builder("images", trImages.ID),
new TreeTreeReference.Builder("css", trCSS.ID),
new TreeTreeReference.Builder("js", trJS.ID)
},
new List <TreeBlobReference>(0)
);
trTemplate = new TreeNode.Builder(
new List <TreeTreeReference>(0),
new List <TreeBlobReference> {
new TreeBlobReference.Builder("header", sblobs[0].Value.ID)
}
);
trRoot = new TreeNode.Builder(
new List <TreeTreeReference> {
new TreeTreeReference.Builder("template", trTemplate.ID),
new TreeTreeReference.Builder("content", trContent.ID),
new TreeTreeReference.Builder("pages", trPages.ID)
},
new List <TreeBlobReference>(0)
);
rootId = trRoot.ID;
trees = new ImmutableContainer <TreeID, TreeNode>(tr => tr.ID, trSection1, trSection2, trSection3, trPages, trImages, trCSS, trJS, trContent, trTemplate, trRoot);
RecursivePrint(trRoot.ID, trees);
}
public Errorable <TreeNode> retrieve(SqlCommand cmd, SqlDataReader dr, int expectedCapacity = 10)
{
TreeNode.Builder tb = new TreeNode.Builder(new List <TreeTreeReference>(), new List <TreeBlobReference>());
// Read the TreeTreeReferences:
while (dr.Read())
{
var name = dr.GetSqlString(0).Value;
var linked_treeid = (TreeID)dr.GetSqlBinary(1).Value;
tb.Trees.Add(new TreeTreeReference.Builder(name, linked_treeid));
}
if (!dr.NextResult())
{
return(new TreeTreePathDoesNotExistError(this._path));
}
// Read the TreeBlobReferences:
while (dr.Read())
{
var name = dr.GetSqlString(0).Value;
var linked_blobid = (BlobID)dr.GetSqlBinary(1).Value.ToArray(20);
tb.Blobs.Add(new TreeBlobReference.Builder(name, linked_blobid));
}
TreeNode tr = tb;
TreeID retrievedId = (TreeID)((SqlBinary)cmd.Parameters["@treeid"].SqlValue).Value;
if (tr.ID != retrievedId)
{
return(new ComputedTreeIDMismatchError(tr.ID, retrievedId));
}
return(tr);
}
public Errorable <TreeNode> retrieve(SqlCommand cmd, SqlDataReader dr)
{
TreeNode.Builder tb = new TreeNode.Builder(new List <TreeTreeReference>(), new List <TreeBlobReference>());
// Read the TreeTreeReferences:
while (dr.Read())
{
var name = dr.GetSqlString(0).Value;
var linked_treeid = (TreeID)dr.GetSqlBinary(1).Value;
tb.Trees.Add(new TreeTreeReference.Builder(name, linked_treeid));
}
if (!dr.NextResult())
{
return(new TreeIDRecordDoesNotExistError(_id));
}
// Read the TreeBlobReferences:
while (dr.Read())
{
var name = dr.GetSqlString(0).Value;
var linked_blobid = (BlobID)dr.GetSqlBinary(1).Value.ToArray(20);
tb.Blobs.Add(new TreeBlobReference.Builder(name, linked_blobid));
}
TreeNode tr = tb;
if (tr.ID != _id)
{
return(new ComputedTreeIDMismatchError(tr.ID, _id));
}
return(tr);
}
public Errorable <TreeTree> retrieve(SqlCommand cmd, SqlDataReader dr, int expectedCapacity = 10)
{
Dictionary <TreeID, TreeNode.Builder> trees = new Dictionary <TreeID, TreeNode.Builder>(expectedCapacity);
// Iterate through rows of the recursive query, assuming ordering of rows guarantees tree depth locality.
while (dr.Read())
{
SqlBinary btreeid = dr.GetSqlBinary(0);
SqlBinary blinked_treeid = dr.GetSqlBinary(1);
SqlString treename = dr.GetSqlString(2);
SqlBinary linked_blobid = dr.GetSqlBinary(3);
SqlString blobname = dr.GetSqlString(4);
TreeID?treeid = btreeid.IsNull ? (TreeID?)null : (TreeID)btreeid.Value;
TreeID?linked_treeid = blinked_treeid.IsNull ? (TreeID?)null : (TreeID)blinked_treeid.Value;
TreeID pullFor = treeid.HasValue ? treeid.Value : linked_treeid.Value;
TreeNode.Builder curr;
if (!trees.TryGetValue(pullFor, out curr))
{
curr = new TreeNode.Builder(new List <TreeTreeReference>(), new List <TreeBlobReference>());
trees.Add(pullFor, curr);
}
// The tree to add the blob link to:
TreeNode.Builder blobTree = curr;
// Add a tree link:
if (treeid.HasValue && linked_treeid.HasValue)
{
// Create the Tree.Builder for the linked_treeid if it does not exist:
if (!trees.TryGetValue(linked_treeid.Value, out blobTree))
{
blobTree = new TreeNode.Builder(new List <TreeTreeReference>(), new List <TreeBlobReference>());
trees.Add(linked_treeid.Value, blobTree);
}
List <TreeTreeReference> treeRefs = curr.Trees;
bool isDupe = false;
if (treeRefs.Count > 0)
{
// Only check the previous ref record for dupe:
// TODO: verify that SQL Server will *always* return rows in an order that supports depth locality from a recursive CTE.
isDupe = (
(treeRefs[treeRefs.Count - 1].TreeID == linked_treeid.Value) &&
(treeRefs[treeRefs.Count - 1].Name == treename.Value)
);
}
// Don't re-add the same tree link:
if (!isDupe)
{
treeRefs.Add(new TreeTreeReference.Builder(treename.Value, linked_treeid.Value));
}
}
// Add a blob link to the child or parent tree:
if (!linked_blobid.IsNull)
{
blobTree.Blobs.Add(new TreeBlobReference.Builder(blobname.Value, (BlobID)linked_blobid.Value.ToArray(20)));
}
}
List <TreeNode> finals = new List <TreeNode>(trees.Count);
foreach (KeyValuePair <TreeID, TreeNode.Builder> pair in trees)
{
TreeNode tr = pair.Value;
if (tr.ID != pair.Key)
{
return(new ComputedTreeIDMismatchError(tr.ID, pair.Key));
}
finals.Add(tr);
}
// Return the final result with immutable objects:
return(new TreeTree(this._id, new ImmutableContainer <TreeID, TreeNode>(tr => tr.ID, finals)));
}
public async Task <Errorable <TreeTree> > PersistTreeNodesByBlobPaths(Maybe <TreeID> rootID, IEnumerable <CanonicalBlobIDPath> paths)
{
TreeNode root;
if (rootID.HasValue)
{
var eroot = await getTree(rootID.Value);
if (eroot.HasErrors)
{
return(eroot.Errors);
}
root = eroot.Value;
}
else
{
root = null;
}
var nodeByPath = new Dictionary <string, Tuple <CanonicalTreePath, TreeNode.Builder> >();
int depthCapacity = 5;
var depthGroups = new List <Tuple <CanonicalTreePath, TreeNode.Builder> > [depthCapacity];
Tuple <CanonicalTreePath, TreeNode.Builder> tpl;
// Add the root node by default:
CanonicalTreePath rootPath = (CanonicalTreePath)"/";
if (rootID.HasValue)
{
tpl = new Tuple <CanonicalTreePath, TreeNode.Builder>(rootPath, new TreeNode.Builder(root));
}
else
{
tpl = new Tuple <CanonicalTreePath, TreeNode.Builder>(rootPath, new TreeNode.Builder(new List <TreeTreeReference>(), new List <TreeBlobReference>()));
}
nodeByPath.Add(rootPath.ToString(), tpl);
// Initialize the depthGroups array:
int deepest = 0;
depthGroups[0] = new List <Tuple <CanonicalTreePath, TreeNode.Builder> >(1)
{
tpl
};
for (int i = 1; i < depthCapacity; ++i)
{
depthGroups[i] = new List <Tuple <CanonicalTreePath, TreeNode.Builder> >();
}
Debug.WriteLine(String.Empty);
foreach (CanonicalBlobIDPath path in paths)
{
TreeNode.Builder tnb;
CanonicalTreePath treePath;
string treePathStr;
// Create node builders for each subfolder in the path below the root:
for (int depth = 1; depth <= path.Path.Tree.Parts.Count; ++depth)
{
treePath = path.Path.Tree.GetPartialTree(depth);
treePathStr = treePath.ToString();
// Get the node builder for the current path or create it:
if (!nodeByPath.ContainsKey(treePathStr))
{
tnb = null;
if (rootID.HasValue)
{
// Get the TreeNode if available, otherwise construct a new builder:
var ecurr = await getTreeIDByPath(root, new TreeTreePath(rootID.Value, treePath));
if (ecurr.HasErrors)
{
return(ecurr.Errors);
}
var curr = ecurr.Value;
if (curr.TreeID.HasValue)
{
// Get the TreeNode:
var etr = await getTree(curr.TreeID.Value);
if (etr.HasErrors)
{
return(etr.Errors);
}
// Create the builder from that node:
tnb = new TreeNode.Builder(etr.Value);
}
}
if (tnb == null)
{
// New builder:
tnb = new TreeNode.Builder(new List <TreeTreeReference>(), new List <TreeBlobReference>());
}
// Keep track of this node builder for the given path:
Debug.WriteLine(treePathStr);
tpl = new Tuple <CanonicalTreePath, TreeNode.Builder>(treePath, tnb);
nodeByPath.Add(treePathStr, tpl);
// Maintain the depthGroups array:
if (depth >= depthCapacity)
{
// TODO: tune this resize policy:
int newCapacity = depth + 2;
// Extend the array:
Array.Resize(ref depthGroups, newCapacity);
// Initialize all the new elements:
for (int i = depthCapacity; i < newCapacity; ++i)
{
depthGroups[i] = new List <Tuple <CanonicalTreePath, TreeNode.Builder> >();
}
depthCapacity = newCapacity;
}
if (depth > deepest)
{
deepest = depth;
}
depthGroups[depth].Add(tpl);
}
}
treePath = path.Path.Tree;
treePathStr = treePath.ToString();
// Get the node builder for the current path or create it:
Tuple <CanonicalTreePath, TreeNode.Builder> ptnb;
bool test = nodeByPath.TryGetValue(treePathStr, out ptnb);
Debug.Assert(test);
// Add or update the TreeBlobReference for this blob path:
string blobName = path.Path.Name;
var trblb = new TreeBlobReference.Builder(blobName, path.BlobID);
int blidx = ptnb.Item2.Blobs.FindIndex(trbl => trbl.Name == blobName);
if (blidx == -1)
{
ptnb.Item2.Blobs.Add(trblb);
}
else
{
ptnb.Item2.Blobs[blidx] = trblb;
}
}
#if DEBUG
Debug.WriteLine(String.Empty);
foreach (var mtpl in nodeByPath.Values)
{
Debug.WriteLine(String.Format(
"{0}: {1}",
mtpl.Item1.ToString(),
String.Join(", ", mtpl.Item2.Blobs.Select(trbl => trbl.Name + ":" + trbl.BlobID.ToString(firstLength: 7)))
));
}
#endif
for (int i = deepest; i >= 0; --i)
{
Debug.WriteLine(String.Empty);
Debug.WriteLine("Depth #{0}", i);
var nodes = depthGroups[i];
foreach (var node in nodes)
{
Debug.WriteLine(node.Item1.ToString());
foreach (var bl in node.Item2.Blobs)
{
Debug.WriteLine(new string('_', i * 2) + bl.Name + ":" + bl.BlobID.ToString(firstLength: 7));
}
}
}
// Persist each tree depth level:
var awaiting = new List <Task <Errorable <TreeNode> > >(depthGroups[deepest].Count);
var lastNodes = new List <Tuple <CanonicalTreePath, TreeNode> >(depthGroups[deepest].Count);
var result = new Dictionary <TreeID, TreeNode>();
Debug.WriteLine(String.Format("Starting depth group #{0}", deepest));
foreach (var dnode in depthGroups[deepest])
{
// Finalize the TreeNode:
TreeNode tn = dnode.Item2;
lastNodes.Add(new Tuple <CanonicalTreePath, TreeNode>(dnode.Item1, tn));
if (!result.ContainsKey(tn.ID))
{
Debug.WriteLine(String.Format("{0}: Persisting TreeID {1}", dnode.Item1.ToString(), tn.ID.ToString(firstLength: 7)));
result.Add(tn.ID, tn);
// Start persistence task and add to `awaiting`
var tsk = Task.Run(() => persistTree(tn));
awaiting.Add(tsk);
}
else
{
Debug.WriteLine(String.Format("{0}: Already persisted TreeID {1}", dnode.Item1.ToString(), tn.ID.ToString(firstLength: 7)));
}
}
for (int i = deepest - 1; i >= 0; --i)
{
// Await last depth group if non-empty:
if (awaiting.Count > 0)
{
Debug.WriteLine(String.Format("Awaiting previous depth group's persistence"));
var errs = await Task.WhenAll(awaiting);
if (errs.Any(err => err.HasErrors))
{
return(errs.Aggregate(new ErrorContainer(), (acc, err) => acc + err.Errors));
}
}
Debug.WriteLine(String.Format("Starting depth group #{0}", i));
awaiting = new List <Task <Errorable <TreeNode> > >(depthGroups[i].Count);
var currNodes = new List <Tuple <CanonicalTreePath, TreeNode> >(depthGroups[i].Count);
// Update each tree node in this depth group:
foreach (var node in depthGroups[i])
{
// Create TreeTreeReferences to point to child TreeIDs:
var childNodes = lastNodes.Where(t => t.Item1.GetParent() == node.Item1);
foreach (var childNode in childNodes)
{
// Add or update the TreeTreeReference to the child tree:
int tridx = node.Item2.Trees.FindIndex(trtr => trtr.Name == childNode.Item1.Name);
var trtrb = new TreeTreeReference.Builder(childNode.Item1.Name, childNode.Item2.ID);
if (tridx == -1)
{
Debug.WriteLine(String.Format("{0}: Adding (name: {1}, treeid: {2})", node.Item1.ToString(), trtrb.Name, trtrb.TreeID.ToString(firstLength: 7)));
node.Item2.Trees.Add(trtrb);
}
else
{
Debug.WriteLine(String.Format("{0}: Updating (name: {1}, old-treeid: {2}, new-treeid: {3})", node.Item1.ToString(), trtrb.Name, node.Item2.Trees[tridx].TreeID, trtrb.TreeID.ToString(firstLength: 7)));
node.Item2.Trees[tridx] = trtrb;
}
}
// Finalize the TreeNode:
TreeNode tn = (TreeNode)node.Item2;
currNodes.Add(new Tuple <CanonicalTreePath, TreeNode>(node.Item1, tn));
if (!result.ContainsKey(tn.ID))
{
Debug.WriteLine(String.Format("{0}: Persisting TreeID {1}", node.Item1.ToString(), tn.ID.ToString(firstLength: 7)));
result.Add(tn.ID, tn);
// Start persistence task and add to `awaiting`
var tsk = Task.Run(() => persistTree(tn));
awaiting.Add(tsk);
}
else
{
Debug.WriteLine(String.Format("{0}: Already persisted TreeID {1}", node.Item1.ToString(), tn.ID.ToString(firstLength: 7)));
}
}
lastNodes = currNodes;
}
if (awaiting.Count > 0)
{
Debug.WriteLine(String.Format("Awaiting previous depth group's persistence"));
var errs = await Task.WhenAll(awaiting);
if (errs.Any(err => err.HasErrors))
{
return(errs.Aggregate(new ErrorContainer(), (acc, err) => acc + err.Errors));
}
}
// Last depth group should be count of 1, the new root TreeNode:
Debug.Assert(lastNodes.Count == 1);
return(new TreeTree(lastNodes[0].Item2.ID, new ImmutableContainer <TreeID, TreeNode>(result)));
}
private async Task <Errorable <TreeNode> > getTree(TreeID id)
{
FileInfo fi = system.getPathByID(id);
if (!fi.Exists)
{
return(new TreeIDRecordDoesNotExistError(id));
}
byte[] buf;
int nr = 0;
using (var fs = new FileStream(fi.FullName, FileMode.Open, FileAccess.Read, FileShare.Read, 16384, true))
{
// TODO: implement an async buffered Stream:
buf = new byte[16384];
nr = await fs.ReadAsync(buf, 0, 16384).ConfigureAwait(continueOnCapturedContext: false);
if (nr >= 16384)
{
// My, what a large tree you have!
throw new NotSupportedException();
}
}
TreeNode.Builder tb = new TreeNode.Builder(new List <TreeTreeReference>(), new List <TreeBlobReference>());
// Parse the Tree:
using (var ms = new MemoryStream(buf, 0, nr, false))
using (var sr = new StreamReader(ms, Encoding.UTF8))
{
string line;
while ((line = sr.ReadLine()) != null)
{
if (line.StartsWith("tree "))
{
string linked_treeid = line.Substring(5, (TreeID.ByteArrayLength * 2));
string name = line.Substring(6 + (TreeID.ByteArrayLength * 2));
// Attempt to parse the TreeID and verify its existence:
Errorable <TreeID> trid = TreeID.TryParse(linked_treeid);
if (trid.HasErrors)
{
return(trid.Errors);
}
if (!system.getPathByID(trid.Value).Exists)
{
return(new TreeIDRecordDoesNotExistError(trid.Value));
}
tb.Trees.Add(new TreeTreeReference.Builder(name, trid.Value));
}
else if (line.StartsWith("blob "))
{
string linked_blobid = line.Substring(5, (TreeID.ByteArrayLength * 2));
string name = line.Substring(6 + (TreeID.ByteArrayLength * 2));
// Attempt to parse the BlobID and verify its existence:
Errorable <BlobID> blid = BlobID.TryParse(linked_blobid);
if (blid.HasErrors)
{
return(blid.Errors);
}
if (!system.getPathByID(blid.Value).Exists)
{
return(new BlobIDRecordDoesNotExistError(blid.Value));
}
tb.Blobs.Add(new TreeBlobReference.Builder(name, blid.Value));
}
}
}
// Create the immutable Tree from the Builder:
TreeNode tr = tb;
// Validate the computed TreeID:
if (tr.ID != id)
{
return(new ComputedTreeIDMismatchError(tr.ID, id));
}
return(tr);
}
