public async Task <Errorable <TreeIDPathMapping>[]> GetTreeIDsByPaths(params TreeTreePath[] paths)
{
// Since we cannot query the database with multiple root TreeIDs at once, group all the
// paths that share the same root TreeID together and submit those in one query.
var rootTreeIDGroups = from path in paths group path by path.RootTreeID;
var tasks = new List <Task <ReadOnlyCollection <Errorable <TreeIDPathMapping> > > >(capacity: paths.Length);
using (var en = rootTreeIDGroups.GetEnumerator())
{
for (int i = 0; en.MoveNext(); ++i)
{
TreeID rootid = en.Current.Key;
CanonicalTreePath[] treePaths = en.Current.Select(tr => tr.Path).ToArray();
tasks.Add(db.ExecuteSingleQueryAsync(new QueryTreeIDsByPaths(rootid, treePaths)));
}
}
// Wait for all the queries to come back:
var allTreeIDs = await Task.WhenAll(tasks);
// Flatten out the array-of-arrays:
var flattenedTreeIDs =
from trArr in allTreeIDs
from tr in trArr
select tr;
return(flattenedTreeIDs.ToArray(paths.Length));
}
internal static void RecursivePrint(TreeID treeID, ImmutableContainer <TreeID, TreeNode> trees, int depth = 1)
{
TreeNode tr;
if (!trees.TryGetValue(treeID, out tr))
{
return;
}
if (depth == 1)
{
Console.WriteLine("tree {1}: {0}/", new string('_', (depth - 1) * 2), tr.ID.ToString(firstLength: 7));
}
// Sort refs by name:
var namedRefs = TreeNode.ComputeChildList(tr.Trees, tr.Blobs);
foreach (var kv in namedRefs)
{
var nref = kv.Value;
switch (nref.Which)
{
case Either <TreeTreeReference, TreeBlobReference> .Selected.Left:
Console.WriteLine("tree {1}: {0}{2}/", new string('_', depth * 2), nref.Left.TreeID.ToString(firstLength: 7), nref.Left.Name);
RecursivePrint(nref.Left.TreeID, trees, depth + 1);
break;
case Either <TreeTreeReference, TreeBlobReference> .Selected.Right:
Console.WriteLine("blob {1}: {0}{2}", new string('_', depth * 2), nref.Right.BlobID.ToString(firstLength: 7), nref.Right.Name);
break;
}
}
}
async Task <CommitID> TestCreateCommit(TreeID treeid, int num)
{
var db = getDataContext();
ICommitRepository cmrepo = new CommitRepository(db);
IRefRepository rfrepo = new RefRepository(db);
Tuple <Ref, Commit> parent = await cmrepo.GetCommitByRef("HEAD");
Commit cm = new Commit.Builder(
pParents: parent == null ? new List <CommitID>(0) : new List <CommitID>(1)
{
parent.Item2.ID
},
pTreeID: treeid,
pCommitter: "James Dunne <*****@*****.**>",
pDateCommitted: DateTimeOffset.Parse("2011-08-29 00:00:00 -0500"),
pMessage: "Commit #" + num.ToString() + "."
);
Console.WriteLine("CommitID {0}", cm.ID);
// Persist the commit:
await cmrepo.PersistCommit(cm);
// Once the commit is persisted, update HEAD ref:
await rfrepo.PersistRef(new Ref.Builder("HEAD", cm.ID));
return(cm.ID);
}
internal FileInfo getPathByID(TreeID id)
{
DirectoryInfo objDir = getObjectsDirectory();
string idStr = id.ToString();
string path = System.IO.Path.Combine(objDir.FullName, idStr.Substring(0, 2), idStr.Substring(2));
return(new FileInfo(path));
}
private async Task <Errorable <Stage> > getStageByName(StageName stageName)
{
FileInfo fiTracker = system.getStagePathByStageName(stageName);
if (!fiTracker.Exists)
{
return(new StageNameDoesNotExistError(stageName));
}
byte[] buf;
int nr = 0;
using (var fs = new FileStream(fiTracker.FullName, FileMode.Open, FileAccess.Read, FileShare.Read, 16384, true))
{
// TODO: implement an async buffered Stream:
buf = new byte[16384];
nr = await fs.ReadAsync(buf, 0, 16384).ConfigureAwait(continueOnCapturedContext: false);
if (nr >= 16384)
{
// My, what a large tag you have!
throw new NotSupportedException();
}
}
Stage.Builder sb = new Stage.Builder();
// Parse the Stage:
using (var ms = new MemoryStream(buf, 0, nr, false))
using (var sr = new StreamReader(ms, Encoding.UTF8))
{
string line = sr.ReadLine();
// Set Name:
line = sr.ReadLine();
if (line == null || !line.StartsWith("name "))
{
return(new StageParseExpectedNameError());
}
sb.Name = (StageName)line.Substring("name ".Length);
// Set TreeID:
if (line == null || !line.StartsWith("tree "))
{
return(new StageParseExpectedTreeError());
}
var ecid = TreeID.TryParse(line.Substring("tree ".Length));
if (ecid.HasErrors)
{
return(ecid.Errors);
}
sb.TreeID = ecid.Value;
return((Stage)sb);
}
}
public Task <Errorable <TreeNode>[]> GetTrees(params TreeID[] ids)
{
Task <Errorable <TreeNode> >[] tasks = new Task <Errorable <TreeNode> > [ids.Length];
for (int i = 0; i < ids.Length; ++i)
{
TreeID id = ids[i];
tasks[i] = db.ExecuteSingleQueryAsync(new QueryTree(id));
}
return(Task.WhenAll(tasks));
}
public Task <Errorable <TreeNode>[]> GetTrees(params TreeID[] ids)
{
Task <Errorable <TreeNode> >[] tasks = new Task <Errorable <TreeNode> > [ids.Length];
for (int i = 0; i < ids.Length; ++i)
{
TreeID id = ids[i];
tasks[i] = getTree(id);
}
return(Task.WhenAll(tasks));
}
private void deleteTree(TreeID id)
{
lock (FileSystem.SystemLock)
{
FileInfo fi = system.getPathByID(id);
if (fi.Exists)
{
fi.Delete();
}
}
}
private static TreeResponse projectTreeJSON(TreeID rootid, ImmutableContainer<TreeID, TreeNode> trees)
{
TreeNode tree;
if (!trees.TryGetValue(rootid, out tree)) return null;
return new TreeResponse
{
id = tree.ID.ToString(),
blobs = tree.Blobs.SelectAsArray(bl => new TreeBlobRefResponse { name = bl.Name, blobid = bl.BlobID.ToString() }),
trees = tree.Trees.SelectAsArray(tr => new TreeTreeRefResponse { name = tr.Name, treeid = tr.TreeID.ToString(), tree = projectTreeJSON(tr.TreeID, trees) })
};
}
async Task TestRetrieveTreeRecursively(TreeID rootid)
{
var db = getDataContext();
Console.WriteLine("Retrieving TreeID {0} recursively...", rootid);
ITreeRepository repo = new TreeRepository(db);
var tree = await repo.GetTreeRecursively(rootid);
// Recursively display trees:
RecursivePrint(tree.Item1, tree.Item2);
}
private async Task <Errorable <TreeIDPathMapping> > getTreeIDByPath(TreeNode root, TreeTreePath path)
{
ReadOnlyCollection <string> parts = path.Path.Parts;
if (parts.Count == 0)
{
return(new TreeIDPathMapping(path, path.RootTreeID));
}
int j = 0;
// Start descending into child nodes:
TreeNode node = root, nextNode;
while (node != null)
{
nextNode = null;
// Check all child nodes against the next name in the path:
for (int i = 0; i < node.Trees.Length; ++i)
{
// TODO: specific string comparison logic!
if (parts[j] == node.Trees[i].Name)
{
TreeID nextID = node.Trees[i].TreeID;
// Run out of path components? This is it!
if (++j == parts.Count)
{
return(new TreeIDPathMapping(path, (TreeID?)nextID));
}
// Load up the next node so we can scan through its child nodes:
var enextNode = await getTree(nextID).ConfigureAwait(continueOnCapturedContext: false);
if (enextNode.HasErrors)
{
return(enextNode.Errors);
}
nextNode = enextNode.Value;
break;
}
}
// Attempt to continue:
node = nextNode;
}
// If we got here it means that we didn't find what we were looking for:
return(new TreeIDPathMapping(path, (TreeID?)null));
}
public Task <Errorable <TreeID> > ResolvePartialID(TreeID.Partial id)
{
FileInfo[] fis = system.getPathsByPartialID(id);
if (fis.Length == 1)
{
return(Task.FromResult(TreeID.TryParse(id.ToString().Substring(0, 2) + fis[0].Name)));
}
if (fis.Length == 0)
{
return(Task.FromResult((Errorable <TreeID>) new TreeIDPartialNoResolutionError(id)));
}
return(Task.FromResult((Errorable <TreeID>) new TreeIDPartialAmbiguousResolutionError(id, fis.SelectAsArray(f => TreeID.TryParse(id.ToString().Substring(0, 2) + f.Name).Value))));
}
private async Task createTrees()
{
PersistingBlob pbHeader = new PersistingBlob("<div>Header</div>".ToStream());
sblobs = await blrepo.PersistBlobs(pbHeader);
trSection1 = new TreeNode.Builder();
trSection2 = new TreeNode.Builder();
trSection3 = new TreeNode.Builder();
trImages = new TreeNode.Builder();
trCSS = new TreeNode.Builder();
trJS = new TreeNode.Builder();
trPages = new TreeNode.Builder(
new List <TreeTreeReference> {
new TreeTreeReference.Builder("section1", trSection1.ID),
new TreeTreeReference.Builder("section2", trSection2.ID),
new TreeTreeReference.Builder("section3", trSection3.ID)
},
new List <TreeBlobReference>(0)
);
trContent = new TreeNode.Builder(
new List <TreeTreeReference> {
new TreeTreeReference.Builder("images", trImages.ID),
new TreeTreeReference.Builder("css", trCSS.ID),
new TreeTreeReference.Builder("js", trJS.ID)
},
new List <TreeBlobReference>(0)
);
trTemplate = new TreeNode.Builder(
new List <TreeTreeReference>(0),
new List <TreeBlobReference> {
new TreeBlobReference.Builder("header", sblobs[0].Value.ID)
}
);
trRoot = new TreeNode.Builder(
new List <TreeTreeReference> {
new TreeTreeReference.Builder("template", trTemplate.ID),
new TreeTreeReference.Builder("content", trContent.ID),
new TreeTreeReference.Builder("pages", trPages.ID)
},
new List <TreeBlobReference>(0)
);
rootId = trRoot.ID;
trees = new ImmutableContainer <TreeID, TreeNode>(tr => tr.ID, trSection1, trSection2, trSection3, trPages, trImages, trCSS, trJS, trContent, trTemplate, trRoot);
RecursivePrint(trRoot.ID, trees);
}
private async Task <Errorable <TreeNode[]> > getTreeRecursively(TreeID id)
{
var eroot = await getTree(id).ConfigureAwait(continueOnCapturedContext: false);
if (eroot.HasErrors)
{
return(eroot.Errors);
}
var root = eroot.Value;
var rootArr = new TreeNode[1] {
root
};
if (root.Trees.Length == 0)
{
return(rootArr);
}
Task <Errorable <TreeNode[]> >[] tasks = new Task <Errorable <TreeNode[]> > [root.Trees.Length];
for (int i = 0; i < root.Trees.Length; ++i)
{
tasks[i] = getTreeRecursively(root.Trees[i].TreeID);
}
// Await all the tree retrievals:
var allTrees = await Task.WhenAll(tasks).ConfigureAwait(continueOnCapturedContext: false);
// Roll up all the errors:
ErrorContainer errors =
(
from etrs in allTrees
where etrs.HasErrors
select etrs.Errors
).Aggregate(new ErrorContainer(), (acc, err) => acc + err);
if (errors.HasAny)
{
return(errors);
}
// Flatten out the tree arrays:
var flattened =
from etrs in allTrees
from tr in etrs.Value
select tr;
// Return the final array:
return(rootArr.Concat(flattened).ToArray(allTrees.Sum(ta => ta.Value.Length) + 1));
}
async Task Create3DeepCommit()
{
// Create a 3-depth commit tree up from HEAD:
for (int i = 0; i < 3; ++i)
{
TreeID rootid = await TestPersistTree();
TestRetrieveTreeRecursively(rootid).Wait();
CommitID cmid = await TestCreateCommit(rootid, i);
TestCreateTag(cmid).Wait();
}
}
public Builder(
List<CommitID> pParents
,TreeID pTreeID
,string pCommitter
,DateTimeOffset pDateCommitted
,string pMessage
)
{
this.Parents = pParents;
this.TreeID = pTreeID;
this.Committer = pCommitter;
this.DateCommitted = pDateCommitted;
this.Message = pMessage;
}
public Builder(
CommitID pID
, TreeID pTreeID
, string pCommitter
, DateTimeOffset pDateCommitted
, string pMessage
)
{
this.ID = pID;
this.TreeID = pTreeID;
this.Committer = pCommitter;
this.DateCommitted = pDateCommitted;
this.Message = pMessage;
}
public async Task <Errorable <TreeTree> > GetTreeRecursively(TreeID rootid)
{
// Get all trees recursively:
var eall = await getTreeRecursively(rootid).ConfigureAwait(continueOnCapturedContext: false);
if (eall.HasErrors)
{
return(eall.Errors);
}
TreeNode[] all = eall.Value;
// Return them (all[0] is the root):
return(new TreeTree(all[0].ID, new ImmutableContainer <TreeID, TreeNode>(tr => tr.ID, all)));
}
protected override async Task <Action <AsyncCodeActivityContext> > ExecuteAsync(AsyncCodeActivityContext context, CancellationToken cancellationToken)
{
// Inputs
var treeid = TreeID.Get(context);
var treeitemkey = TreeItemKey.Get(context);
///////////////////////////
// Add execution logic HERE
SAPAuto objSAPAuto = new SAPAuto();
objSAPAuto.TreeView_Select(treeid, treeitemkey);
///////////////////////////
// Outputs
return((ctx) => {
});
}
public async Task <Errorable <TreeID> > DeleteTreeRecursively(TreeID rootid)
{
var etrees = await getTreeRecursively(rootid).ConfigureAwait(continueOnCapturedContext: false);
if (etrees.HasErrors)
{
return(etrees.Errors);
}
var trees = etrees.Value;
// TODO: test that 'tr' is captured properly in the lambda.
Task[] tasks = trees.SelectAsArray(tr => Task.Run(() => deleteTree(tr.ID)));
await Task.WhenAll(tasks).ConfigureAwait(continueOnCapturedContext: false);
return(rootid);
}
public Errorable <TreeNode> retrieve(SqlCommand cmd, SqlDataReader dr, int expectedCapacity = 10)
{
TreeNode.Builder tb = new TreeNode.Builder(new List <TreeTreeReference>(), new List <TreeBlobReference>());
// Read the TreeTreeReferences:
while (dr.Read())
{
var name = dr.GetSqlString(0).Value;
var linked_treeid = (TreeID)dr.GetSqlBinary(1).Value;
tb.Trees.Add(new TreeTreeReference.Builder(name, linked_treeid));
}
if (!dr.NextResult())
{
return(new TreeTreePathDoesNotExistError(this._path));
}
// Read the TreeBlobReferences:
while (dr.Read())
{
var name = dr.GetSqlString(0).Value;
var linked_blobid = (BlobID)dr.GetSqlBinary(1).Value.ToArray(20);
tb.Blobs.Add(new TreeBlobReference.Builder(name, linked_blobid));
}
TreeNode tr = tb;
TreeID retrievedId = (TreeID)((SqlBinary)cmd.Parameters["@treeid"].SqlValue).Value;
if (tr.ID != retrievedId)
{
return(new ComputedTreeIDMismatchError(tr.ID, retrievedId));
}
return(tr);
}
public Builder(
StageName pName
,TreeID pTreeID
)
{
this.Name = pName;
this.TreeID = pTreeID;
}
public Stage(Builder b)
{
this.Name = b.Name;
this.TreeID = b.TreeID;
}
public Builder(TreeTreeReference imm)
{
this.Name = imm.Name;
this.TreeID = imm.TreeID;
}
public Builder(
CommitID pID
,TreeID pTreeID
,string pCommitter
,DateTimeOffset pDateCommitted
,string pMessage
)
{
this.ID = pID;
this.TreeID = pTreeID;
this.Committer = pCommitter;
this.DateCommitted = pDateCommitted;
this.Message = pMessage;
}
public QueryTreeRecursively(TreeID id)
{
this._id = id;
}
public ComputedTreeIDMismatchError(TreeID computedID, TreeID expectedID) : base("Computed TreeID {0} does not match expected TreeID {1}", computedID, expectedID) { }
public Task<Errorable<TreeTree>> GetTreeRecursively(TreeID rootid)
{
return db.ExecuteListQueryAsync(new QueryTreeRecursively(rootid));
}
internal FileInfo[] getPathsByPartialID(TreeID.Partial partial)
{
DirectoryInfo objDir = getObjectsDirectory();
string idStr = partial.ToString();
string path = System.IO.Path.Combine(objDir.FullName, idStr.Substring(0, 2));
var di = new DirectoryInfo(path);
if (!di.Exists) return new FileInfo[0];
return di.GetFiles(idStr.Substring(2) + "*");
}
internal FileInfo getPathByID(TreeID id)
{
DirectoryInfo objDir = getObjectsDirectory();
string idStr = id.ToString();
string path = System.IO.Path.Combine(objDir.FullName, idStr.Substring(0, 2), idStr.Substring(2));
return new FileInfo(path);
}
private async Task <Errorable <TreeNode> > getTree(TreeID id)
{
FileInfo fi = system.getPathByID(id);
if (!fi.Exists)
{
return(new TreeIDRecordDoesNotExistError(id));
}
byte[] buf;
int nr = 0;
using (var fs = new FileStream(fi.FullName, FileMode.Open, FileAccess.Read, FileShare.Read, 16384, true))
{
// TODO: implement an async buffered Stream:
buf = new byte[16384];
nr = await fs.ReadAsync(buf, 0, 16384).ConfigureAwait(continueOnCapturedContext: false);
if (nr >= 16384)
{
// My, what a large tree you have!
throw new NotSupportedException();
}
}
TreeNode.Builder tb = new TreeNode.Builder(new List <TreeTreeReference>(), new List <TreeBlobReference>());
// Parse the Tree:
using (var ms = new MemoryStream(buf, 0, nr, false))
using (var sr = new StreamReader(ms, Encoding.UTF8))
{
string line;
while ((line = sr.ReadLine()) != null)
{
if (line.StartsWith("tree "))
{
string linked_treeid = line.Substring(5, (TreeID.ByteArrayLength * 2));
string name = line.Substring(6 + (TreeID.ByteArrayLength * 2));
// Attempt to parse the TreeID and verify its existence:
Errorable <TreeID> trid = TreeID.TryParse(linked_treeid);
if (trid.HasErrors)
{
return(trid.Errors);
}
if (!system.getPathByID(trid.Value).Exists)
{
return(new TreeIDRecordDoesNotExistError(trid.Value));
}
tb.Trees.Add(new TreeTreeReference.Builder(name, trid.Value));
}
else if (line.StartsWith("blob "))
{
string linked_blobid = line.Substring(5, (TreeID.ByteArrayLength * 2));
string name = line.Substring(6 + (TreeID.ByteArrayLength * 2));
// Attempt to parse the BlobID and verify its existence:
Errorable <BlobID> blid = BlobID.TryParse(linked_blobid);
if (blid.HasErrors)
{
return(blid.Errors);
}
if (!system.getPathByID(blid.Value).Exists)
{
return(new BlobIDRecordDoesNotExistError(blid.Value));
}
tb.Blobs.Add(new TreeBlobReference.Builder(name, blid.Value));
}
}
}
// Create the immutable Tree from the Builder:
TreeNode tr = tb;
// Validate the computed TreeID:
if (tr.ID != id)
{
return(new ComputedTreeIDMismatchError(tr.ID, id));
}
return(tr);
}
public async Task <Errorable <TreeNode> > PersistTree(TreeID rootid, ImmutableContainer <TreeID, TreeNode> trees)
{
if (trees == null)
{
throw new ArgumentNullException("trees");
}
// TODO: better return value than `null`
if (trees.Count == 0)
{
return((TreeNode)null);
}
// This code scans the tree breadth-first and builds a reversed depth-ordered stack:
var reverseDepthOrder = new { id = rootid, depth = 0 }.StackOf(trees.Count);
reverseDepthOrder.Pop();
var breadthFirstQueue = new { id = rootid, depth = 0 }.QueueOf(trees.Count);
while (breadthFirstQueue.Count > 0)
{
var curr = breadthFirstQueue.Dequeue();
// Add it to the reverse stack:
reverseDepthOrder.Push(curr);
TreeNode node;
if (!trees.TryGetValue(curr.id, out node))
{
// TODO: didn't find the TreeID in the given collection, assume already persisted?
continue;
}
// Queue up the child TreeIDs:
foreach (var trtr in node.Trees)
{
breadthFirstQueue.Enqueue(new { id = trtr.TreeID, depth = curr.depth + 1 });
}
}
// This code takes the reverse depth-ordered stack and persists the tree nodes in groups per depth level.
// This ensures that all child nodes across the breadth of the tree at each depth level are persisted
// before moving up to their parents.
List <Task <Errorable <TreeNode> > > persistTasks = new List <Task <Errorable <TreeNode> > >();
HashSet <TreeID> isPersisting = new HashSet <TreeID>();
int lastDepth = reverseDepthOrder.Peek().depth;
foreach (var curr in reverseDepthOrder)
{
Debug.WriteLine(String.Format("{0}: {1}", curr.depth, curr.id.ToString(firstLength: 7)));
// An invariant of the algorithm, enforced via assert:
Debug.Assert(curr.depth <= lastDepth);
// Did we move to the next depth group:
if (curr.depth != lastDepth)
{
Debug.WriteLine(String.Format("Awaiting depth group {0}...", lastDepth));
// Wait for the last depth group to finish persisting:
await Task.WhenAll(persistTasks);
// TODO: roll up errors!
// Start a new depth group:
persistTasks = new List <Task <Errorable <TreeNode> > >();
}
// Don't re-persist the same TreeID (this is a legit case - the same TreeID may be seen in different nodes of the tree):
if (isPersisting.Contains(curr.id))
{
Debug.WriteLine(String.Format("Already persisting {0}", curr.id.ToString(firstLength: 7)));
// Keep track of the last depth level:
lastDepth = curr.depth;
continue;
}
// Get the TreeNode and persist it:
TreeNode node = trees[curr.id];
isPersisting.Add(curr.id);
// Fire up a task to persist this tree node:
var tsk = Task.Run(() => persistTree(node));
// Add the task to the depth group to await:
Debug.WriteLine(String.Format("Adding to depth group {0}...", curr.depth));
persistTasks.Add(tsk);
// Keep track of the last depth level:
lastDepth = curr.depth;
}
// The final depth group should be depth 0 with at most 1 element: the root node.
Debug.Assert(lastDepth == 0);
if (persistTasks.Count > 0)
{
// Await the last group (the root node):
Debug.WriteLine(String.Format("Awaiting depth group {0}...", lastDepth));
await Task.WhenAll(persistTasks);
}
// Return the root TreeNode:
return(persistTasks[0].Result);
}
public TreeTree(TreeID rootID, ImmutableContainer<TreeID, TreeNode> trees)
{
this.RootID = rootID;
this.Trees = trees;
}
public Task<Errorable<TreeNode>> GetTree(TreeID id)
{
return db.ExecuteSingleQueryAsync(new QueryTree(id));
}
public Task <Errorable <TreeNode> > GetTree(TreeID id)
{
return(db.ExecuteSingleQueryAsync(new QueryTree(id)));
}
public async Task<Errorable<TreeNode>> PersistTree(TreeID rootid, ImmutableContainer<TreeID, TreeNode> trees)
{
if (trees == null) throw new ArgumentNullException("trees");
// TODO: better return value than `null`
if (trees.Count == 0) return (TreeNode)null;
// Start a query to check what Trees exist already:
var existTrees = await db.ExecuteListQueryAsync(new QueryTreesExist(trees.Keys), expectedCapacity: trees.Count);
// This code scans the tree breadth-first and builds a reversed depth-ordered stack:
var reverseDepthOrder = new { id = rootid, depth = 0 }.StackOf(trees.Count);
reverseDepthOrder.Pop();
var breadthFirstQueue = new { id = rootid, depth = 0 }.QueueOf(trees.Count);
while (breadthFirstQueue.Count > 0)
{
var curr = breadthFirstQueue.Dequeue();
// Add it to the reverse stack:
reverseDepthOrder.Push(curr);
TreeNode node;
if (!trees.TryGetValue(curr.id, out node))
{
// TODO: didn't find the TreeID in the given collection, assume already persisted?
continue;
}
// Queue up the child TreeIDs:
foreach (var trtr in node.Trees)
breadthFirstQueue.Enqueue(new { id = trtr.TreeID, depth = curr.depth + 1 });
}
// This code takes the reverse depth-ordered stack and persists the tree nodes in groups per depth level.
// This ensures that all child nodes across the breadth of the tree at each depth level are persisted
// before moving up to their parents.
List<Task<Errorable<TreeNode>>> persistTasks = new List<Task<Errorable<TreeNode>>>();
// Initialize the `isPersisting` set with the set of TreeIDs that already exist.
HashSet<TreeID> isPersisting = new HashSet<TreeID>(existTrees);
int lastDepth = reverseDepthOrder.Peek().depth;
foreach (var curr in reverseDepthOrder)
{
Debug.WriteLine(String.Format("{0}: {1}", curr.depth, curr.id.ToString(firstLength: 7)));
// An invariant of the algorithm, enforced via assert:
Debug.Assert(curr.depth <= lastDepth);
// Did we move to the next depth group:
if ((persistTasks.Count > 0) && (curr.depth != lastDepth))
{
Debug.WriteLine(String.Format("Awaiting depth group {0}...", lastDepth));
// Wait for the last depth group to finish persisting:
await Task.WhenAll(persistTasks);
// Start a new depth group:
persistTasks = new List<Task<Errorable<TreeNode>>>();
}
// Don't re-persist the same TreeID (this is a legit case - the same TreeID may be seen in different nodes of the tree):
if (isPersisting.Contains(curr.id))
{
Debug.WriteLine(String.Format("Already persisting {0}", curr.id.ToString(firstLength: 7)));
// Keep track of the last depth level:
lastDepth = curr.depth;
continue;
}
// Get the TreeNode and persist it:
TreeNode node = trees[curr.id];
isPersisting.Add(curr.id);
// Fire up a task to persist this tree node:
var tsk = db.ExecuteNonQueryAsync(new PersistTree(node));
// Add the task to the depth group to await:
Debug.WriteLine(String.Format("Adding to depth group {0}...", curr.depth));
persistTasks.Add(tsk);
// Keep track of the last depth level:
lastDepth = curr.depth;
}
Debug.Assert(lastDepth == 0);
if (persistTasks.Count > 0)
{
// Await the last group (the root node):
Debug.WriteLine(String.Format("Awaiting depth group {0}...", lastDepth));
await Task.WhenAll(persistTasks);
}
// Return the root TreeNode:
return trees[rootid];
}
public QueryTree(TreeID id)
{
this._id = id;
}
public TreeRecordAlreadyExistsError(TreeID id) : base("A tree with TreeID {0} already exists", id)
{
}
public Errorable <TreeTree> retrieve(SqlCommand cmd, SqlDataReader dr, int expectedCapacity = 10)
{
Dictionary <TreeID, TreeNode.Builder> trees = new Dictionary <TreeID, TreeNode.Builder>(expectedCapacity);
// Iterate through rows of the recursive query, assuming ordering of rows guarantees tree depth locality.
while (dr.Read())
{
SqlBinary btreeid = dr.GetSqlBinary(0);
SqlBinary blinked_treeid = dr.GetSqlBinary(1);
SqlString treename = dr.GetSqlString(2);
SqlBinary linked_blobid = dr.GetSqlBinary(3);
SqlString blobname = dr.GetSqlString(4);
TreeID?treeid = btreeid.IsNull ? (TreeID?)null : (TreeID)btreeid.Value;
TreeID?linked_treeid = blinked_treeid.IsNull ? (TreeID?)null : (TreeID)blinked_treeid.Value;
TreeID pullFor = treeid.HasValue ? treeid.Value : linked_treeid.Value;
TreeNode.Builder curr;
if (!trees.TryGetValue(pullFor, out curr))
{
curr = new TreeNode.Builder(new List <TreeTreeReference>(), new List <TreeBlobReference>());
trees.Add(pullFor, curr);
}
// The tree to add the blob link to:
TreeNode.Builder blobTree = curr;
// Add a tree link:
if (treeid.HasValue && linked_treeid.HasValue)
{
// Create the Tree.Builder for the linked_treeid if it does not exist:
if (!trees.TryGetValue(linked_treeid.Value, out blobTree))
{
blobTree = new TreeNode.Builder(new List <TreeTreeReference>(), new List <TreeBlobReference>());
trees.Add(linked_treeid.Value, blobTree);
}
List <TreeTreeReference> treeRefs = curr.Trees;
bool isDupe = false;
if (treeRefs.Count > 0)
{
// Only check the previous ref record for dupe:
// TODO: verify that SQL Server will *always* return rows in an order that supports depth locality from a recursive CTE.
isDupe = (
(treeRefs[treeRefs.Count - 1].TreeID == linked_treeid.Value) &&
(treeRefs[treeRefs.Count - 1].Name == treename.Value)
);
}
// Don't re-add the same tree link:
if (!isDupe)
{
treeRefs.Add(new TreeTreeReference.Builder(treename.Value, linked_treeid.Value));
}
}
// Add a blob link to the child or parent tree:
if (!linked_blobid.IsNull)
{
blobTree.Blobs.Add(new TreeBlobReference.Builder(blobname.Value, (BlobID)linked_blobid.Value.ToArray(20)));
}
}
List <TreeNode> finals = new List <TreeNode>(trees.Count);
foreach (KeyValuePair <TreeID, TreeNode.Builder> pair in trees)
{
TreeNode tr = pair.Value;
if (tr.ID != pair.Key)
{
return(new ComputedTreeIDMismatchError(tr.ID, pair.Key));
}
finals.Add(tr);
}
// Return the final result with immutable objects:
return(new TreeTree(this._id, new ImmutableContainer <TreeID, TreeNode>(tr => tr.ID, finals)));
}
public QueryTreeIDsByPaths(TreeID rootid, params CanonicalTreePath[] paths)
{
this._rootid = rootid;
this._paths = paths;
}
public TreeTreeReference(Builder b)
{
this.Name = b.Name;
this.TreeID = b.TreeID;
}
public TreeIDPartialAmbiguousResolutionError(TreeID.Partial id, params TreeID[] ids) : base("Partial TreeID {0} resolves to multiple TreeID", id, ids) { }
public Builder(
string pName
,TreeID pTreeID
)
{
this.Name = pName;
this.TreeID = pTreeID;
}
public TreeIDPartialNoResolutionError(TreeID.Partial id) : base("Partial TreeID {0} does not resolve to a TreeID", id) { }
public Builder(Stage imm)
{
this.Name = imm.Name;
this.TreeID = imm.TreeID;
}
public async Task<Errorable<TreeTree>> GetTreeRecursively(TreeID rootid)
{
// Get all trees recursively:
var eall = await getTreeRecursively(rootid).ConfigureAwait(continueOnCapturedContext: false);
if (eall.HasErrors) return eall.Errors;
TreeNode[] all = eall.Value;
// Return them (all[0] is the root):
return new TreeTree(all[0].ID, new ImmutableContainer<TreeID, TreeNode>(tr => tr.ID, all));
}
public Task<Errorable<TreeID>> ResolvePartialID(TreeID.Partial id)
{
FileInfo[] fis = system.getPathsByPartialID(id);
if (fis.Length == 1) return Task.FromResult(TreeID.TryParse(id.ToString().Substring(0, 2) + fis[0].Name));
if (fis.Length == 0) return Task.FromResult((Errorable<TreeID>)new TreeIDPartialNoResolutionError(id));
return Task.FromResult((Errorable<TreeID>)new TreeIDPartialAmbiguousResolutionError(id, fis.SelectAsArray(f => TreeID.TryParse(id.ToString().Substring(0, 2) + f.Name).Value)));
}
public Task <Errorable <TreeNode> > GetTree(TreeID id)
{
return(getTree(id));
}
public Task<Errorable<TreeID>> DeleteTreeRecursively(TreeID rootid)
{
throw new NotImplementedException();
}
public TreeIDPathMapping(TreeTreePath path, TreeID? id)
{
this.Path = path;
this.TreeID = id;
}
public async Task<Errorable<TreeID>> ResolvePartialID(TreeID.Partial id)
{
var resolvedIDs = await db.ExecuteListQueryAsync(new ResolvePartialTreeID(id));
if (resolvedIDs.Length == 1) return resolvedIDs[0];
if (resolvedIDs.Length == 0) return new TreeIDPartialNoResolutionError(id);
return new TreeIDPartialAmbiguousResolutionError(id, resolvedIDs);
}
private void deleteTree(TreeID id)
{
lock (FileSystem.SystemLock)
{
FileInfo fi = system.getPathByID(id);
if (fi.Exists) fi.Delete();
}
}
public TreeIDRecordDoesNotExistError(TreeID treeID) : base("A tree with TreeID {0} does not exist", treeID)
{
}
private async Task<Errorable<TreeNode[]>> getTreeRecursively(TreeID id)
{
var eroot = await getTree(id).ConfigureAwait(continueOnCapturedContext: false);
if (eroot.HasErrors) return eroot.Errors;
var root = eroot.Value;
var rootArr = new TreeNode[1] { root };
if (root.Trees.Length == 0)
return rootArr;
Task<Errorable<TreeNode[]>>[] tasks = new Task<Errorable<TreeNode[]>>[root.Trees.Length];
for (int i = 0; i < root.Trees.Length; ++i)
{
tasks[i] = getTreeRecursively(root.Trees[i].TreeID);
}
// Await all the tree retrievals:
var allTrees = await Task.WhenAll(tasks).ConfigureAwait(continueOnCapturedContext: false);
// Roll up all the errors:
ErrorContainer errors =
(
from etrs in allTrees
where etrs.HasErrors
select etrs.Errors
).Aggregate(new ErrorContainer(), (acc, err) => acc + err);
if (errors.HasAny) return errors;
// Flatten out the tree arrays:
var flattened =
from etrs in allTrees
from tr in etrs.Value
select tr;
// Return the final array:
return rootArr.Concat(flattened).ToArray(allTrees.Sum(ta => ta.Value.Length) + 1);
}
public ComputedTreeIDMismatchError(TreeID computedID, TreeID expectedID) : base("Computed TreeID {0} does not match expected TreeID {1}", computedID, expectedID)
{
}
private async Task<Errorable<TreeNode>> getTree(TreeID id)
{
FileInfo fi = system.getPathByID(id);
if (!fi.Exists) return new TreeIDRecordDoesNotExistError(id);
byte[] buf;
int nr = 0;
using (var fs = new FileStream(fi.FullName, FileMode.Open, FileAccess.Read, FileShare.Read, 16384, true))
{
// TODO: implement an async buffered Stream:
buf = new byte[16384];
nr = await fs.ReadAsync(buf, 0, 16384).ConfigureAwait(continueOnCapturedContext: false);
if (nr >= 16384)
{
// My, what a large tree you have!
throw new NotSupportedException();
}
}
TreeNode.Builder tb = new TreeNode.Builder(new List<TreeTreeReference>(), new List<TreeBlobReference>());
// Parse the Tree:
using (var ms = new MemoryStream(buf, 0, nr, false))
using (var sr = new StreamReader(ms, Encoding.UTF8))
{
string line;
while ((line = sr.ReadLine()) != null)
{
if (line.StartsWith("tree "))
{
string linked_treeid = line.Substring(5, (TreeID.ByteArrayLength * 2));
string name = line.Substring(6 + (TreeID.ByteArrayLength * 2));
// Attempt to parse the TreeID and verify its existence:
Errorable<TreeID> trid = TreeID.TryParse(linked_treeid);
if (trid.HasErrors) return trid.Errors;
if (!system.getPathByID(trid.Value).Exists) return new TreeIDRecordDoesNotExistError(trid.Value);
tb.Trees.Add(new TreeTreeReference.Builder(name, trid.Value));
}
else if (line.StartsWith("blob "))
{
string linked_blobid = line.Substring(5, (TreeID.ByteArrayLength * 2));
string name = line.Substring(6 + (TreeID.ByteArrayLength * 2));
// Attempt to parse the BlobID and verify its existence:
Errorable<BlobID> blid = BlobID.TryParse(linked_blobid);
if (blid.HasErrors) return blid.Errors;
if (!system.getPathByID(blid.Value).Exists) return new BlobIDRecordDoesNotExistError(blid.Value);
tb.Blobs.Add(new TreeBlobReference.Builder(name, blid.Value));
}
}
}
// Create the immutable Tree from the Builder:
TreeNode tr = tb;
// Validate the computed TreeID:
if (tr.ID != id) return new ComputedTreeIDMismatchError(tr.ID, id);
return tr;
}
public TreeBlobPath(TreeID rootTreeID, CanonicalBlobPath path)
{
this.RootTreeID = rootTreeID;
this.Path = path;
}
public Task<Errorable<TreeNode>> GetTree(TreeID id)
{
return getTree(id);
}
public TreePathStreamedBlob(TreeID rootTreeID, CanonicalBlobPath path, IStreamedBlob blob)
: this(new TreeBlobPath(rootTreeID, path), blob)
{
}
public async Task<Errorable<TreeID>> DeleteTreeRecursively(TreeID rootid)
{
var etrees = await getTreeRecursively(rootid).ConfigureAwait(continueOnCapturedContext: false);
if (etrees.HasErrors) return etrees.Errors;
var trees = etrees.Value;
// TODO: test that 'tr' is captured properly in the lambda.
Task[] tasks = trees.SelectAsArray(tr => Task.Run(() => deleteTree(tr.ID)));
await Task.WhenAll(tasks).ConfigureAwait(continueOnCapturedContext: false);
return rootid;
}
