internal static void RecursivePrint(TreeID treeID, ImmutableContainer <TreeID, TreeNode> trees, int depth = 1)
{
TreeNode tr;
if (!trees.TryGetValue(treeID, out tr))
{
return;
}
if (depth == 1)
{
Console.WriteLine("tree {1}: {0}/", new string('_', (depth - 1) * 2), tr.ID.ToString(firstLength: 7));
}
// Sort refs by name:
var namedRefs = TreeNode.ComputeChildList(tr.Trees, tr.Blobs);
foreach (var kv in namedRefs)
{
var nref = kv.Value;
switch (nref.Which)
{
case Either <TreeTreeReference, TreeBlobReference> .Selected.Left:
Console.WriteLine("tree {1}: {0}{2}/", new string('_', depth * 2), nref.Left.TreeID.ToString(firstLength: 7), nref.Left.Name);
RecursivePrint(nref.Left.TreeID, trees, depth + 1);
break;
case Either <TreeTreeReference, TreeBlobReference> .Selected.Right:
Console.WriteLine("blob {1}: {0}{2}", new string('_', depth * 2), nref.Right.BlobID.ToString(firstLength: 7), nref.Right.Name);
break;
}
}
}
private async Task createCommits()
{
PersistingBlob pblReadme = new PersistingBlob("Readme file.".ToStream());
var sblobs = await blrepo.PersistBlobs(pblReadme);
trRoot = new TreeNode.Builder(
new List <TreeTreeReference>(0),
new List <TreeBlobReference>
{
new TreeBlobReference.Builder("README", sblobs[0].Value.ID)
}
);
var trees = new ImmutableContainer <TreeID, TreeNode>(tr => tr.ID, trRoot);
await trrepo.PersistTree(trRoot.ID, trees);
TreeRepositoryTestMethods.RecursivePrint(trRoot.ID, trees);
cmRoot = new Commit.Builder(
new List <CommitID>(0),
trRoot.ID,
"James S. Dunne",
DateTimeOffset.Now,
"Hello world."
);
await cmrepo.PersistCommit(cmRoot);
}
async Task TestLargeBlobPersistence()
{
var db = getDataContext();
IBlobRepository blrepo = new BlobRepository(db);
Random rnd = new Random(0x5555AAC0);
const int count = 4000;
Console.WriteLine("Constructing {0} blobs...", count);
Blob[] bls = new Blob[count];
for (int i = 0; i < count; ++i)
{
byte[] c = new byte[8040];
rnd.NextBytes(c);
bls[i] = new Blob.Builder(c);
}
ImmutableContainer <BlobID, Blob> blobs = new ImmutableContainer <BlobID, Blob>(bl => bl.ID, bls);
Console.WriteLine("Persisting {0} blobs...", count);
Stopwatch sw = Stopwatch.StartNew();
ImmutableContainer <BlobID, Blob> pBlobs = await blrepo.PersistBlobs(blobs);
Console.WriteLine("Waiting...");
sw.Stop();
Console.WriteLine("Completed in {0} ms, {1} blobs/sec, {2} bytes/sec",
sw.ElapsedMilliseconds,
(double)count / (double)sw.ElapsedMilliseconds * 1000d,
(double)(count * 8040) / (double)sw.ElapsedMilliseconds * 1000d);
}
private static TreeResponse projectTreeJSON(TreeID rootid, ImmutableContainer<TreeID, TreeNode> trees)
{
TreeNode tree;
if (!trees.TryGetValue(rootid, out tree)) return null;
return new TreeResponse
{
id = tree.ID.ToString(),
blobs = tree.Blobs.SelectAsArray(bl => new TreeBlobRefResponse { name = bl.Name, blobid = bl.BlobID.ToString() }),
trees = tree.Trees.SelectAsArray(tr => new TreeTreeRefResponse { name = tr.Name, treeid = tr.TreeID.ToString(), tree = projectTreeJSON(tr.TreeID, trees) })
};
}
private async Task createTrees()
{
PersistingBlob pbHeader = new PersistingBlob("<div>Header</div>".ToStream());
sblobs = await blrepo.PersistBlobs(pbHeader);
trSection1 = new TreeNode.Builder();
trSection2 = new TreeNode.Builder();
trSection3 = new TreeNode.Builder();
trImages = new TreeNode.Builder();
trCSS = new TreeNode.Builder();
trJS = new TreeNode.Builder();
trPages = new TreeNode.Builder(
new List <TreeTreeReference> {
new TreeTreeReference.Builder("section1", trSection1.ID),
new TreeTreeReference.Builder("section2", trSection2.ID),
new TreeTreeReference.Builder("section3", trSection3.ID)
},
new List <TreeBlobReference>(0)
);
trContent = new TreeNode.Builder(
new List <TreeTreeReference> {
new TreeTreeReference.Builder("images", trImages.ID),
new TreeTreeReference.Builder("css", trCSS.ID),
new TreeTreeReference.Builder("js", trJS.ID)
},
new List <TreeBlobReference>(0)
);
trTemplate = new TreeNode.Builder(
new List <TreeTreeReference>(0),
new List <TreeBlobReference> {
new TreeBlobReference.Builder("header", sblobs[0].Value.ID)
}
);
trRoot = new TreeNode.Builder(
new List <TreeTreeReference> {
new TreeTreeReference.Builder("template", trTemplate.ID),
new TreeTreeReference.Builder("content", trContent.ID),
new TreeTreeReference.Builder("pages", trPages.ID)
},
new List <TreeBlobReference>(0)
);
rootId = trRoot.ID;
trees = new ImmutableContainer <TreeID, TreeNode>(tr => tr.ID, trSection1, trSection2, trSection3, trPages, trImages, trCSS, trJS, trContent, trTemplate, trRoot);
RecursivePrint(trRoot.ID, trees);
}
private CommitTreeResponse toJSON(CommitID id, ImmutableContainer<CommitID, ICommit> commits)
{
ICommit cm;
if (!commits.TryGetValue(id, out cm)) return null;
return new CommitTreeResponse()
{
id = cm.ID.ToString(),
treeid = cm.TreeID.ToString(),
committer = cm.Committer.ToString(),
date_committed = JSONTranslateExtensions.FromDate(cm.DateCommitted),
parents_retrieved = cm.IsComplete,
message = cm.Message,
parents = cm.Parents.SelectAsArray(cmid => toJSON(cmid, commits))
};
}
static void RecursivePrint(CommitID cmID, ImmutableContainer <CommitID, ICommit> commits, int depth = 1)
{
ICommit cm = commits[cmID];
if (cm.IsComplete)
{
Console.WriteLine("{0}c {1}: ({2})", new string(' ', (depth - 1) * 2), cm.ID.ToString(firstLength: 7), String.Join(",", cm.Parents.Select(id => id.ToString(firstLength: 7))));
foreach (CommitID parentID in cm.Parents)
{
RecursivePrint(parentID, commits, depth + 1);
}
}
else
{
Console.WriteLine("{0}p {1}: ?", new string(' ', (depth - 1) * 2), cm.ID.ToString(firstLength: 7));
}
}
void TestQueryBlobs()
{
// Create some Blobs:
Blob bl0 = new Blob.Builder(Encoding.UTF8.GetBytes("Sample README content."));
Console.WriteLine(bl0.ID.ToString());
Blob bl1 = new Blob.Builder(Encoding.UTF8.GetBytes("Sample content."));
Console.WriteLine(bl1.ID.ToString());
var blobs = new ImmutableContainer <BlobID, Blob>(bl => bl.ID, bl0, bl1);
var db = getDataContext();
// Check which blobs exist already:
var qBlobs = db.ExecuteListQueryAsync(new QueryBlobsExist(bl0.ID, bl1.ID), expectedCapacity: blobs.Count);
qBlobs.Wait();
// Find the blobs to persist:
var blobIDsToPersist = blobs.Keys.Except(qBlobs.Result).ToArray();
// Persist each blob asynchronously:
Task <Blob>[] persists = new Task <Blob> [blobIDsToPersist.Length];
for (int i = 0; i < blobIDsToPersist.Length; ++i)
{
BlobID id = blobIDsToPersist[i];
Console.WriteLine("PERSIST {0}", id.ToString());
persists[i] = db.ExecuteNonQueryAsync(new PersistBlob(blobs[id]));
}
Console.WriteLine("Waiting for persists...");
Task.WaitAll(persists);
Console.WriteLine("Complete.");
}
public async Task<ActionResult> CreateTree(TreeRequest tm)
{
TreeID root;
ImmutableContainer<TreeID, TreeNode> trees;
// Recursively convert the JSON-friendly `TreeModel` into our domain-friendly `Tree`s:
TreeNode[] treeArr = convertRecursively(tm);
root = treeArr[0].ID;
trees = new ImmutableContainer<TreeID, TreeNode>(tr => tr.ID, treeArr);
// Persist the tree:
var etree = await cms.trrepo.PersistTree(root, trees);
if (etree.HasErrors) return ErrorJson(etree);
// Return the `Tree` recursive model we persisted:
return Json(new { tree = projectTreeJSON(root, trees) });
}
public async Task<Errorable<TreeNode>> PersistTree(TreeID rootid, ImmutableContainer<TreeID, TreeNode> trees)
{
if (trees == null) throw new ArgumentNullException("trees");
// TODO: better return value than `null`
if (trees.Count == 0) return (TreeNode)null;
// Start a query to check what Trees exist already:
var existTrees = await db.ExecuteListQueryAsync(new QueryTreesExist(trees.Keys), expectedCapacity: trees.Count);
// This code scans the tree breadth-first and builds a reversed depth-ordered stack:
var reverseDepthOrder = new { id = rootid, depth = 0 }.StackOf(trees.Count);
reverseDepthOrder.Pop();
var breadthFirstQueue = new { id = rootid, depth = 0 }.QueueOf(trees.Count);
while (breadthFirstQueue.Count > 0)
{
var curr = breadthFirstQueue.Dequeue();
// Add it to the reverse stack:
reverseDepthOrder.Push(curr);
TreeNode node;
if (!trees.TryGetValue(curr.id, out node))
{
// TODO: didn't find the TreeID in the given collection, assume already persisted?
continue;
}
// Queue up the child TreeIDs:
foreach (var trtr in node.Trees)
breadthFirstQueue.Enqueue(new { id = trtr.TreeID, depth = curr.depth + 1 });
}
// This code takes the reverse depth-ordered stack and persists the tree nodes in groups per depth level.
// This ensures that all child nodes across the breadth of the tree at each depth level are persisted
// before moving up to their parents.
List<Task<Errorable<TreeNode>>> persistTasks = new List<Task<Errorable<TreeNode>>>();
// Initialize the `isPersisting` set with the set of TreeIDs that already exist.
HashSet<TreeID> isPersisting = new HashSet<TreeID>(existTrees);
int lastDepth = reverseDepthOrder.Peek().depth;
foreach (var curr in reverseDepthOrder)
{
Debug.WriteLine(String.Format("{0}: {1}", curr.depth, curr.id.ToString(firstLength: 7)));
// An invariant of the algorithm, enforced via assert:
Debug.Assert(curr.depth <= lastDepth);
// Did we move to the next depth group:
if ((persistTasks.Count > 0) && (curr.depth != lastDepth))
{
Debug.WriteLine(String.Format("Awaiting depth group {0}...", lastDepth));
// Wait for the last depth group to finish persisting:
await Task.WhenAll(persistTasks);
// Start a new depth group:
persistTasks = new List<Task<Errorable<TreeNode>>>();
}
// Don't re-persist the same TreeID (this is a legit case - the same TreeID may be seen in different nodes of the tree):
if (isPersisting.Contains(curr.id))
{
Debug.WriteLine(String.Format("Already persisting {0}", curr.id.ToString(firstLength: 7)));
// Keep track of the last depth level:
lastDepth = curr.depth;
continue;
}
// Get the TreeNode and persist it:
TreeNode node = trees[curr.id];
isPersisting.Add(curr.id);
// Fire up a task to persist this tree node:
var tsk = db.ExecuteNonQueryAsync(new PersistTree(node));
// Add the task to the depth group to await:
Debug.WriteLine(String.Format("Adding to depth group {0}...", curr.depth));
persistTasks.Add(tsk);
// Keep track of the last depth level:
lastDepth = curr.depth;
}
Debug.Assert(lastDepth == 0);
if (persistTasks.Count > 0)
{
// Await the last group (the root node):
Debug.WriteLine(String.Format("Awaiting depth group {0}...", lastDepth));
await Task.WhenAll(persistTasks);
}
// Return the root TreeNode:
return trees[rootid];
}
public TreeTree(TreeID rootID, ImmutableContainer<TreeID, TreeNode> trees)
{
this.RootID = rootID;
this.Trees = trees;
}
public async Task <Errorable <TreeNode> > PersistTree(TreeID rootid, ImmutableContainer <TreeID, TreeNode> trees)
{
if (trees == null)
{
throw new ArgumentNullException("trees");
}
// TODO: better return value than `null`
if (trees.Count == 0)
{
return((TreeNode)null);
}
// This code scans the tree breadth-first and builds a reversed depth-ordered stack:
var reverseDepthOrder = new { id = rootid, depth = 0 }.StackOf(trees.Count);
reverseDepthOrder.Pop();
var breadthFirstQueue = new { id = rootid, depth = 0 }.QueueOf(trees.Count);
while (breadthFirstQueue.Count > 0)
{
var curr = breadthFirstQueue.Dequeue();
// Add it to the reverse stack:
reverseDepthOrder.Push(curr);
TreeNode node;
if (!trees.TryGetValue(curr.id, out node))
{
// TODO: didn't find the TreeID in the given collection, assume already persisted?
continue;
}
// Queue up the child TreeIDs:
foreach (var trtr in node.Trees)
{
breadthFirstQueue.Enqueue(new { id = trtr.TreeID, depth = curr.depth + 1 });
}
}
// This code takes the reverse depth-ordered stack and persists the tree nodes in groups per depth level.
// This ensures that all child nodes across the breadth of the tree at each depth level are persisted
// before moving up to their parents.
List <Task <Errorable <TreeNode> > > persistTasks = new List <Task <Errorable <TreeNode> > >();
HashSet <TreeID> isPersisting = new HashSet <TreeID>();
int lastDepth = reverseDepthOrder.Peek().depth;
foreach (var curr in reverseDepthOrder)
{
Debug.WriteLine(String.Format("{0}: {1}", curr.depth, curr.id.ToString(firstLength: 7)));
// An invariant of the algorithm, enforced via assert:
Debug.Assert(curr.depth <= lastDepth);
// Did we move to the next depth group:
if (curr.depth != lastDepth)
{
Debug.WriteLine(String.Format("Awaiting depth group {0}...", lastDepth));
// Wait for the last depth group to finish persisting:
await Task.WhenAll(persistTasks);
// TODO: roll up errors!
// Start a new depth group:
persistTasks = new List <Task <Errorable <TreeNode> > >();
}
// Don't re-persist the same TreeID (this is a legit case - the same TreeID may be seen in different nodes of the tree):
if (isPersisting.Contains(curr.id))
{
Debug.WriteLine(String.Format("Already persisting {0}", curr.id.ToString(firstLength: 7)));
// Keep track of the last depth level:
lastDepth = curr.depth;
continue;
}
// Get the TreeNode and persist it:
TreeNode node = trees[curr.id];
isPersisting.Add(curr.id);
// Fire up a task to persist this tree node:
var tsk = Task.Run(() => persistTree(node));
// Add the task to the depth group to await:
Debug.WriteLine(String.Format("Adding to depth group {0}...", curr.depth));
persistTasks.Add(tsk);
// Keep track of the last depth level:
lastDepth = curr.depth;
}
// The final depth group should be depth 0 with at most 1 element: the root node.
Debug.Assert(lastDepth == 0);
if (persistTasks.Count > 0)
{
// Await the last group (the root node):
Debug.WriteLine(String.Format("Awaiting depth group {0}...", lastDepth));
await Task.WhenAll(persistTasks);
}
// Return the root TreeNode:
return(persistTasks[0].Result);
}
public CommitTree(CommitID rootID, ImmutableContainer <CommitID, ICommit> commits)
{
this.RootID = rootID;
this.Commits = commits;
}
public TreeTree(TreeID rootID, ImmutableContainer <TreeID, TreeNode> trees)
{
this.RootID = rootID;
this.Trees = trees;
}
async Task DemonstrateHashing()
{
var db = getDataContext();
IBlobRepository blrepo = new BlobRepository(db);
ITreeRepository trrepo = new TreeRepository(db);
ICommitRepository cmrepo = new CommitRepository(db);
IRefRepository rfrepo = new RefRepository(db);
// Create a sample set of blobs:
Blob readmeBlob;
ImmutableContainer <BlobID, Blob> blobs = new ImmutableContainer <BlobID, Blob>(
bl => bl.ID,
readmeBlob = new Blob.Builder(pContents: Encoding.UTF8.GetBytes(@"Hello world"))
);
Console.Out.WriteAsync(String.Format("Blob {0} = \"{1}\"" + Environment.NewLine, readmeBlob.ID.ToString(firstLength: 7), Encoding.UTF8.GetString(readmeBlob.Contents)));
// Persist them:
var persistingBlobs = blrepo.PersistBlobs(blobs);
// Create an initial tree:
Tree trRoot;
ImmutableContainer <TreeID, Tree> trees = new ImmutableContainer <TreeID, Tree>(
tr => tr.ID,
new Tree[] {
trRoot = new Tree.Builder(
pTrees: new List <TreeTreeReference>(0),
pBlobs: new List <TreeBlobReference> {
new TreeBlobReference.Builder(pName: "README", pBlobID: readmeBlob.ID)
}
)
}
);
// Dump the tree:
RecursivePrint(trRoot.ID, trees);
// Now wait for the blob persistence to complete:
await persistingBlobs;
// Persist our tree now:
var persistTrees = trrepo.PersistTree(trRoot.ID, trees);
await persistTrees;
// Let's make a commit out of all that:
Commit cm1 = new Commit.Builder(
pParents: new List <CommitID>(0),
pTreeID: trRoot.ID,
pCommitter: @"James Dunne <*****@*****.**>",
pDateCommitted: DateTimeOffset.Now,
pMessage: "Initial commit."
);
// Persist that commit:
Console.Out.WriteAsync(String.Format("Persisting commit {0}..." + Environment.NewLine, cm1.ID.ToString(firstLength: 7)));
await cmrepo.PersistCommit(cm1);
// Let's create a ref to point to it:
await rfrepo.DestroyRefByName("demo/HEAD");
await rfrepo.PersistRef(new Ref.Builder(pName: "demo/HEAD", pCommitID: cm1.ID));
await Console.Out.WriteAsync(String.Format("Pointed demo/HEAD to commit {0}" + Environment.NewLine, cm1.ID.ToString(firstLength: 7)));
// Now let's create a new blob with some revised contents: (adding a period at the end)
Blob readmeBlob2;
blobs = new ImmutableContainer <BlobID, Blob>(
bl => bl.ID,
readmeBlob2 = new Blob.Builder(Encoding.UTF8.GetBytes(@"Hello world."))
);
var persistBlobs2 = blrepo.PersistBlobs(blobs);
// Make a new tree out of the old tree:
Tree.Builder trRoot2b = new Tree.Builder(trRoot);
// Point README to the new BlobID:
trRoot2b.Blobs[0] = new TreeBlobReference.Builder(trRoot2b.Blobs[0])
{
BlobID = readmeBlob2.ID
};
// Freeze our Tree.Builder to a Tree:
Tree trRoot2;
trees = new ImmutableContainer <TreeID, Tree>(tr => tr.ID, trRoot2 = trRoot2b);
// Wait for the blobs to persist:
await persistBlobs2;
// Now persist the new tree:
await trrepo.PersistTree(trRoot2.ID, trees);
// Load a streamed blob:
IStreamedBlob strbl = await blrepo.GetStreamedBlob(readmeBlob.ID);
await strbl.ReadStream(blsr =>
{
Console.WriteLine("blob is {0} length", blsr.Length);
byte[] dum = new byte[8040];
int count = 8040;
try
{
while ((count = blsr.Read(dum, 0, 8040)) > 0)
{
for (int i = 0; i < (count / 40) + ((count % 40) > 0 ? 1 : 0); ++i)
{
Console.WriteLine(dum.ToHexString(i * 40, Math.Min(count - (i * 40), 40)));
}
}
}
catch (Exception ex)
{
Console.Error.WriteLine(ex.ToString());
}
});
}
async Task <TreeID> TestPersistTree()
{
// Create a Blob:
Blob bl = new Blob.Builder(Encoding.UTF8.GetBytes("Sample README content."));
ImmutableContainer <BlobID, Blob> blobs = new ImmutableContainer <BlobID, Blob>(b => b.ID, bl);
// Create a Tree:
Tree trPersists = new Tree.Builder(
new List <TreeTreeReference>(0),
new List <TreeBlobReference> {
new TreeBlobReference.Builder("HelloWorld.cs", bl.ID),
new TreeBlobReference.Builder("PersistBlob.cs", bl.ID),
new TreeBlobReference.Builder("PersistTree.cs", bl.ID),
}
);
Tree trSrc = new Tree.Builder(
new List <TreeTreeReference> {
new TreeTreeReference.Builder("Persists", trPersists.ID),
},
new List <TreeBlobReference> {
new TreeBlobReference.Builder("blah", bl.ID),
}
);
Tree trData = new Tree.Builder(
new List <TreeTreeReference>(0),
new List <TreeBlobReference> {
new TreeBlobReference.Builder("myTest.xml", bl.ID),
new TreeBlobReference.Builder("myTest2.xml", bl.ID),
new TreeBlobReference.Builder("myTest3.xml", bl.ID),
}
);
Tree trRoot = new Tree.Builder(
new List <TreeTreeReference> {
new TreeTreeReference.Builder("src", trSrc.ID),
new TreeTreeReference.Builder("data", trData.ID),
},
new List <TreeBlobReference> {
new TreeBlobReference.Builder("README", bl.ID),
new TreeBlobReference.Builder("main.xml", bl.ID),
new TreeBlobReference.Builder("test.xml", bl.ID),
}
);
ImmutableContainer <TreeID, Tree> trees = new ImmutableContainer <TreeID, Tree>(tr => tr.ID, trRoot, trSrc, trData, trPersists);
var db = getDataContext();
IBlobRepository blrepo = new BlobRepository(db);
ITreeRepository trrepo = new TreeRepository(db);
// Persist the tree and its blobs:
await blrepo.PersistBlobs(blobs);
var ptr = await trrepo.PersistTree(trRoot.ID, trees);
// Make sure we got back what's expected of the API:
Debug.Assert(trRoot.ID == ptr.ID);
Console.WriteLine("Root TreeID = {0}", trRoot.ID);
return(trRoot.ID);
}
