internal static void RecursivePrint(TreeID treeID, ImmutableContainer <TreeID, TreeNode> trees, int depth = 1) { TreeNode tr; if (!trees.TryGetValue(treeID, out tr)) { return; } if (depth == 1) { Console.WriteLine("tree {1}: {0}/", new string('_', (depth - 1) * 2), tr.ID.ToString(firstLength: 7)); } // Sort refs by name: var namedRefs = TreeNode.ComputeChildList(tr.Trees, tr.Blobs); foreach (var kv in namedRefs) { var nref = kv.Value; switch (nref.Which) { case Either <TreeTreeReference, TreeBlobReference> .Selected.Left: Console.WriteLine("tree {1}: {0}{2}/", new string('_', depth * 2), nref.Left.TreeID.ToString(firstLength: 7), nref.Left.Name); RecursivePrint(nref.Left.TreeID, trees, depth + 1); break; case Either <TreeTreeReference, TreeBlobReference> .Selected.Right: Console.WriteLine("blob {1}: {0}{2}", new string('_', depth * 2), nref.Right.BlobID.ToString(firstLength: 7), nref.Right.Name); break; } } }
private async Task createCommits() { PersistingBlob pblReadme = new PersistingBlob("Readme file.".ToStream()); var sblobs = await blrepo.PersistBlobs(pblReadme); trRoot = new TreeNode.Builder( new List <TreeTreeReference>(0), new List <TreeBlobReference> { new TreeBlobReference.Builder("README", sblobs[0].Value.ID) } ); var trees = new ImmutableContainer <TreeID, TreeNode>(tr => tr.ID, trRoot); await trrepo.PersistTree(trRoot.ID, trees); TreeRepositoryTestMethods.RecursivePrint(trRoot.ID, trees); cmRoot = new Commit.Builder( new List <CommitID>(0), trRoot.ID, "James S. Dunne", DateTimeOffset.Now, "Hello world." ); await cmrepo.PersistCommit(cmRoot); }
async Task TestLargeBlobPersistence() { var db = getDataContext(); IBlobRepository blrepo = new BlobRepository(db); Random rnd = new Random(0x5555AAC0); const int count = 4000; Console.WriteLine("Constructing {0} blobs...", count); Blob[] bls = new Blob[count]; for (int i = 0; i < count; ++i) { byte[] c = new byte[8040]; rnd.NextBytes(c); bls[i] = new Blob.Builder(c); } ImmutableContainer <BlobID, Blob> blobs = new ImmutableContainer <BlobID, Blob>(bl => bl.ID, bls); Console.WriteLine("Persisting {0} blobs...", count); Stopwatch sw = Stopwatch.StartNew(); ImmutableContainer <BlobID, Blob> pBlobs = await blrepo.PersistBlobs(blobs); Console.WriteLine("Waiting..."); sw.Stop(); Console.WriteLine("Completed in {0} ms, {1} blobs/sec, {2} bytes/sec", sw.ElapsedMilliseconds, (double)count / (double)sw.ElapsedMilliseconds * 1000d, (double)(count * 8040) / (double)sw.ElapsedMilliseconds * 1000d); }
private static TreeResponse projectTreeJSON(TreeID rootid, ImmutableContainer<TreeID, TreeNode> trees) { TreeNode tree; if (!trees.TryGetValue(rootid, out tree)) return null; return new TreeResponse { id = tree.ID.ToString(), blobs = tree.Blobs.SelectAsArray(bl => new TreeBlobRefResponse { name = bl.Name, blobid = bl.BlobID.ToString() }), trees = tree.Trees.SelectAsArray(tr => new TreeTreeRefResponse { name = tr.Name, treeid = tr.TreeID.ToString(), tree = projectTreeJSON(tr.TreeID, trees) }) }; }
private async Task createTrees() { PersistingBlob pbHeader = new PersistingBlob("<div>Header</div>".ToStream()); sblobs = await blrepo.PersistBlobs(pbHeader); trSection1 = new TreeNode.Builder(); trSection2 = new TreeNode.Builder(); trSection3 = new TreeNode.Builder(); trImages = new TreeNode.Builder(); trCSS = new TreeNode.Builder(); trJS = new TreeNode.Builder(); trPages = new TreeNode.Builder( new List <TreeTreeReference> { new TreeTreeReference.Builder("section1", trSection1.ID), new TreeTreeReference.Builder("section2", trSection2.ID), new TreeTreeReference.Builder("section3", trSection3.ID) }, new List <TreeBlobReference>(0) ); trContent = new TreeNode.Builder( new List <TreeTreeReference> { new TreeTreeReference.Builder("images", trImages.ID), new TreeTreeReference.Builder("css", trCSS.ID), new TreeTreeReference.Builder("js", trJS.ID) }, new List <TreeBlobReference>(0) ); trTemplate = new TreeNode.Builder( new List <TreeTreeReference>(0), new List <TreeBlobReference> { new TreeBlobReference.Builder("header", sblobs[0].Value.ID) } ); trRoot = new TreeNode.Builder( new List <TreeTreeReference> { new TreeTreeReference.Builder("template", trTemplate.ID), new TreeTreeReference.Builder("content", trContent.ID), new TreeTreeReference.Builder("pages", trPages.ID) }, new List <TreeBlobReference>(0) ); rootId = trRoot.ID; trees = new ImmutableContainer <TreeID, TreeNode>(tr => tr.ID, trSection1, trSection2, trSection3, trPages, trImages, trCSS, trJS, trContent, trTemplate, trRoot); RecursivePrint(trRoot.ID, trees); }
private CommitTreeResponse toJSON(CommitID id, ImmutableContainer<CommitID, ICommit> commits) { ICommit cm; if (!commits.TryGetValue(id, out cm)) return null; return new CommitTreeResponse() { id = cm.ID.ToString(), treeid = cm.TreeID.ToString(), committer = cm.Committer.ToString(), date_committed = JSONTranslateExtensions.FromDate(cm.DateCommitted), parents_retrieved = cm.IsComplete, message = cm.Message, parents = cm.Parents.SelectAsArray(cmid => toJSON(cmid, commits)) }; }
static void RecursivePrint(CommitID cmID, ImmutableContainer <CommitID, ICommit> commits, int depth = 1) { ICommit cm = commits[cmID]; if (cm.IsComplete) { Console.WriteLine("{0}c {1}: ({2})", new string(' ', (depth - 1) * 2), cm.ID.ToString(firstLength: 7), String.Join(",", cm.Parents.Select(id => id.ToString(firstLength: 7)))); foreach (CommitID parentID in cm.Parents) { RecursivePrint(parentID, commits, depth + 1); } } else { Console.WriteLine("{0}p {1}: ?", new string(' ', (depth - 1) * 2), cm.ID.ToString(firstLength: 7)); } }
void TestQueryBlobs() { // Create some Blobs: Blob bl0 = new Blob.Builder(Encoding.UTF8.GetBytes("Sample README content.")); Console.WriteLine(bl0.ID.ToString()); Blob bl1 = new Blob.Builder(Encoding.UTF8.GetBytes("Sample content.")); Console.WriteLine(bl1.ID.ToString()); var blobs = new ImmutableContainer <BlobID, Blob>(bl => bl.ID, bl0, bl1); var db = getDataContext(); // Check which blobs exist already: var qBlobs = db.ExecuteListQueryAsync(new QueryBlobsExist(bl0.ID, bl1.ID), expectedCapacity: blobs.Count); qBlobs.Wait(); // Find the blobs to persist: var blobIDsToPersist = blobs.Keys.Except(qBlobs.Result).ToArray(); // Persist each blob asynchronously: Task <Blob>[] persists = new Task <Blob> [blobIDsToPersist.Length]; for (int i = 0; i < blobIDsToPersist.Length; ++i) { BlobID id = blobIDsToPersist[i]; Console.WriteLine("PERSIST {0}", id.ToString()); persists[i] = db.ExecuteNonQueryAsync(new PersistBlob(blobs[id])); } Console.WriteLine("Waiting for persists..."); Task.WaitAll(persists); Console.WriteLine("Complete."); }
public async Task<ActionResult> CreateTree(TreeRequest tm) { TreeID root; ImmutableContainer<TreeID, TreeNode> trees; // Recursively convert the JSON-friendly `TreeModel` into our domain-friendly `Tree`s: TreeNode[] treeArr = convertRecursively(tm); root = treeArr[0].ID; trees = new ImmutableContainer<TreeID, TreeNode>(tr => tr.ID, treeArr); // Persist the tree: var etree = await cms.trrepo.PersistTree(root, trees); if (etree.HasErrors) return ErrorJson(etree); // Return the `Tree` recursive model we persisted: return Json(new { tree = projectTreeJSON(root, trees) }); }
public async Task<Errorable<TreeNode>> PersistTree(TreeID rootid, ImmutableContainer<TreeID, TreeNode> trees) { if (trees == null) throw new ArgumentNullException("trees"); // TODO: better return value than `null` if (trees.Count == 0) return (TreeNode)null; // Start a query to check what Trees exist already: var existTrees = await db.ExecuteListQueryAsync(new QueryTreesExist(trees.Keys), expectedCapacity: trees.Count); // This code scans the tree breadth-first and builds a reversed depth-ordered stack: var reverseDepthOrder = new { id = rootid, depth = 0 }.StackOf(trees.Count); reverseDepthOrder.Pop(); var breadthFirstQueue = new { id = rootid, depth = 0 }.QueueOf(trees.Count); while (breadthFirstQueue.Count > 0) { var curr = breadthFirstQueue.Dequeue(); // Add it to the reverse stack: reverseDepthOrder.Push(curr); TreeNode node; if (!trees.TryGetValue(curr.id, out node)) { // TODO: didn't find the TreeID in the given collection, assume already persisted? continue; } // Queue up the child TreeIDs: foreach (var trtr in node.Trees) breadthFirstQueue.Enqueue(new { id = trtr.TreeID, depth = curr.depth + 1 }); } // This code takes the reverse depth-ordered stack and persists the tree nodes in groups per depth level. // This ensures that all child nodes across the breadth of the tree at each depth level are persisted // before moving up to their parents. List<Task<Errorable<TreeNode>>> persistTasks = new List<Task<Errorable<TreeNode>>>(); // Initialize the `isPersisting` set with the set of TreeIDs that already exist. HashSet<TreeID> isPersisting = new HashSet<TreeID>(existTrees); int lastDepth = reverseDepthOrder.Peek().depth; foreach (var curr in reverseDepthOrder) { Debug.WriteLine(String.Format("{0}: {1}", curr.depth, curr.id.ToString(firstLength: 7))); // An invariant of the algorithm, enforced via assert: Debug.Assert(curr.depth <= lastDepth); // Did we move to the next depth group: if ((persistTasks.Count > 0) && (curr.depth != lastDepth)) { Debug.WriteLine(String.Format("Awaiting depth group {0}...", lastDepth)); // Wait for the last depth group to finish persisting: await Task.WhenAll(persistTasks); // Start a new depth group: persistTasks = new List<Task<Errorable<TreeNode>>>(); } // Don't re-persist the same TreeID (this is a legit case - the same TreeID may be seen in different nodes of the tree): if (isPersisting.Contains(curr.id)) { Debug.WriteLine(String.Format("Already persisting {0}", curr.id.ToString(firstLength: 7))); // Keep track of the last depth level: lastDepth = curr.depth; continue; } // Get the TreeNode and persist it: TreeNode node = trees[curr.id]; isPersisting.Add(curr.id); // Fire up a task to persist this tree node: var tsk = db.ExecuteNonQueryAsync(new PersistTree(node)); // Add the task to the depth group to await: Debug.WriteLine(String.Format("Adding to depth group {0}...", curr.depth)); persistTasks.Add(tsk); // Keep track of the last depth level: lastDepth = curr.depth; } Debug.Assert(lastDepth == 0); if (persistTasks.Count > 0) { // Await the last group (the root node): Debug.WriteLine(String.Format("Awaiting depth group {0}...", lastDepth)); await Task.WhenAll(persistTasks); } // Return the root TreeNode: return trees[rootid]; }
public TreeTree(TreeID rootID, ImmutableContainer<TreeID, TreeNode> trees) { this.RootID = rootID; this.Trees = trees; }
public async Task <Errorable <TreeNode> > PersistTree(TreeID rootid, ImmutableContainer <TreeID, TreeNode> trees) { if (trees == null) { throw new ArgumentNullException("trees"); } // TODO: better return value than `null` if (trees.Count == 0) { return((TreeNode)null); } // This code scans the tree breadth-first and builds a reversed depth-ordered stack: var reverseDepthOrder = new { id = rootid, depth = 0 }.StackOf(trees.Count); reverseDepthOrder.Pop(); var breadthFirstQueue = new { id = rootid, depth = 0 }.QueueOf(trees.Count); while (breadthFirstQueue.Count > 0) { var curr = breadthFirstQueue.Dequeue(); // Add it to the reverse stack: reverseDepthOrder.Push(curr); TreeNode node; if (!trees.TryGetValue(curr.id, out node)) { // TODO: didn't find the TreeID in the given collection, assume already persisted? continue; } // Queue up the child TreeIDs: foreach (var trtr in node.Trees) { breadthFirstQueue.Enqueue(new { id = trtr.TreeID, depth = curr.depth + 1 }); } } // This code takes the reverse depth-ordered stack and persists the tree nodes in groups per depth level. // This ensures that all child nodes across the breadth of the tree at each depth level are persisted // before moving up to their parents. List <Task <Errorable <TreeNode> > > persistTasks = new List <Task <Errorable <TreeNode> > >(); HashSet <TreeID> isPersisting = new HashSet <TreeID>(); int lastDepth = reverseDepthOrder.Peek().depth; foreach (var curr in reverseDepthOrder) { Debug.WriteLine(String.Format("{0}: {1}", curr.depth, curr.id.ToString(firstLength: 7))); // An invariant of the algorithm, enforced via assert: Debug.Assert(curr.depth <= lastDepth); // Did we move to the next depth group: if (curr.depth != lastDepth) { Debug.WriteLine(String.Format("Awaiting depth group {0}...", lastDepth)); // Wait for the last depth group to finish persisting: await Task.WhenAll(persistTasks); // TODO: roll up errors! // Start a new depth group: persistTasks = new List <Task <Errorable <TreeNode> > >(); } // Don't re-persist the same TreeID (this is a legit case - the same TreeID may be seen in different nodes of the tree): if (isPersisting.Contains(curr.id)) { Debug.WriteLine(String.Format("Already persisting {0}", curr.id.ToString(firstLength: 7))); // Keep track of the last depth level: lastDepth = curr.depth; continue; } // Get the TreeNode and persist it: TreeNode node = trees[curr.id]; isPersisting.Add(curr.id); // Fire up a task to persist this tree node: var tsk = Task.Run(() => persistTree(node)); // Add the task to the depth group to await: Debug.WriteLine(String.Format("Adding to depth group {0}...", curr.depth)); persistTasks.Add(tsk); // Keep track of the last depth level: lastDepth = curr.depth; } // The final depth group should be depth 0 with at most 1 element: the root node. Debug.Assert(lastDepth == 0); if (persistTasks.Count > 0) { // Await the last group (the root node): Debug.WriteLine(String.Format("Awaiting depth group {0}...", lastDepth)); await Task.WhenAll(persistTasks); } // Return the root TreeNode: return(persistTasks[0].Result); }
public CommitTree(CommitID rootID, ImmutableContainer <CommitID, ICommit> commits) { this.RootID = rootID; this.Commits = commits; }
public TreeTree(TreeID rootID, ImmutableContainer <TreeID, TreeNode> trees) { this.RootID = rootID; this.Trees = trees; }
async Task DemonstrateHashing() { var db = getDataContext(); IBlobRepository blrepo = new BlobRepository(db); ITreeRepository trrepo = new TreeRepository(db); ICommitRepository cmrepo = new CommitRepository(db); IRefRepository rfrepo = new RefRepository(db); // Create a sample set of blobs: Blob readmeBlob; ImmutableContainer <BlobID, Blob> blobs = new ImmutableContainer <BlobID, Blob>( bl => bl.ID, readmeBlob = new Blob.Builder(pContents: Encoding.UTF8.GetBytes(@"Hello world")) ); Console.Out.WriteAsync(String.Format("Blob {0} = \"{1}\"" + Environment.NewLine, readmeBlob.ID.ToString(firstLength: 7), Encoding.UTF8.GetString(readmeBlob.Contents))); // Persist them: var persistingBlobs = blrepo.PersistBlobs(blobs); // Create an initial tree: Tree trRoot; ImmutableContainer <TreeID, Tree> trees = new ImmutableContainer <TreeID, Tree>( tr => tr.ID, new Tree[] { trRoot = new Tree.Builder( pTrees: new List <TreeTreeReference>(0), pBlobs: new List <TreeBlobReference> { new TreeBlobReference.Builder(pName: "README", pBlobID: readmeBlob.ID) } ) } ); // Dump the tree: RecursivePrint(trRoot.ID, trees); // Now wait for the blob persistence to complete: await persistingBlobs; // Persist our tree now: var persistTrees = trrepo.PersistTree(trRoot.ID, trees); await persistTrees; // Let's make a commit out of all that: Commit cm1 = new Commit.Builder( pParents: new List <CommitID>(0), pTreeID: trRoot.ID, pCommitter: @"James Dunne <*****@*****.**>", pDateCommitted: DateTimeOffset.Now, pMessage: "Initial commit." ); // Persist that commit: Console.Out.WriteAsync(String.Format("Persisting commit {0}..." + Environment.NewLine, cm1.ID.ToString(firstLength: 7))); await cmrepo.PersistCommit(cm1); // Let's create a ref to point to it: await rfrepo.DestroyRefByName("demo/HEAD"); await rfrepo.PersistRef(new Ref.Builder(pName: "demo/HEAD", pCommitID: cm1.ID)); await Console.Out.WriteAsync(String.Format("Pointed demo/HEAD to commit {0}" + Environment.NewLine, cm1.ID.ToString(firstLength: 7))); // Now let's create a new blob with some revised contents: (adding a period at the end) Blob readmeBlob2; blobs = new ImmutableContainer <BlobID, Blob>( bl => bl.ID, readmeBlob2 = new Blob.Builder(Encoding.UTF8.GetBytes(@"Hello world.")) ); var persistBlobs2 = blrepo.PersistBlobs(blobs); // Make a new tree out of the old tree: Tree.Builder trRoot2b = new Tree.Builder(trRoot); // Point README to the new BlobID: trRoot2b.Blobs[0] = new TreeBlobReference.Builder(trRoot2b.Blobs[0]) { BlobID = readmeBlob2.ID }; // Freeze our Tree.Builder to a Tree: Tree trRoot2; trees = new ImmutableContainer <TreeID, Tree>(tr => tr.ID, trRoot2 = trRoot2b); // Wait for the blobs to persist: await persistBlobs2; // Now persist the new tree: await trrepo.PersistTree(trRoot2.ID, trees); // Load a streamed blob: IStreamedBlob strbl = await blrepo.GetStreamedBlob(readmeBlob.ID); await strbl.ReadStream(blsr => { Console.WriteLine("blob is {0} length", blsr.Length); byte[] dum = new byte[8040]; int count = 8040; try { while ((count = blsr.Read(dum, 0, 8040)) > 0) { for (int i = 0; i < (count / 40) + ((count % 40) > 0 ? 1 : 0); ++i) { Console.WriteLine(dum.ToHexString(i * 40, Math.Min(count - (i * 40), 40))); } } } catch (Exception ex) { Console.Error.WriteLine(ex.ToString()); } }); }
async Task <TreeID> TestPersistTree() { // Create a Blob: Blob bl = new Blob.Builder(Encoding.UTF8.GetBytes("Sample README content.")); ImmutableContainer <BlobID, Blob> blobs = new ImmutableContainer <BlobID, Blob>(b => b.ID, bl); // Create a Tree: Tree trPersists = new Tree.Builder( new List <TreeTreeReference>(0), new List <TreeBlobReference> { new TreeBlobReference.Builder("HelloWorld.cs", bl.ID), new TreeBlobReference.Builder("PersistBlob.cs", bl.ID), new TreeBlobReference.Builder("PersistTree.cs", bl.ID), } ); Tree trSrc = new Tree.Builder( new List <TreeTreeReference> { new TreeTreeReference.Builder("Persists", trPersists.ID), }, new List <TreeBlobReference> { new TreeBlobReference.Builder("blah", bl.ID), } ); Tree trData = new Tree.Builder( new List <TreeTreeReference>(0), new List <TreeBlobReference> { new TreeBlobReference.Builder("myTest.xml", bl.ID), new TreeBlobReference.Builder("myTest2.xml", bl.ID), new TreeBlobReference.Builder("myTest3.xml", bl.ID), } ); Tree trRoot = new Tree.Builder( new List <TreeTreeReference> { new TreeTreeReference.Builder("src", trSrc.ID), new TreeTreeReference.Builder("data", trData.ID), }, new List <TreeBlobReference> { new TreeBlobReference.Builder("README", bl.ID), new TreeBlobReference.Builder("main.xml", bl.ID), new TreeBlobReference.Builder("test.xml", bl.ID), } ); ImmutableContainer <TreeID, Tree> trees = new ImmutableContainer <TreeID, Tree>(tr => tr.ID, trRoot, trSrc, trData, trPersists); var db = getDataContext(); IBlobRepository blrepo = new BlobRepository(db); ITreeRepository trrepo = new TreeRepository(db); // Persist the tree and its blobs: await blrepo.PersistBlobs(blobs); var ptr = await trrepo.PersistTree(trRoot.ID, trees); // Make sure we got back what's expected of the API: Debug.Assert(trRoot.ID == ptr.ID); Console.WriteLine("Root TreeID = {0}", trRoot.ID); return(trRoot.ID); }