internal async Task PersistRefTest()
{
TreeNode tr = new TreeNode.Builder(new List<TreeTreeReference>(0), new List<TreeBlobReference>(0));
await trrepo.PersistTree(tr.ID, new IVO.Definition.Containers.ImmutableContainer<TreeID, TreeNode>(trx => trx.ID, tr));
Commit cm = new Commit.Builder(new List<CommitID>(0), tr.ID, "James S. Dunne", DateTimeOffset.Now, "Initial commit.");
await cmrepo.PersistCommit(cm);
Ref rf = new Ref.Builder((RefName)"v1.0", cm.ID);
await rfrepo.PersistRef(rf);
}
internal async Task GetRefByNameTest()
{
TreeNode tr = new TreeNode.Builder(new List<TreeTreeReference>(0), new List<TreeBlobReference>(0));
await trrepo.PersistTree(tr.ID, new IVO.Definition.Containers.ImmutableContainer<TreeID, TreeNode>(trx => trx.ID, tr));
Commit cm = new Commit.Builder(new List<CommitID>(0), tr.ID, "James S. Dunne", DateTimeOffset.Now, "Initial commit.");
await cmrepo.PersistCommit(cm);
Ref rf = new Ref.Builder((RefName)"v1.0", cm.ID);
await rfrepo.PersistRef(rf);
var errf = await rfrepo.GetRefByName((RefName)"v1.0");
Assert.IsFalse(errf.HasErrors);
Ref rrf = errf.Value;
Assert.IsNotNull(rrf);
Assert.AreEqual(rf.Name.ToString(), rrf.Name.ToString());
Assert.AreEqual(rf.CommitID, rrf.CommitID);
}
public async Task TestImportAbsolute()
{
var tc = getTestContext();
PersistingBlob blHeader = new PersistingBlob("<div>Header</div>".ToStream());
PersistingBlob blFooter = new PersistingBlob("<div>Footer</div>".ToStream());
PersistingBlob blTest = new PersistingBlob("<div><cms-import path=\"/template/header\" /><cms-import path=\"/template/footer\" /></div>".ToStream());
// Persist the blob contents:
var sblobs = await tc.blrepo.PersistBlobs(blHeader, blFooter, blTest);
TreeNode trTemplate = new TreeNode.Builder(
new List<TreeTreeReference>(0),
new List<TreeBlobReference> {
new TreeBlobReference.Builder("header", sblobs[0].Value.ID),
new TreeBlobReference.Builder("footer", sblobs[1].Value.ID)
}
);
TreeNode trPages = new TreeNode.Builder(
new List<TreeTreeReference>(0),
new List<TreeBlobReference> {
new TreeBlobReference.Builder("test", sblobs[2].Value.ID)
}
);
TreeNode trRoot = new TreeNode.Builder(
new List<TreeTreeReference> {
new TreeTreeReference.Builder("template", trTemplate.ID),
new TreeTreeReference.Builder("pages", trPages.ID)
},
new List<TreeBlobReference>(0)
);
// Persist the trees:
var trTask = await tc.trrepo.PersistTree(trRoot.ID, new ImmutableContainer<TreeID, TreeNode>(tr => tr.ID, trTemplate, trPages, trRoot));
assertTranslated(
tc,
sblobs[2].Value, // aka blTest
trRoot.ID,
"<div><div>Header</div><div>Footer</div></div>"
);
}
private static TreeNode[] convertRecursively(TreeRequest tm)
{
int treeCount = tm.trees != null ? tm.trees.Length : 0;
int blobCount = tm.blobs != null ? tm.blobs.Length : 0;
TreeNode.Builder tb = new TreeNode.Builder(
new List<TreeTreeReference>(treeCount),
new List<TreeBlobReference>(blobCount)
);
// Add the blobs to the Tree.Builder:
if ((tm.blobs != null) && (blobCount > 0))
tb.Blobs.AddRange(from bl in tm.blobs select (TreeBlobReference)new TreeBlobReference.Builder(bl.name, BlobID.TryParse(bl.blobid).Value));
// Create our output list:
List<TreeNode> trees = new List<TreeNode>(1 + treeCount /* + more, could calculate recursively but why bother */);
// Dummy placeholder for this Tree:
trees.Add((TreeNode)null);
for (int i = 0; i < treeCount; ++i)
{
// If we have a `treeid` then skip recursion:
if (!String.IsNullOrEmpty(tm.trees[i].treeid))
{
tb.Trees.Add(new TreeTreeReference.Builder(tm.trees[i].name, TreeID.TryParse(tm.trees[i].treeid).Value));
continue;
}
// Convert the child trees:
TreeNode[] childTrees = convertRecursively(tm.trees[i].tree);
// Add them to the output list:
trees.AddRange(childTrees);
// Add the child TreeTreeReference to this Tree.Builder:
tb.Trees.Add(new TreeTreeReference.Builder(tm.trees[i].name, childTrees[0].ID));
}
// Set the first element (was a placeholder) to the built Tree:
trees[0] = tb;
return trees.ToArray();
}
public async Task<Errorable<TreeTree>> PersistTreeNodesByBlobPaths(Maybe<TreeID> rootID, IEnumerable<CanonicalBlobIDPath> paths)
{
TreeNode root;
if (rootID.HasValue)
{
var eroot = await getTree(rootID.Value);
if (eroot.HasErrors) return eroot.Errors;
root = eroot.Value;
}
else root = null;
var nodeByPath = new Dictionary<string, Tuple<CanonicalTreePath, TreeNode.Builder>>();
int depthCapacity = 5;
var depthGroups = new List<Tuple<CanonicalTreePath, TreeNode.Builder>>[depthCapacity];
Tuple<CanonicalTreePath, TreeNode.Builder> tpl;
// Add the root node by default:
CanonicalTreePath rootPath = (CanonicalTreePath)"/";
if (rootID.HasValue)
tpl = new Tuple<CanonicalTreePath, TreeNode.Builder>(rootPath, new TreeNode.Builder(root));
else
tpl = new Tuple<CanonicalTreePath, TreeNode.Builder>(rootPath, new TreeNode.Builder(new List<TreeTreeReference>(), new List<TreeBlobReference>()));
nodeByPath.Add(rootPath.ToString(), tpl);
// Initialize the depthGroups array:
int deepest = 0;
depthGroups[0] = new List<Tuple<CanonicalTreePath, TreeNode.Builder>>(1) { tpl };
for (int i = 1; i < depthCapacity; ++i) depthGroups[i] = new List<Tuple<CanonicalTreePath, TreeNode.Builder>>();
Debug.WriteLine(String.Empty);
foreach (CanonicalBlobIDPath path in paths)
{
TreeNode.Builder tnb;
CanonicalTreePath treePath;
string treePathStr;
// Create node builders for each subfolder in the path below the root:
for (int depth = 1; depth <= path.Path.Tree.Parts.Count; ++depth)
{
treePath = path.Path.Tree.GetPartialTree(depth);
treePathStr = treePath.ToString();
// Get the node builder for the current path or create it:
if (!nodeByPath.ContainsKey(treePathStr))
{
tnb = null;
if (rootID.HasValue)
{
// Get the TreeNode if available, otherwise construct a new builder:
var ecurr = await getTreeIDByPath(root, new TreeTreePath(rootID.Value, treePath));
if (ecurr.HasErrors) return ecurr.Errors;
var curr = ecurr.Value;
if (curr.TreeID.HasValue)
{
// Get the TreeNode:
var etr = await getTree(curr.TreeID.Value);
if (etr.HasErrors) return etr.Errors;
// Create the builder from that node:
tnb = new TreeNode.Builder(etr.Value);
}
}
if (tnb == null)
{
// New builder:
tnb = new TreeNode.Builder(new List<TreeTreeReference>(), new List<TreeBlobReference>());
}
// Keep track of this node builder for the given path:
Debug.WriteLine(treePathStr);
tpl = new Tuple<CanonicalTreePath, TreeNode.Builder>(treePath, tnb);
nodeByPath.Add(treePathStr, tpl);
// Maintain the depthGroups array:
if (depth >= depthCapacity)
{
// TODO: tune this resize policy:
int newCapacity = depth + 2;
// Extend the array:
Array.Resize(ref depthGroups, newCapacity);
// Initialize all the new elements:
for (int i = depthCapacity; i < newCapacity; ++i)
depthGroups[i] = new List<Tuple<CanonicalTreePath, TreeNode.Builder>>();
depthCapacity = newCapacity;
}
if (depth > deepest) deepest = depth;
depthGroups[depth].Add(tpl);
}
}
treePath = path.Path.Tree;
treePathStr = treePath.ToString();
// Get the node builder for the current path or create it:
Tuple<CanonicalTreePath, TreeNode.Builder> ptnb;
bool test = nodeByPath.TryGetValue(treePathStr, out ptnb);
Debug.Assert(test);
// Add or update the TreeBlobReference for this blob path:
string blobName = path.Path.Name;
var trblb = new TreeBlobReference.Builder(blobName, path.BlobID);
int blidx = ptnb.Item2.Blobs.FindIndex(trbl => trbl.Name == blobName);
if (blidx == -1)
ptnb.Item2.Blobs.Add(trblb);
else
ptnb.Item2.Blobs[blidx] = trblb;
}
#if DEBUG
Debug.WriteLine(String.Empty);
foreach (var mtpl in nodeByPath.Values)
{
Debug.WriteLine(String.Format(
"{0}: {1}",
mtpl.Item1.ToString(),
String.Join(", ", mtpl.Item2.Blobs.Select(trbl => trbl.Name + ":" + trbl.BlobID.ToString(firstLength: 7)))
));
}
#endif
for (int i = deepest; i >= 0; --i)
{
Debug.WriteLine(String.Empty);
Debug.WriteLine("Depth #{0}", i);
var nodes = depthGroups[i];
foreach (var node in nodes)
{
Debug.WriteLine(node.Item1.ToString());
foreach (var bl in node.Item2.Blobs)
{
Debug.WriteLine(new string('_', i * 2) + bl.Name + ":" + bl.BlobID.ToString(firstLength: 7));
}
}
}
// Persist each tree depth level:
var awaiting = new List<Task<Errorable<TreeNode>>>(depthGroups[deepest].Count);
var lastNodes = new List<Tuple<CanonicalTreePath, TreeNode>>(depthGroups[deepest].Count);
var result = new Dictionary<TreeID, TreeNode>();
Debug.WriteLine(String.Format("Starting depth group #{0}", deepest));
foreach (var dnode in depthGroups[deepest])
{
// Finalize the TreeNode:
TreeNode tn = dnode.Item2;
lastNodes.Add(new Tuple<CanonicalTreePath, TreeNode>(dnode.Item1, tn));
if (!result.ContainsKey(tn.ID))
{
Debug.WriteLine(String.Format("{0}: Persisting TreeID {1}", dnode.Item1.ToString(), tn.ID.ToString(firstLength: 7)));
result.Add(tn.ID, tn);
// Start persistence task and add to `awaiting`
var tsk = Task.Run(() => persistTree(tn));
awaiting.Add(tsk);
}
else
{
Debug.WriteLine(String.Format("{0}: Already persisted TreeID {1}", dnode.Item1.ToString(), tn.ID.ToString(firstLength: 7)));
}
}
for (int i = deepest - 1; i >= 0; --i)
{
// Await last depth group if non-empty:
if (awaiting.Count > 0)
{
Debug.WriteLine(String.Format("Awaiting previous depth group's persistence"));
var errs = await Task.WhenAll(awaiting);
if (errs.Any(err => err.HasErrors))
return errs.Aggregate(new ErrorContainer(), (acc, err) => acc + err.Errors);
}
Debug.WriteLine(String.Format("Starting depth group #{0}", i));
awaiting = new List<Task<Errorable<TreeNode>>>(depthGroups[i].Count);
var currNodes = new List<Tuple<CanonicalTreePath, TreeNode>>(depthGroups[i].Count);
// Update each tree node in this depth group:
foreach (var node in depthGroups[i])
{
// Create TreeTreeReferences to point to child TreeIDs:
var childNodes = lastNodes.Where(t => t.Item1.GetParent() == node.Item1);
foreach (var childNode in childNodes)
{
// Add or update the TreeTreeReference to the child tree:
int tridx = node.Item2.Trees.FindIndex(trtr => trtr.Name == childNode.Item1.Name);
var trtrb = new TreeTreeReference.Builder(childNode.Item1.Name, childNode.Item2.ID);
if (tridx == -1)
{
Debug.WriteLine(String.Format("{0}: Adding (name: {1}, treeid: {2})", node.Item1.ToString(), trtrb.Name, trtrb.TreeID.ToString(firstLength: 7)));
node.Item2.Trees.Add(trtrb);
}
else
{
Debug.WriteLine(String.Format("{0}: Updating (name: {1}, old-treeid: {2}, new-treeid: {3})", node.Item1.ToString(), trtrb.Name, node.Item2.Trees[tridx].TreeID, trtrb.TreeID.ToString(firstLength: 7)));
node.Item2.Trees[tridx] = trtrb;
}
}
// Finalize the TreeNode:
TreeNode tn = (TreeNode)node.Item2;
currNodes.Add(new Tuple<CanonicalTreePath, TreeNode>(node.Item1, tn));
if (!result.ContainsKey(tn.ID))
{
Debug.WriteLine(String.Format("{0}: Persisting TreeID {1}", node.Item1.ToString(), tn.ID.ToString(firstLength: 7)));
result.Add(tn.ID, tn);
// Start persistence task and add to `awaiting`
var tsk = Task.Run(() => persistTree(tn));
awaiting.Add(tsk);
}
else
{
Debug.WriteLine(String.Format("{0}: Already persisted TreeID {1}", node.Item1.ToString(), tn.ID.ToString(firstLength: 7)));
}
}
lastNodes = currNodes;
}
if (awaiting.Count > 0)
{
Debug.WriteLine(String.Format("Awaiting previous depth group's persistence"));
var errs = await Task.WhenAll(awaiting);
if (errs.Any(err => err.HasErrors))
return errs.Aggregate(new ErrorContainer(), (acc, err) => acc + err.Errors);
}
// Last depth group should be count of 1, the new root TreeNode:
Debug.Assert(lastNodes.Count == 1);
return new TreeTree(lastNodes[0].Item2.ID, new ImmutableContainer<TreeID, TreeNode>(result));
}
private async Task<Errorable<TreeNode>> getTree(TreeID id)
{
FileInfo fi = system.getPathByID(id);
if (!fi.Exists) return new TreeIDRecordDoesNotExistError(id);
byte[] buf;
int nr = 0;
using (var fs = new FileStream(fi.FullName, FileMode.Open, FileAccess.Read, FileShare.Read, 16384, true))
{
// TODO: implement an async buffered Stream:
buf = new byte[16384];
nr = await fs.ReadAsync(buf, 0, 16384).ConfigureAwait(continueOnCapturedContext: false);
if (nr >= 16384)
{
// My, what a large tree you have!
throw new NotSupportedException();
}
}
TreeNode.Builder tb = new TreeNode.Builder(new List<TreeTreeReference>(), new List<TreeBlobReference>());
// Parse the Tree:
using (var ms = new MemoryStream(buf, 0, nr, false))
using (var sr = new StreamReader(ms, Encoding.UTF8))
{
string line;
while ((line = sr.ReadLine()) != null)
{
if (line.StartsWith("tree "))
{
string linked_treeid = line.Substring(5, (TreeID.ByteArrayLength * 2));
string name = line.Substring(6 + (TreeID.ByteArrayLength * 2));
// Attempt to parse the TreeID and verify its existence:
Errorable<TreeID> trid = TreeID.TryParse(linked_treeid);
if (trid.HasErrors) return trid.Errors;
if (!system.getPathByID(trid.Value).Exists) return new TreeIDRecordDoesNotExistError(trid.Value);
tb.Trees.Add(new TreeTreeReference.Builder(name, trid.Value));
}
else if (line.StartsWith("blob "))
{
string linked_blobid = line.Substring(5, (TreeID.ByteArrayLength * 2));
string name = line.Substring(6 + (TreeID.ByteArrayLength * 2));
// Attempt to parse the BlobID and verify its existence:
Errorable<BlobID> blid = BlobID.TryParse(linked_blobid);
if (blid.HasErrors) return blid.Errors;
if (!system.getPathByID(blid.Value).Exists) return new BlobIDRecordDoesNotExistError(blid.Value);
tb.Blobs.Add(new TreeBlobReference.Builder(name, blid.Value));
}
}
}
// Create the immutable Tree from the Builder:
TreeNode tr = tb;
// Validate the computed TreeID:
if (tr.ID != id) return new ComputedTreeIDMismatchError(tr.ID, id);
return tr;
}
private async Task testImportTemplateFail(string templateMain, string pagesTest, SemanticError[] expectedErrors, SemanticWarning[] expectedWarnings)
{
var tc = getTestContext();
PersistingBlob blHeader = new PersistingBlob(templateMain.ToStream());
PersistingBlob blTest = new PersistingBlob(pagesTest.ToStream());
// Persist the blob contents:
var sblobs = await tc.blrepo.PersistBlobs(blHeader, blTest);
TreeNode trTemplate = new TreeNode.Builder(
new List<TreeTreeReference>(0),
new List<TreeBlobReference> {
new TreeBlobReference.Builder("main", sblobs[0].Value.ID),
}
);
TreeNode trPages = new TreeNode.Builder(
new List<TreeTreeReference>(0),
new List<TreeBlobReference> {
new TreeBlobReference.Builder("test", sblobs[1].Value.ID)
}
);
TreeNode trRoot = new TreeNode.Builder(
new List<TreeTreeReference> {
new TreeTreeReference.Builder("template", trTemplate.ID),
new TreeTreeReference.Builder("pages", trPages.ID)
},
new List<TreeBlobReference>(0)
);
// Persist the trees:
var trTask = await tc.trrepo.PersistTree(trRoot.ID, new ImmutableContainer<TreeID, TreeNode>(tr => tr.ID, trTemplate, trPages, trRoot));
output(new TreePathStreamedBlob(trRoot.ID, (CanonicalBlobPath)"/templates/main", sblobs[0].Value));
assumeFail(tc, new TreePathStreamedBlob(trRoot.ID, (CanonicalBlobPath)"/pages/test", sblobs[1].Value), expectedErrors, expectedWarnings);
}
public async Task SpeedTestRenderBlob()
{
DateTimeOffset a = new DateTimeOffset(2011, 09, 1, 0, 0, 0, 0, TimeSpan.FromHours(-5));
DateTimeOffset b = a.AddDays(15);
DateTimeOffset c = a.AddDays(30);
// Use a + 5 days as the viewing date for scheduling:
var tc = getTestContext(a.AddDays(5));
string tmp = Path.GetTempFileName();
using (var fs = new FileStream(tmp, FileMode.Open, FileAccess.Write, FileShare.None))
using (var sw = new StreamWriter(fs))
{
for (int i = 0; i < 160; ++i)
{
//<cms-import path=""/template/header"" />In between content.<cms-import path=""/template/footer"" />
sw.WriteLine(String.Format(
@"<div>
<cms-scheduled>
<range from=""{0}"" to=""{2}""/>
<range from=""{1}"" to=""{2}""/>
<content><cms-import path=""/template/header"" />In between content.<cms-import path=""/template/footer"" /></content>
<else>Else here?</else>
</cms-scheduled>
</div>",
a.ToString(),
b.ToString(),
c.ToString()
));
}
}
var pblHeader = new PersistingBlob("HEADER".ToStream());
var pblFooter = new PersistingBlob("FOOTER".ToStream());
var pblTest = new PersistingBlob(new FileStream(tmp, FileMode.Open, FileAccess.Read, FileShare.Read));
var bls = await tc.blrepo.PersistBlobs(pblHeader, pblFooter, pblTest);
TreeNode trTmpl = new TreeNode.Builder(
new List<TreeTreeReference>(0),
new List<TreeBlobReference> {
new TreeBlobReference.Builder("header", bls[0].Value.ID),
new TreeBlobReference.Builder("footer", bls[1].Value.ID)
}
);
TreeNode trRoot = new TreeNode.Builder(
new List<TreeTreeReference> { new TreeTreeReference.Builder("template", trTmpl.ID) },
new List<TreeBlobReference> { new TreeBlobReference.Builder("test", bls[2].Value.ID) }
);
await tc.trrepo.PersistTree(trRoot.ID, new ImmutableContainer<TreeID, TreeNode>(tr => tr.ID, trRoot, trTmpl));
Stopwatch stpw = Stopwatch.StartNew();
await tc.ce.RenderBlob(new TreePathStreamedBlob(trRoot.ID, (CanonicalBlobPath)"/test", bls[2].Value));
stpw.Stop();
var errs = tc.ce.GetErrors();
Console.WriteLine("Errors: {0}", errs.Count);
for (int i = 0; i < 10 && i < errs.Count; ++i)
{
Console.WriteLine(" {0}", errs[i].ToString());
}
Console.WriteLine();
Console.WriteLine("Time: {0} ms", stpw.ElapsedMilliseconds);
}
public async Task TestNestedElementsSkip()
{
DateTimeOffset a = new DateTimeOffset(2011, 09, 1, 0, 0, 0, 0, TimeSpan.FromHours(-5));
DateTimeOffset b = a.AddDays(15);
DateTimeOffset c = a.AddDays(30);
// Use a - 5 days as the viewing date for scheduling:
var tc = getTestContext(a.AddDays(-5));
PersistingBlob blHeader = new PersistingBlob("<div>Header</div>".ToStream());
PersistingBlob blFooter = new PersistingBlob("<div>Footer</div>".ToStream());
PersistingBlob blTest = new PersistingBlob(String.Format(
@"<div>
<cms-scheduled>
<range from=""{0}"" to=""{2}""/>
<range from=""{1}"" to=""{2}""/>
<content><cms-import path=""/template/header"" />In between content.<cms-import path=""/template/footer"" /></content>
<else>Else here?</else>
</cms-scheduled>
</div>",
a.ToString(),
b.ToString(),
c.ToString()
).ToStream());
// Persist the blob contents:
var sblobs = await tc.blrepo.PersistBlobs(blHeader, blFooter, blTest);
TreeNode trTemplate = new TreeNode.Builder(
new List<TreeTreeReference>(0),
new List<TreeBlobReference> {
new TreeBlobReference.Builder("header", sblobs[0].Value.ID),
new TreeBlobReference.Builder("footer", sblobs[1].Value.ID)
}
);
TreeNode trPages = new TreeNode.Builder(
new List<TreeTreeReference>(0),
new List<TreeBlobReference> {
new TreeBlobReference.Builder("test", sblobs[2].Value.ID)
}
);
TreeNode trRoot = new TreeNode.Builder(
new List<TreeTreeReference> {
new TreeTreeReference.Builder("template", trTemplate.ID),
new TreeTreeReference.Builder("pages", trPages.ID)
},
new List<TreeBlobReference>(0)
);
// Persist the trees:
var trTask = await tc.trrepo.PersistTree(trRoot.ID, new ImmutableContainer<TreeID, TreeNode>(tr => tr.ID, trTemplate, trPages, trRoot));
output(new TreePathStreamedBlob(trRoot.ID, (CanonicalBlobPath)"/template/header", sblobs[0].Value));
output(new TreePathStreamedBlob(trRoot.ID, (CanonicalBlobPath)"/template/footer", sblobs[1].Value));
assertTranslated(
tc,
sblobs[2].Value,
trRoot.ID,
@"<div>
Else here?
</div>"
);
}