internal async Task PersistBlobsTest()
        {
            const int numBlobs = 16;

            // Create an immutable container that points to the new blobs:
            Console.WriteLine("Creating {0} random blobs...", numBlobs);
            PersistingBlob[] blobs = createBlobs(numBlobs);

            // Now persist those blobs to the filesystem:
            Console.WriteLine("Persisting {0} random blobs...", numBlobs);
            Stopwatch sw = Stopwatch.StartNew();
            var streamedBlobs = await blrepo.PersistBlobs(blobs);
            sw.Stop();

            Console.WriteLine("Completed in {0} ms, {1} bytes/sec", sw.ElapsedMilliseconds, streamedBlobs.Sum(b => b.Value.Length) * 1000d / sw.ElapsedMilliseconds);
            
            // Repeat!
            for (int i = 0; i < numBlobs; ++i)
            {
                var fs = (FileStream)blobs[i].Stream;
                // Re-open the temporary filestream:
                blobs[i] = new PersistingBlob(new FileStream(fs.Name, FileMode.Open, FileAccess.Read, FileShare.Read, 8040, true));
            }

            sw = Stopwatch.StartNew();
            streamedBlobs = await blrepo.PersistBlobs(blobs);
            sw.Stop();

            Console.WriteLine("Completed in {0} ms, {1} bytes/sec", sw.ElapsedMilliseconds, streamedBlobs.Sum(b => b.Value.Length) * 1000d / sw.ElapsedMilliseconds);
        }
Exemple #2
0
        public async Task<ActionResult> CreateBlob(Errorable<MaybeTreeBlobPath> epath, Errorable<StageName> estage)
        {
            Debug.Assert(epath != null);
            if (epath.HasErrors) return ErrorJson(epath);
            if (estage != null && estage.HasErrors) return ErrorJson(estage);

            PersistingBlob pbl = new PersistingBlob(Request.InputStream);

            // Persist the blob from the input stream:
            var eblob = await cms.blrepo.PersistBlob(pbl);
            if (eblob.HasErrors) return ErrorJson(eblob);
            var blob = eblob.Value;

            // Now update the given root TreeID:
            var path = epath.Value;

            // Persist the new blob's effect on the Tree:
            var eptr = await cms.trrepo.PersistTreeNodesByBlobPaths(path.RootTreeID, new CanonicalBlobIDPath[] { new CanonicalBlobIDPath(path.Path, blob.ID) });
            if (eptr.HasErrors) return ErrorJson(eptr);

            TreeID newRootTreeID = eptr.Value.RootID;

            // optional 5) update stage with new root TreeID
            if (estage != null)
            {
                var epst = await cms.strepo.PersistStage(new Stage.Builder(estage.Value, newRootTreeID));
                if (epst.HasErrors) return ErrorJson(epst);
            }

            // Return the new information:
            return Json(new { blobid = blob.ID.ToString(), treeid = newRootTreeID.ToString() });
        }
        private PersistingBlob[] createBlobs(int numBlobs)
        {
            PersistingBlob[] blobs = new PersistingBlob[numBlobs];
            for (int i = 0; i < numBlobs; ++i)
            {
                string tmpFileName = System.IO.Path.GetTempFileName();
                using (var fs = new FileStream(tmpFileName, FileMode.Open, FileAccess.Write, FileShare.Read, 1048576, true))
                {
                    new RandomDataStream(1048576 * 2).CopyTo(fs);
                }

                blobs[i] = new PersistingBlob(new FileStream(tmpFileName, FileMode.Open, FileAccess.Read, FileShare.Read, 8040, true));
            }
            return blobs;
        }
        public async Task TestImportAbsolute()
        {
            var tc = getTestContext();

            PersistingBlob blHeader = new PersistingBlob("<div>Header</div>".ToStream());
            PersistingBlob blFooter = new PersistingBlob("<div>Footer</div>".ToStream());
            PersistingBlob blTest = new PersistingBlob("<div><cms-import path=\"/template/header\" /><cms-import path=\"/template/footer\" /></div>".ToStream());

            // Persist the blob contents:
            var sblobs = await tc.blrepo.PersistBlobs(blHeader, blFooter, blTest);

            TreeNode trTemplate = new TreeNode.Builder(
                new List<TreeTreeReference>(0),
                new List<TreeBlobReference> {
                        new TreeBlobReference.Builder("header", sblobs[0].Value.ID),
                        new TreeBlobReference.Builder("footer", sblobs[1].Value.ID)
                    }
            );
            TreeNode trPages = new TreeNode.Builder(
                new List<TreeTreeReference>(0),
                new List<TreeBlobReference> {
                        new TreeBlobReference.Builder("test", sblobs[2].Value.ID)
                    }
            );
            TreeNode trRoot = new TreeNode.Builder(
                new List<TreeTreeReference> {
                        new TreeTreeReference.Builder("template", trTemplate.ID),
                        new TreeTreeReference.Builder("pages", trPages.ID)
                    },
                new List<TreeBlobReference>(0)
            );

            // Persist the trees:
            var trTask = await tc.trrepo.PersistTree(trRoot.ID, new ImmutableContainer<TreeID, TreeNode>(tr => tr.ID, trTemplate, trPages, trRoot));

            assertTranslated(
                tc,
                sblobs[2].Value,   // aka blTest
                trRoot.ID,
                "<div><div>Header</div><div>Footer</div></div>"
            );
        }
        private async Task<Errorable<IStreamedBlob>> persistBlob(PersistingBlob blob)
        {
            Debug.WriteLine(String.Format("Starting persistence of blob"));

            // Find a temporary filename:
            FileInfo tmpPath = system.getTemporaryFile();

            long length = -1;
            BlobID blid;

            // Open a new stream to the source blob contents:
            using (var sr = blob.Stream)
            {
                length = sr.Length;

                // Create a new file and set its length so we can asynchronously write to it:
                using (var tmpFi = File.Open(tmpPath.FullName, FileMode.CreateNew, FileAccess.Write, FileShare.None))
                {
                    Debug.WriteLine(String.Format("New BLOB temp '{0}' length {1}", tmpPath.FullName, length));
                    tmpFi.SetLength(length);
                    tmpFi.Close();
                }

                // Determine the best buffer size to use for writing contents:
                int bufSize = Math.Min(Math.Max((int)length, 8), largeBufferSize);

                // Open a new FileStream to asynchronously write the blob contents:
                using (var fs = new FileStream(tmpPath.FullName, FileMode.Open, FileAccess.Write, FileShare.Read, bufSize, useAsync: true))
                using (var sha1 = new SHA1StreamWriter(fs))
                {
                    // Copy the contents asynchronously (expected copy in order):
                    await sr.CopyToAsync(sha1, bufSize).ConfigureAwait(continueOnCapturedContext: false);

                    // Create the BlobID from the SHA1 hash calculated during copy:
                    blid = new BlobID(sha1.GetHash());
                }
            }

            // Serialize access to the official blob file:
            lock (FileSystem.SystemLock)
            {
                // Create the blob's subdirectory under 'objects':
                FileInfo path = system.getPathByID(blid);
                path.Refresh();

                if (!path.Directory.Exists)
                {
                    Debug.WriteLine(String.Format("New DIR '{0}'", path.Directory.FullName));
                    path.Directory.Create();
                }

                // Don't recreate an existing blob:
                if (path.Exists)
                {
                    Debug.WriteLine(String.Format("Blob already exists at path '{0}', deleting temporary...", path.FullName));
                    tmpPath.Delete();
                    return new Errorable<IStreamedBlob>((IStreamedBlob)new StreamedBlob(this, blid, length));
                }

                // Move the temp file to the final blob filename:
                File.Move(tmpPath.FullName, path.FullName);
            }

            return new Errorable<IStreamedBlob>((IStreamedBlob)new StreamedBlob(this, blid, length));
        }
 public Task<Errorable<IStreamedBlob>> PersistBlob(PersistingBlob blob)
 {
     return persistBlob(blob);
 }
        private async Task testImportTemplateFail(string templateMain, string pagesTest, SemanticError[] expectedErrors, SemanticWarning[] expectedWarnings)
        {
            var tc = getTestContext();

            PersistingBlob blHeader = new PersistingBlob(templateMain.ToStream());
            PersistingBlob blTest = new PersistingBlob(pagesTest.ToStream());

            // Persist the blob contents:
            var sblobs = await tc.blrepo.PersistBlobs(blHeader, blTest);

            TreeNode trTemplate = new TreeNode.Builder(
                new List<TreeTreeReference>(0),
                new List<TreeBlobReference> {
                    new TreeBlobReference.Builder("main", sblobs[0].Value.ID),
                }
            );
            TreeNode trPages = new TreeNode.Builder(
                new List<TreeTreeReference>(0),
                new List<TreeBlobReference> {
                    new TreeBlobReference.Builder("test", sblobs[1].Value.ID)
                }
            );
            TreeNode trRoot = new TreeNode.Builder(
                new List<TreeTreeReference> {
                    new TreeTreeReference.Builder("template", trTemplate.ID),
                    new TreeTreeReference.Builder("pages", trPages.ID)
                },
                new List<TreeBlobReference>(0)
            );

            // Persist the trees:
            var trTask = await tc.trrepo.PersistTree(trRoot.ID, new ImmutableContainer<TreeID, TreeNode>(tr => tr.ID, trTemplate, trPages, trRoot));

            output(new TreePathStreamedBlob(trRoot.ID, (CanonicalBlobPath)"/templates/main", sblobs[0].Value));
            assumeFail(tc, new TreePathStreamedBlob(trRoot.ID, (CanonicalBlobPath)"/pages/test", sblobs[1].Value), expectedErrors, expectedWarnings);
        }
        public async Task SpeedTestRenderBlob()
        {
            DateTimeOffset a = new DateTimeOffset(2011, 09, 1, 0, 0, 0, 0, TimeSpan.FromHours(-5));
            DateTimeOffset b = a.AddDays(15);
            DateTimeOffset c = a.AddDays(30);
            // Use a + 5 days as the viewing date for scheduling:
            var tc = getTestContext(a.AddDays(5));

            string tmp = Path.GetTempFileName();

            using (var fs = new FileStream(tmp, FileMode.Open, FileAccess.Write, FileShare.None))
            using (var sw = new StreamWriter(fs))
            {
                for (int i = 0; i < 160; ++i)
                {
                    //<cms-import path=""/template/header"" />In between content.<cms-import path=""/template/footer"" />
                    sw.WriteLine(String.Format(
@"<div>
  <cms-scheduled>
    <range from=""{0}"" to=""{2}""/>
    <range from=""{1}"" to=""{2}""/>
    <content><cms-import path=""/template/header"" />In between content.<cms-import path=""/template/footer"" /></content>
    <else>Else here?</else>
  </cms-scheduled>
</div>",
                        a.ToString(),
                        b.ToString(),
                        c.ToString()
                    ));
                }
            }

            var pblHeader = new PersistingBlob("HEADER".ToStream());
            var pblFooter = new PersistingBlob("FOOTER".ToStream());
            var pblTest = new PersistingBlob(new FileStream(tmp, FileMode.Open, FileAccess.Read, FileShare.Read));

            var bls = await tc.blrepo.PersistBlobs(pblHeader, pblFooter, pblTest);

            TreeNode trTmpl = new TreeNode.Builder(
                new List<TreeTreeReference>(0),
                new List<TreeBlobReference> {
                    new TreeBlobReference.Builder("header", bls[0].Value.ID),
                    new TreeBlobReference.Builder("footer", bls[1].Value.ID)
                }
            );
            TreeNode trRoot = new TreeNode.Builder(
                new List<TreeTreeReference> { new TreeTreeReference.Builder("template", trTmpl.ID) },
                new List<TreeBlobReference> { new TreeBlobReference.Builder("test", bls[2].Value.ID) }
            );

            await tc.trrepo.PersistTree(trRoot.ID, new ImmutableContainer<TreeID, TreeNode>(tr => tr.ID, trRoot, trTmpl));

            Stopwatch stpw = Stopwatch.StartNew();
            await tc.ce.RenderBlob(new TreePathStreamedBlob(trRoot.ID, (CanonicalBlobPath)"/test", bls[2].Value));
            stpw.Stop();

            var errs = tc.ce.GetErrors();
            Console.WriteLine("Errors: {0}", errs.Count);
            for (int i = 0; i < 10 && i < errs.Count; ++i)
            {
                Console.WriteLine("  {0}", errs[i].ToString());
            }
            Console.WriteLine();
            Console.WriteLine("Time: {0} ms", stpw.ElapsedMilliseconds);
        }
        public async Task TestNestedElementsSkip()
        {
            DateTimeOffset a = new DateTimeOffset(2011, 09, 1, 0, 0, 0, 0, TimeSpan.FromHours(-5));
            DateTimeOffset b = a.AddDays(15);
            DateTimeOffset c = a.AddDays(30);
            // Use a - 5 days as the viewing date for scheduling:
            var tc = getTestContext(a.AddDays(-5));

            PersistingBlob blHeader = new PersistingBlob("<div>Header</div>".ToStream());
            PersistingBlob blFooter = new PersistingBlob("<div>Footer</div>".ToStream());
            PersistingBlob blTest = new PersistingBlob(String.Format(
@"<div>
  <cms-scheduled>
    <range from=""{0}"" to=""{2}""/>
    <range from=""{1}"" to=""{2}""/>
    <content><cms-import path=""/template/header"" />In between content.<cms-import path=""/template/footer"" /></content>
    <else>Else here?</else>
  </cms-scheduled>
</div>",
                a.ToString(),
                b.ToString(),
                c.ToString()
            ).ToStream());

            // Persist the blob contents:
            var sblobs = await tc.blrepo.PersistBlobs(blHeader, blFooter, blTest);

            TreeNode trTemplate = new TreeNode.Builder(
                new List<TreeTreeReference>(0),
                new List<TreeBlobReference> {
                    new TreeBlobReference.Builder("header", sblobs[0].Value.ID),
                    new TreeBlobReference.Builder("footer", sblobs[1].Value.ID)
                }
            );
            TreeNode trPages = new TreeNode.Builder(
                new List<TreeTreeReference>(0),
                new List<TreeBlobReference> {
                    new TreeBlobReference.Builder("test", sblobs[2].Value.ID)
                }
            );
            TreeNode trRoot = new TreeNode.Builder(
                new List<TreeTreeReference> {
                    new TreeTreeReference.Builder("template", trTemplate.ID),
                    new TreeTreeReference.Builder("pages", trPages.ID)
                },
                new List<TreeBlobReference>(0)
            );

            // Persist the trees:
            var trTask = await tc.trrepo.PersistTree(trRoot.ID, new ImmutableContainer<TreeID, TreeNode>(tr => tr.ID, trTemplate, trPages, trRoot));

            output(new TreePathStreamedBlob(trRoot.ID, (CanonicalBlobPath)"/template/header", sblobs[0].Value));
            output(new TreePathStreamedBlob(trRoot.ID, (CanonicalBlobPath)"/template/footer", sblobs[1].Value));
            assertTranslated(
                tc,
                sblobs[2].Value,
                trRoot.ID,
@"<div>
  Else here?
</div>"
            );
        }