internal async Task PersistBlobsTest()
{
const int numBlobs = 16;
// Create an immutable container that points to the new blobs:
Console.WriteLine("Creating {0} random blobs...", numBlobs);
PersistingBlob[] blobs = createBlobs(numBlobs);
// Now persist those blobs to the filesystem:
Console.WriteLine("Persisting {0} random blobs...", numBlobs);
Stopwatch sw = Stopwatch.StartNew();
var streamedBlobs = await blrepo.PersistBlobs(blobs);
sw.Stop();
Console.WriteLine("Completed in {0} ms, {1} bytes/sec", sw.ElapsedMilliseconds, streamedBlobs.Sum(b => b.Value.Length) * 1000d / sw.ElapsedMilliseconds);
// Repeat!
for (int i = 0; i < numBlobs; ++i)
{
var fs = (FileStream)blobs[i].Stream;
// Re-open the temporary filestream:
blobs[i] = new PersistingBlob(new FileStream(fs.Name, FileMode.Open, FileAccess.Read, FileShare.Read, 8040, true));
}
sw = Stopwatch.StartNew();
streamedBlobs = await blrepo.PersistBlobs(blobs);
sw.Stop();
Console.WriteLine("Completed in {0} ms, {1} bytes/sec", sw.ElapsedMilliseconds, streamedBlobs.Sum(b => b.Value.Length) * 1000d / sw.ElapsedMilliseconds);
}
private async Task createCommits()
{
PersistingBlob pblReadme = new PersistingBlob("Readme file.".ToStream());
var sblobs = await blrepo.PersistBlobs(pblReadme);
trRoot = new TreeNode.Builder(
new List <TreeTreeReference>(0),
new List <TreeBlobReference>
{
new TreeBlobReference.Builder("README", sblobs[0].Value.ID)
}
);
var trees = new ImmutableContainer <TreeID, TreeNode>(tr => tr.ID, trRoot);
await trrepo.PersistTree(trRoot.ID, trees);
TreeRepositoryTestMethods.RecursivePrint(trRoot.ID, trees);
cmRoot = new Commit.Builder(
new List <CommitID>(0),
trRoot.ID,
"James S. Dunne",
DateTimeOffset.Now,
"Hello world."
);
await cmrepo.PersistCommit(cmRoot);
}
private async Task createTrees()
{
PersistingBlob pbHeader = new PersistingBlob("<div>Header</div>".ToStream());
sblobs = await blrepo.PersistBlobs(pbHeader);
trSection1 = new TreeNode.Builder();
trSection2 = new TreeNode.Builder();
trSection3 = new TreeNode.Builder();
trImages = new TreeNode.Builder();
trCSS = new TreeNode.Builder();
trJS = new TreeNode.Builder();
trPages = new TreeNode.Builder(
new List <TreeTreeReference> {
new TreeTreeReference.Builder("section1", trSection1.ID),
new TreeTreeReference.Builder("section2", trSection2.ID),
new TreeTreeReference.Builder("section3", trSection3.ID)
},
new List <TreeBlobReference>(0)
);
trContent = new TreeNode.Builder(
new List <TreeTreeReference> {
new TreeTreeReference.Builder("images", trImages.ID),
new TreeTreeReference.Builder("css", trCSS.ID),
new TreeTreeReference.Builder("js", trJS.ID)
},
new List <TreeBlobReference>(0)
);
trTemplate = new TreeNode.Builder(
new List <TreeTreeReference>(0),
new List <TreeBlobReference> {
new TreeBlobReference.Builder("header", sblobs[0].Value.ID)
}
);
trRoot = new TreeNode.Builder(
new List <TreeTreeReference> {
new TreeTreeReference.Builder("template", trTemplate.ID),
new TreeTreeReference.Builder("content", trContent.ID),
new TreeTreeReference.Builder("pages", trPages.ID)
},
new List <TreeBlobReference>(0)
);
rootId = trRoot.ID;
trees = new ImmutableContainer <TreeID, TreeNode>(tr => tr.ID, trSection1, trSection2, trSection3, trPages, trImages, trCSS, trJS, trContent, trTemplate, trRoot);
RecursivePrint(trRoot.ID, trees);
}
private PersistingBlob[] createBlobs(int numBlobs)
{
PersistingBlob[] blobs = new PersistingBlob[numBlobs];
for (int i = 0; i < numBlobs; ++i)
{
string tmpFileName = System.IO.Path.GetTempFileName();
using (var fs = new FileStream(tmpFileName, FileMode.Open, FileAccess.Write, FileShare.Read, 1048576, true))
{
new RandomDataStream(1048576 * 2).CopyTo(fs);
}
blobs[i] = new PersistingBlob(new FileStream(tmpFileName, FileMode.Open, FileAccess.Read, FileShare.Read, 8040, true));
}
return(blobs);
}
public async Task <Errorable <IStreamedBlob> > PersistBlob(PersistingBlob blob)
{
SqlCommand insert, update;
insert = new SqlCommand(@"INSERT INTO [dbo].[Blob] ([blobid], [contents]) VALUES (@pk, @chunk)");
insert.Parameters.Add("@pk", System.Data.SqlDbType.Binary, 21);
insert.Parameters.Add("@chunk", System.Data.SqlDbType.Binary);
update = new SqlCommand(@"UPDATE [dbo].[Blob] SET [contents].WRITE(@chunk, NULL, NULL) WHERE [blobid] = @pk");
update.Parameters.Add("@pk", System.Data.SqlDbType.Binary, 21);
update.Parameters.Add("@chunk", System.Data.SqlDbType.Binary);
// Create a random dummy ID for uploading:
// FIXME: we need guaranteed uniqueness!
byte[] dummyID = new byte[21];
using (var rng = System.Security.Cryptography.RandomNumberGenerator.Create())
rng.GetBytes(dummyID);
dummyID[20] = 0xFF;
// Create a BlobWriter to INSERT and UPDATE to the database in chunks:
var bw = new BlobWriter(db, insert, update, dummyID);
// Create a SHA1 calculating wrapper around the input stream:
var sha1Reader = new SHA1StreamReader(blob.Stream);
// Asynchronously upload the blob:
await bw.UploadAsync(sha1Reader);
// Get the hash and use that as the BlobID:
byte[] blobHash = sha1Reader.GetHash();
// Now update the Blob record with the new BlobID:
await db.ExecuteNonQueryAsync(new BlobIDUpdate(dummyID, blobHash));
// Return the StreamedBlob instance that can read from the uploaded blob record:
return(new StreamedBlob(this, new BlobID(blobHash), bw.Length));
}
private async Task <Errorable <IStreamedBlob> > persistBlob(PersistingBlob blob)
{
Debug.WriteLine(String.Format("Starting persistence of blob"));
// Find a temporary filename:
FileInfo tmpPath = system.getTemporaryFile();
long length = -1;
BlobID blid;
// Open a new stream to the source blob contents:
using (var sr = blob.Stream)
{
length = sr.Length;
// Create a new file and set its length so we can asynchronously write to it:
using (var tmpFi = File.Open(tmpPath.FullName, FileMode.CreateNew, FileAccess.Write, FileShare.None))
{
Debug.WriteLine(String.Format("New BLOB temp '{0}' length {1}", tmpPath.FullName, length));
tmpFi.SetLength(length);
tmpFi.Close();
}
// Determine the best buffer size to use for writing contents:
int bufSize = Math.Min(Math.Max((int)length, 8), largeBufferSize);
// Open a new FileStream to asynchronously write the blob contents:
using (var fs = new FileStream(tmpPath.FullName, FileMode.Open, FileAccess.Write, FileShare.Read, bufSize, useAsync: true))
using (var sha1 = new SHA1StreamWriter(fs))
{
// Copy the contents asynchronously (expected copy in order):
await sr.CopyToAsync(sha1, bufSize).ConfigureAwait(continueOnCapturedContext: false);
// Create the BlobID from the SHA1 hash calculated during copy:
blid = new BlobID(sha1.GetHash());
}
}
// Serialize access to the official blob file:
lock (FileSystem.SystemLock)
{
// Create the blob's subdirectory under 'objects':
FileInfo path = system.getPathByID(blid);
path.Refresh();
if (!path.Directory.Exists)
{
Debug.WriteLine(String.Format("New DIR '{0}'", path.Directory.FullName));
path.Directory.Create();
}
// Don't recreate an existing blob:
if (path.Exists)
{
Debug.WriteLine(String.Format("Blob already exists at path '{0}', deleting temporary...", path.FullName));
tmpPath.Delete();
return(new Errorable <IStreamedBlob>((IStreamedBlob) new StreamedBlob(this, blid, length)));
}
// Move the temp file to the final blob filename:
File.Move(tmpPath.FullName, path.FullName);
}
return(new Errorable <IStreamedBlob>((IStreamedBlob) new StreamedBlob(this, blid, length)));
}
public Task <Errorable <IStreamedBlob> > PersistBlob(PersistingBlob blob)
{
return(persistBlob(blob));
}
public PersistBlob(StreamedBlobRepository blrepo, PersistingBlob bl)
{
this._bl = bl;
this._blrepo = blrepo;
this._length = -1;
}