public async Task WriteOidAsync()
{
MemoryStream ms = new MemoryStream();
BsonDataWriter writer = new BsonDataWriter(ms);
byte[] oid = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 };
await writer.WriteStartObjectAsync();
await writer.WritePropertyNameAsync("_oid");
writer.WriteObjectId(oid);
await writer.WriteEndObjectAsync();
string bson = BytesToHex(ms.ToArray());
Assert.AreEqual("17-00-00-00-07-5F-6F-69-64-00-01-02-03-04-05-06-07-08-09-0A-0B-0C-00", bson);
ms.Seek(0, SeekOrigin.Begin);
BsonDataReader reader = new BsonDataReader(ms);
Assert.IsTrue(await reader.ReadAsync());
Assert.AreEqual(JsonToken.StartObject, reader.TokenType);
Assert.IsTrue(await reader.ReadAsync());
Assert.AreEqual(JsonToken.PropertyName, reader.TokenType);
Assert.IsTrue(await reader.ReadAsync());
Assert.AreEqual(JsonToken.Bytes, reader.TokenType);
CollectionAssert.AreEquivalent(oid, (byte[])reader.Value);
Assert.IsTrue(await reader.ReadAsync());
Assert.AreEqual(JsonToken.EndObject, reader.TokenType);
}
public async Task WriteBytesAsync()
{
byte[] data = Encoding.UTF8.GetBytes("Hello world!");
MemoryStream ms = new MemoryStream();
BsonDataWriter writer = new BsonDataWriter(ms);
await writer.WriteStartArrayAsync();
await writer.WriteValueAsync("a");
await writer.WriteValueAsync("b");
await writer.WriteValueAsync(data);
await writer.WriteEndArrayAsync();
await writer.FlushAsync();
ms.Seek(0, SeekOrigin.Begin);
string expected = "2B-00-00-00-02-30-00-02-00-00-00-61-00-02-31-00-02-00-00-00-62-00-05-32-00-0C-00-00-00-00-48-65-6C-6C-6F-20-77-6F-72-6C-64-21-00";
string bson = BytesToHex(ms.ToArray());
Assert.AreEqual(expected, bson);
BsonDataReader reader = new BsonDataReader(new MemoryStream(ms.ToArray()));
reader.ReadRootValueAsArray = true;
await reader.ReadAsync();
await reader.ReadAsync();
await reader.ReadAsync();
await reader.ReadAsync();
Assert.AreEqual(JsonToken.Bytes, reader.TokenType);
CollectionAssert.AreEquivalent(data, (byte[])reader.Value);
}
public async Task WriteBigIntegerAsync()
{
BigInteger i = BigInteger.Parse("1999999999999999999999999999999999999999999999999999999999990");
MemoryStream ms = new MemoryStream();
BsonDataWriter writer = new BsonDataWriter(ms);
await writer.WriteStartObjectAsync();
await writer.WritePropertyNameAsync("Blah");
await writer.WriteValueAsync(i);
await writer.WriteEndObjectAsync();
string bson = BytesToHex(ms.ToArray());
Assert.AreEqual("2A-00-00-00-05-42-6C-61-68-00-1A-00-00-00-00-F6-FF-FF-FF-FF-FF-FF-1F-B2-21-CB-28-59-84-C4-AE-03-8A-44-34-2F-4C-4E-9E-3E-01-00", bson);
ms.Seek(0, SeekOrigin.Begin);
BsonDataReader reader = new BsonDataReader(ms);
Assert.IsTrue(await reader.ReadAsync());
Assert.AreEqual(JsonToken.StartObject, reader.TokenType);
Assert.IsTrue(await reader.ReadAsync());
Assert.AreEqual(JsonToken.PropertyName, reader.TokenType);
Assert.IsTrue(await reader.ReadAsync());
Assert.AreEqual(JsonToken.Bytes, reader.TokenType);
CollectionAssert.AreEqual(new byte[] { 246, 255, 255, 255, 255, 255, 255, 31, 178, 33, 203, 40, 89, 132, 196, 174, 3, 138, 68, 52, 47, 76, 78, 158, 62, 1 }, (byte[])reader.Value);
Assert.AreEqual(i, new BigInteger((byte[])reader.Value));
Assert.IsTrue(await reader.ReadAsync());
Assert.AreEqual(JsonToken.EndObject, reader.TokenType);
Assert.IsFalse(await reader.ReadAsync());
}
public async Task WriteDateTimesAsync()
{
MemoryStream ms = new MemoryStream();
BsonDataWriter writer = new BsonDataWriter(ms);
writer.DateTimeKindHandling = DateTimeKind.Unspecified;
await writer.WriteStartArrayAsync();
await writer.WriteValueAsync(new DateTime(2000, 10, 12, 20, 55, 0, DateTimeKind.Utc));
await writer.WriteValueAsync(new DateTime(2000, 10, 12, 20, 55, 0, DateTimeKind.Local));
await writer.WriteValueAsync(new DateTime(2000, 10, 12, 20, 55, 0, DateTimeKind.Unspecified));
await writer.WriteEndArrayAsync();
ms.Seek(0, SeekOrigin.Begin);
BsonDataReader reader = new BsonDataReader(ms);
reader.ReadRootValueAsArray = true;
reader.DateTimeKindHandling = DateTimeKind.Utc;
Assert.IsTrue(await reader.ReadAsync());
Assert.AreEqual(JsonToken.StartArray, reader.TokenType);
Assert.IsTrue(await reader.ReadAsync());
Assert.AreEqual(JsonToken.Date, reader.TokenType);
Assert.AreEqual(new DateTime(2000, 10, 12, 20, 55, 0, DateTimeKind.Utc), reader.Value);
Assert.IsTrue(await reader.ReadAsync());
Assert.AreEqual(JsonToken.Date, reader.TokenType);
Assert.AreEqual(new DateTime(2000, 10, 12, 20, 55, 0, DateTimeKind.Utc), reader.Value);
Assert.IsTrue(await reader.ReadAsync());
Assert.AreEqual(JsonToken.Date, reader.TokenType);
Assert.AreEqual(new DateTime(2000, 10, 12, 20, 55, 0, DateTimeKind.Utc), reader.Value);
Assert.IsTrue(await reader.ReadAsync());
Assert.AreEqual(JsonToken.EndArray, reader.TokenType);
Assert.IsFalse(await reader.ReadAsync());
}
public async Task WriteReadEmptyAndNullStringsAsync()
{
MemoryStream ms = new MemoryStream();
BsonDataWriter writer = new BsonDataWriter(ms);
await writer.WriteStartArrayAsync();
await writer.WriteValueAsync("Content!");
await writer.WriteValueAsync("");
await writer.WriteValueAsync((string)null);
await writer.WriteEndArrayAsync();
ms.Seek(0, SeekOrigin.Begin);
BsonDataReader reader = new BsonDataReader(ms);
reader.ReadRootValueAsArray = true;
Assert.IsTrue(await reader.ReadAsync());
Assert.AreEqual(JsonToken.StartArray, reader.TokenType);
Assert.IsTrue(await reader.ReadAsync());
Assert.AreEqual(JsonToken.String, reader.TokenType);
Assert.AreEqual("Content!", reader.Value);
Assert.IsTrue(await reader.ReadAsync());
Assert.AreEqual(JsonToken.String, reader.TokenType);
Assert.AreEqual("", reader.Value);
Assert.IsTrue(await reader.ReadAsync());
Assert.AreEqual(JsonToken.Null, reader.TokenType);
Assert.AreEqual(null, reader.Value);
Assert.IsTrue(await reader.ReadAsync());
Assert.AreEqual(JsonToken.EndArray, reader.TokenType);
Assert.IsFalse(await reader.ReadAsync());
}
async Task <IReadOnlyDictionary <string, FileFingerprint> > LoadBlobsImplAsync(CancellationToken cancellationToken)
{
var blobs = new Dictionary <string, FileFingerprint>();
var needRebuild = false;
var blobCount = 0;
var fileCount = 0;
_fileSequence.Rescan();
var totalSize = 0L;
var compressedSize = 0L;
foreach (var fileInfo in _fileSequence.Files)
{
++fileCount;
try
{
fileInfo.Refresh();
if (fileInfo.Length < 5)
{
continue;
}
using (var fileStream = OpenBsonFileForRead(fileInfo))
using (var decodeStream = new DeflateStream(fileStream, CompressionMode.Decompress))
using (var bs = new SequentialReadStream(decodeStream))
using (var br = new BsonDataReader(bs)
{
DateTimeKindHandling = DateTimeKind.Utc, SupportMultipleContent = true
})
{
while (await br.ReadAsync(cancellationToken).ConfigureAwait(false))
{
cancellationToken.ThrowIfCancellationRequested();
try
{
var fileFingerprint = await ReadFileFingerprintAsync(br, cancellationToken).ConfigureAwait(false);
if (null == fileFingerprint)
{
needRebuild = true;
break;
}
if (blobs.ContainsKey(fileFingerprint.FullFilePath))
{
Debug.WriteLine($"Collision for {fileFingerprint.FullFilePath}");
}
blobs[fileFingerprint.FullFilePath] = fileFingerprint;
++blobCount;
}
catch (IOException ex)
{
needRebuild = true;
// The entry might or might not be valid.
Debug.WriteLine("BsonFileFingerprintStore.LoadBlobsImplAsync() read failed: " + ex.Message);
}
}
totalSize += bs.Position;
compressedSize += fileStream.Length;
}
}
catch (IOException)
{
needRebuild = true;
}
catch (InvalidDataException)
{
needRebuild = true;
}
catch (JsonException)
{
needRebuild = true;
}
}
Debug.WriteLine($"Read {totalSize.BytesToMiB():F2}MiB bytes from {compressedSize.BytesToMiB():F2}MiB file");
var count = (double)blobs.Count;
Debug.WriteLine($"Average size {totalSize / count:F1} bytes or {compressedSize / count:F1} compressed");
if (blobCount > blobs.Count + 100 + blobs.Count / 8)
{
needRebuild = true;
}
if (fileCount > 16)
{
needRebuild = true;
}
if (needRebuild)
{
Console.WriteLine("Rebuilding cache files");
await RebuildCacheAsync(blobs, cancellationToken).ConfigureAwait(false);
}
return(blobs);
}