public void RestoreCollectionData(FileInfo fileInfo, IMongoDatabase database)
{
string collectionName = fileInfo.Name.Replace(".bson", "");
var collection = database.GetCollection <BsonDocument>(collectionName);
var items = new List <BsonDocument>();
StreamReader sr = null;
try
{
sr = new StreamReader(fileInfo.FullName);
BsonReader bsonReader = new BsonBinaryReader(sr.BaseStream);
while (!bsonReader.IsAtEndOfFile())
{
var bson = BsonSerializer.Deserialize <BsonDocument>(bsonReader);
items.Add(bson);
}
if (items.Any())
{
collection.InsertMany(items);
}
else
{
//this is a hack to force creation of the collection
var doc = new BsonDocument();
collection.InsertOne(doc);
collection.DeleteMany(doc);
}
}
finally
{
sr?.Close();
}
}
public void BsonBinaryReader_should_support_reading_multiple_documents(
[Range(0, 3)]
int numberOfDocuments)
{
var document = new BsonDocument("x", 1);
var bson = document.ToBson();
var input = Enumerable.Repeat(bson, numberOfDocuments).Aggregate(Enumerable.Empty <byte>(), (a, b) => a.Concat(b)).ToArray();
var expectedResult = Enumerable.Repeat(document, numberOfDocuments);
using (var stream = new MemoryStream(input))
using (var binaryReader = new BsonBinaryReader(stream))
{
var result = new List <BsonDocument>();
while (!binaryReader.IsAtEndOfFile())
{
binaryReader.ReadStartDocument();
var name = binaryReader.ReadName();
var value = binaryReader.ReadInt32();
binaryReader.ReadEndDocument();
var resultDocument = new BsonDocument(name, value);
result.Add(resultDocument);
}
result.Should().Equal(expectedResult);
}
}
public void TestIsAtEndOfFileWithTwoDocuments()
{
var expected = new BsonDocument("x", 1);
byte[] bson;
using (var stream = new MemoryStream())
using (var writer = new BsonBinaryWriter(stream))
{
BsonSerializer.Serialize(writer, expected);
BsonSerializer.Serialize(writer, expected);
bson = stream.ToArray();
}
using (var stream = new MemoryStream(bson))
using (var reader = new BsonBinaryReader(stream))
{
var count = 0;
while (!reader.IsAtEndOfFile())
{
var document = BsonSerializer.Deserialize <BsonDocument>(reader);
Assert.Equal(expected, document);
count++;
}
Assert.Equal(2, count);
}
}
public void BsonBinaryReader_should_support_reading_more_than_2GB()
{
RequireEnvironment.Check().EnvironmentVariable("EXPLICIT");
var binaryData = new BsonBinaryData(new byte[1024 * 1024]);
var tempFileName = Path.GetTempFileName();
try
{
using (var stream = new FileStream(tempFileName, FileMode.Open))
{
using (var binaryWriter = new BsonBinaryWriter(stream))
{
while (stream.Position < (long)int.MaxValue * 4)
{
binaryWriter.WriteStartDocument();
binaryWriter.WriteName("x");
binaryWriter.WriteBinaryData(binaryData);
binaryWriter.WriteEndDocument();
}
}
var endOfFilePosition = stream.Position;
stream.Position = 0;
using (var binaryReader = new BsonBinaryReader(stream))
{
while (!binaryReader.IsAtEndOfFile())
{
binaryReader.ReadStartDocument();
var bookmark = binaryReader.GetBookmark();
binaryReader.ReadName("x");
binaryReader.ReturnToBookmark(bookmark);
binaryReader.ReadName("x");
var readBinaryData = binaryReader.ReadBinaryData();
Assert.Equal(binaryData.Bytes.Length, readBinaryData.Bytes.Length);
binaryReader.ReadEndDocument();
}
}
Assert.Equal(endOfFilePosition, stream.Position);
}
}
finally
{
try
{
File.Delete(tempFileName);
}
catch
{
// ignore exceptions
}
}
}
private void BsonToJson(string input, string output, CancellationToken token)
{
string[] files = Directory.GetFiles(Path.GetDirectoryName(input), Path.GetFileName(input), SearchOption.TopDirectoryOnly);
foreach (string file in files)
{
token.ThrowIfCancellationRequested();
FileInfo fileToConvert = new FileInfo(file);
string outputFile = Path.Combine(output, Path.GetFileNameWithoutExtension(file) + ".json");
string outputDir = Path.GetDirectoryName(outputFile);
Directory.CreateDirectory(outputDir);
using (FileStream originalFileStream = fileToConvert.OpenRead())
{
if ((File.GetAttributes(fileToConvert.FullName) &
FileAttributes.Hidden) != FileAttributes.Hidden & fileToConvert.Extension != ".json")
{
using (FileStream jsonFileStream = File.Create(outputFile))
{
StreamWriter writer = new StreamWriter(jsonFileStream, Encoding.UTF8);
using (var reader = new BsonBinaryReader(originalFileStream))
{
while (!reader.IsAtEndOfFile())
{
token.ThrowIfCancellationRequested();
var bson = BsonSerializer.Deserialize <BsonDocument>(reader);
string json = bson.ToJson(new JsonWriterSettings()
{
OutputMode = JsonOutputMode.CanonicalExtendedJson
});
writer.Write(json);
}
writer.Flush();
}
}
}
FileInfo info = new FileInfo(outputFile);
_logger.Information("Converted {From} => {To}", fileToConvert.Name, info.Name);
}
}
}
private void RunDecodeErrorsTest(BsonDocument test)
{
JsonDrivenHelper.EnsureAllFieldsAreValid(test, "type", "description", "bson");
var bson = BsonUtils.ParseHexString(test["bson"].AsString);
var exception = Record.Exception(() =>
{
using (var stream = new MemoryStream(bson))
using (var reader = new BsonBinaryReader(stream))
{
while (!reader.IsAtEndOfFile())
{
_ = BsonSerializer.Deserialize <BsonDocument>(reader);
}
}
});
AssertException(exception);
}
private void Open(String path)
{
// TODO - test for file/directory
if (!File.Exists(path))
{
throw new FTDCException($"Cannot load ${path}");
}
using var sourceStream = new FileStream(path, FileMode.Open);
using var reader = new BsonBinaryReader(sourceStream);
while (!reader.IsAtEndOfFile())
{
var result = BsonSerializer.Deserialize <FTDCFileContents>(reader);
// FIXME - we should only decompress the data we actually want to use, but this will
// have to wait until we have timeseries parsing in place
result.DecompressData();
Contents.Add(result);
}
MetricsStart = Contents.Min(d => d._id);
MetricsEnd = Contents.Max(d => d._id);
}
private void BsonToSql(string tableName, string input, CancellationToken token)
{
const string PARAM_FILENAME = "@fileName";
const string PARAM_DOCUMENT = "@document";
const string PARAM_PROCESSID = "@processId";
const string QUERY = @"insert {0} (fileName,document,processId) values (@fileName,@document,@processId);";
using (var cn = new SqlConnection(_attributes[CONNECTION_STRING]))
{
cn.Open();
try
{
using (SqlCommand cmd = new SqlCommand(String.Format(QUERY, tableName), cn))
{
cmd.Parameters.Add(PARAM_FILENAME, SqlDbType.NVarChar, 500);
cmd.Parameters.Add(PARAM_DOCUMENT, SqlDbType.NVarChar, -1);
var param = cmd.Parameters.Add(PARAM_PROCESSID, SqlDbType.Int);
param.Value = Int32.Parse(_attributes[RUNID]);
cmd.Prepare();
using (token.Register(cmd.Cancel))
{
string[] files = Directory.GetFiles(Path.GetDirectoryName(input), Path.GetFileName(input), SearchOption.TopDirectoryOnly);
foreach (string file in files)
{
token.ThrowIfCancellationRequested();
CheckTable(tableName, token);
cmd.Parameters[PARAM_FILENAME].Value = file;
FileInfo fileToLoad = new FileInfo(file);
using (FileStream originalFileStream = fileToLoad.OpenRead())
{
if ((File.GetAttributes(fileToLoad.FullName) &
FileAttributes.Hidden) != FileAttributes.Hidden & fileToLoad.Extension != ".json")
{
using (var reader = new BsonBinaryReader(originalFileStream))
{
while (!reader.IsAtEndOfFile())
{
token.ThrowIfCancellationRequested();
var bson = BsonSerializer.Deserialize <BsonDocument>(reader);
string json = bson.ToJson(new JsonWriterSettings()
{
OutputMode = JsonOutputMode.CanonicalExtendedJson
});
cmd.Parameters[PARAM_DOCUMENT].Value = json;
cmd.ExecuteNonQuery();
}
}
}
_logger.Information("Loaded {From} => {To}", fileToLoad.Name, tableName);
}
}
}
}
}
catch (SqlException ex)
{
throw ex;
//_logger.Write(String.Format("SqlServer exception: {0}", ex.Message));
//result = WfResult.Create(WfStatus.Failed, ex.Message, ex.ErrorCode);
}
finally
{
if (cn.State != ConnectionState.Closed)
{
cn.Close();
}
}
}
}
public static string ConvertToJson(string bsonString)
{
StringBuilder jsonString = new StringBuilder();
string nameStr;
string valueStr;
Stack <string> stack = new Stack <string>();
string stackTop;
Stack <bool> stackFirst = new Stack <bool>();
byte[] buffer = Convert.FromBase64String(bsonString);
string text = Encoding.UTF8.GetString(buffer);
using (var stream = new MemoryStream(buffer))
using (var reader = new BsonBinaryReader(stream))
{
bool exit = false;
bool firstElem = true;
reader.ReadStartDocument();
jsonString.Append("{ ");
while (!reader.IsAtEndOfFile() && !exit)
{
var bsonType = reader.ReadBsonType(); //.CurrentBsonType;
switch (bsonType)
{
case MongoDB.Bson.BsonType.Array:
nameStr = reader.ReadName();
reader.ReadStartArray();
if (firstElem)
{
firstElem = false;
}
else
{
jsonString.Append(", ");
}
jsonString.Append($"\"{nameStr}\" : [ ");
stack.Push("Array");
stackFirst.Push(firstElem);
firstElem = true;
break;
case MongoDB.Bson.BsonType.Binary:
break;
case MongoDB.Bson.BsonType.Boolean:
nameStr = reader.ReadName();
valueStr = reader.ReadBoolean().ToString();
if (firstElem)
{
firstElem = false;
}
else
{
jsonString.Append(", ");
}
jsonString.Append($"\"{nameStr}\" : \"{valueStr}\"");
break;
case MongoDB.Bson.BsonType.DateTime:
nameStr = reader.ReadName();
valueStr = reader.ReadString();
if (firstElem)
{
firstElem = false;
}
else
{
jsonString.Append(", ");
}
jsonString.Append($"\"{nameStr}\" : \"{valueStr}\"");
break;
case MongoDB.Bson.BsonType.Decimal128:
nameStr = reader.ReadName();
valueStr = reader.ReadDecimal128().ToString();
if (firstElem)
{
firstElem = false;
}
else
{
jsonString.Append(", ");
}
jsonString.Append($"\"{nameStr}\" : \"{valueStr}\"");
break;
case MongoDB.Bson.BsonType.Document:
nameStr = reader.ReadName();
reader.ReadStartDocument();
if (firstElem)
{
firstElem = false;
}
else
{
jsonString.Append(", ");
}
jsonString.Append($"\"{nameStr}\" : {{ ");
stack.Push("Document");
stackFirst.Push(firstElem);
firstElem = true;
break;
case MongoDB.Bson.BsonType.Double:
nameStr = reader.ReadName();
valueStr = reader.ReadDouble().ToString();
if (firstElem)
{
firstElem = false;
}
else
{
jsonString.Append(", ");
}
jsonString.Append($"\"{nameStr}\" : \"{valueStr}\"");
break;
case MongoDB.Bson.BsonType.EndOfDocument:
if (stack.Count == 0)
{
jsonString.Append("} ");
exit = true;
}
else
{
stackTop = stack.Pop();
firstElem = stackFirst.Pop();
switch (stackTop)
{
case "Array":
jsonString.Append("]");
reader.ReadEndArray();
break;
case "Document":
jsonString.Append("}");
reader.ReadEndDocument();
break;
default:
break;
}
}
break;
case MongoDB.Bson.BsonType.Int32:
nameStr = reader.ReadName();
valueStr = reader.ReadInt32().ToString();
if (firstElem)
{
firstElem = false;
}
else
{
jsonString.Append(", ");
}
jsonString.Append($"\"{nameStr}\" : \"{valueStr}\"");
break;
case MongoDB.Bson.BsonType.Int64:
nameStr = reader.ReadName();
valueStr = reader.ReadInt64().ToString();
if (firstElem)
{
firstElem = false;
}
else
{
jsonString.Append(", ");
}
jsonString.Append($"\"{nameStr}\" : \"{valueStr}\"");
break;
case MongoDB.Bson.BsonType.JavaScript:
break;
case MongoDB.Bson.BsonType.JavaScriptWithScope:
break;
case MongoDB.Bson.BsonType.MaxKey:
nameStr = reader.ReadName();
reader.ReadMaxKey();
valueStr = "";
jsonString.Append($"\"{nameStr}\" : \"{valueStr}\"");
break;
case MongoDB.Bson.BsonType.MinKey:
nameStr = reader.ReadName();
reader.ReadMinKey();
valueStr = "";
jsonString.Append($"\"{nameStr}\" : \"{valueStr}\"");
break;
case MongoDB.Bson.BsonType.Null:
nameStr = reader.ReadName();
reader.ReadNull();
valueStr = "null";
if (firstElem)
{
firstElem = false;
}
else
{
jsonString.Append(", ");
}
jsonString.Append($"\"{nameStr}\" : \"{valueStr}\"");
break;
case MongoDB.Bson.BsonType.ObjectId:
nameStr = reader.ReadName();
valueStr = reader.ReadObjectId().ToString();
if (firstElem)
{
firstElem = false;
}
else
{
jsonString.Append(", ");
}
jsonString.Append($"\"{nameStr}\" : \"{valueStr}\"");
break;
case MongoDB.Bson.BsonType.RegularExpression:
break;
case MongoDB.Bson.BsonType.String:
if (firstElem)
{
firstElem = false;
}
else
{
jsonString.Append(", ");
}
if (reader.State == BsonReaderState.Name)
{
nameStr = reader.ReadName();
jsonString.Append($"\"{nameStr}\" : ");
}
valueStr = reader.ReadString();
jsonString.Append($"\"{valueStr}\"");
break;
case MongoDB.Bson.BsonType.Symbol:
nameStr = reader.ReadName();
valueStr = reader.ReadSymbol();
if (firstElem)
{
firstElem = false;
}
else
{
jsonString.Append(", ");
}
jsonString.Append($"\"{nameStr}\" : \"{valueStr}\"");
break;
case MongoDB.Bson.BsonType.Timestamp:
nameStr = reader.ReadName();
valueStr = reader.ReadTimestamp().ToString();
if (firstElem)
{
firstElem = false;
}
else
{
jsonString.Append(", ");
}
jsonString.Append($"\"{nameStr}\" : \"{valueStr}\"");
break;
case MongoDB.Bson.BsonType.Undefined:
nameStr = reader.ReadName();
reader.ReadUndefined();
valueStr = "undefined";
if (firstElem)
{
firstElem = false;
}
else
{
jsonString.Append(", ");
}
jsonString.Append($"\"{nameStr}\" : \"{valueStr}\"");
break;
default:
break;
}
}
reader.ReadEndDocument();
}
return(jsonString.ToString());;
}
