public BinaryDataObjectSplitterResult(
GenericDataContainer containerWithoutBinaryData,
byte[] binaryData)
{
ContainerWithoutBinaryData = containerWithoutBinaryData ?? throw new ArgumentNullException(nameof(containerWithoutBinaryData));
BinaryData = binaryData ?? throw new ArgumentNullException(nameof(binaryData));
}
public GenericDataContainer Reassemble(GenericDataContainer containerWithoutBinaryData, byte[] binaryData)
{
var containerWithData = CloneGenericDataContainer(containerWithoutBinaryData);
containerWithData.Data.Set(binaryDataPath, BsonString.Create(Convert.ToBase64String(binaryData)));
return(containerWithData);
}
public async Task <StoreResult> StoreAsync(string dataType, GenericDataContainer container, bool overwrite)
{
var id = container.Id;
var exists = await ExistsAsync(dataType, id);
if (exists && !overwrite)
{
throw new DocumentAlreadyExistsException($"Object of type '{dataType}' with ID '{id}' already exists");
}
var isNewCollection = await IsNewCollection(dataType);
var binaryDataObjectSplitter = binaryDataObjectSplitters[dataType];
var splitResult = binaryDataObjectSplitter.Split(container);
await metadataStorage.StoreAsync(dataType, splitResult.ContainerWithoutBinaryData, true);
var blobContainer = binaryDataStorage.GetContainer(dataType);
await blobContainer.CreateIfNotExistsAsync();
var blob = blobContainer.GetBlob(id);
await blob.WriteAsync(splitResult.BinaryData);
var dataModificationType = exists
? DataModificationType.Replaced
: DataModificationType.Created;
return(new StoreResult(id, dataModificationType, isNewCollection));
}
public static string CreateFromContainer(string dataType, GenericDataContainer container)
{
var keyValuePairs = GenericDataContainerTraversal.Traverse(container);
var query = $"UPDATE {dataType} SET ";
var firstItem = true;
foreach (var(key, value) in keyValuePairs)
{
if (key.InSet("Id", "CreatedTimeUtc", "OriginalSubmitter"))
{
continue;
}
if (!firstItem)
{
query += ", ";
}
else
{
firstItem = false;
}
if (key == "Id")
{
query += $"{key}={value}";
}
else
{
query += $"{key}='{value}'";
}
}
query += $" WHERE Id='{container.Id}'";
return(query);
}
private async Task <GenericDataContainer> BuildContainer(SqlDataReader reader, SqlTableSetup tableSetup)
{
var id = (int)reader[tableSetup.IdColumnName];
var metadata = await metadataStorage.GetFromIdAsync(tableSetup.DataType, id.ToString());
if (metadata == null)
{
var utcNow = DateTime.UtcNow;
metadata = new GenericDataContainer(
id.ToString(),
"unknown",
utcNow,
"unknown",
utcNow,
ApiVersion.Current,
new BsonDocument());
}
var data = BsonDocumentBuilder.BuildFromDelimitedColumns(reader, tableSetup);
return(new GenericDataContainer(
id.ToString(),
metadata.OriginalSubmitter,
metadata.CreatedTimeUtc,
metadata.Submitter,
metadata.SubmissionTimeUtc,
metadata.ApiVersion,
data));
}
public async Task <StoreResult> StoreAsync(string dataType, GenericDataContainer container, bool overwrite)
{
var isNewCollection = await IsNewCollectionAsync(dataType);
if (isNewCollection)
{
await CreateTable(dataType);
}
var exists = await ExistsAsync(dataType, container.Id);
if (exists && !overwrite)
{
throw new DocumentAlreadyExistsException($"Object of type '{dataType}' with ID '{container.Id}' already exists");
}
await EnsureColumnsExistAsync(dataType, container);
var query = exists
? SqlUpdateStatement.CreateFromContainer(dataType, container)
: SqlInsertStatement.CreateFromContainer(dataType, container);
await sqlQueryExecutor.ExecuteQueryAsync(query);
var modificationType = !exists
? DataModificationType.Created
: DataModificationType.Replaced;
return(new StoreResult(container.Id, modificationType, isNewCollection));
}
public async Task <StoreResult> StoreAsync(string dataType, GenericDataContainer container, bool overwrite)
{
var isNewCollection = await IsNewCollectionAsync(dataType);
var collection = rdDataClient.DataDatabase.GetCollection <GenericDataContainer>(dataType);
var id = container.Id;
if (overwrite)
{
var replaceResult = await collection.ReplaceOneAsync(x => x.Id == id, container, new ReplaceOptions { IsUpsert = true });
var dataModificationType = replaceResult.MatchedCount == 0 ? DataModificationType.Created : DataModificationType.Replaced;
return(new StoreResult(id, dataModificationType, isNewCollection));
}
try
{
await collection.InsertOneAsync(container);
return(new StoreResult(id, DataModificationType.Created, isNewCollection));
}
catch (MongoWriteException writeException)
{
if (writeException.WriteError.Category == ServerErrorCategory.DuplicateKey)
{
throw new DocumentAlreadyExistsException($"Object of type '{dataType}' with ID '{id}' already exists");
}
throw;
}
}
public static string CreateFromContainer(string dataType, GenericDataContainer container)
{
var keyValuePairs = GenericDataContainerTraversal.Traverse(container);
var query = $"INSERT INTO {dataType} ({string.Join(", ", keyValuePairs.Keys)}) "
+ $"VALUES ({string.Join(", ", keyValuePairs.Select(kvp => $"'{kvp.Value}'"))})";
return(query);
}
private BsonDocument MapContainerUsingSelect(GenericDataContainer container, DataApiSqlQuery parsedQuery)
{
var containerJObject = JObject.FromObject(container);
var dataJObject = JObject.Parse(DataEncoder.DecodeToJson(container.Data));
var fieldMappings = parsedQuery.SelectArguments.Split(',').Select(x => x.Trim());
var bsonDocument = new BsonDocument();
foreach (var fieldMapping in fieldMappings)
{
var match = Regex.Match(fieldMapping, "^(?<Path>[^\\s]+)(?<Map>\\s+AS\\s+(?<NewName>[^\\s]+))?", RegexOptions.IgnoreCase);
if (!match.Success)
{
throw new FormatException($"Invalid SELECT statement '{fieldMapping}'");
}
var fieldPath = match.Groups["Path"].Value;
string newName;
var hasMap = match.Groups["Map"].Success;
if (hasMap)
{
newName = match.Groups["NewName"].Value;
}
else
{
newName = fieldPath.Replace('.', '_');
}
JToken jToken;
if (fieldPath.StartsWith("Data."))
{
var token = hasMap ? newName : fieldPath.Substring("Data.".Length);
jToken = dataJObject.SelectToken(token);
}
else if (fieldPath == "Data")
{
jToken = dataJObject;
}
else
{
jToken = containerJObject.SelectToken(fieldPath);
}
BsonValue bsonValue;
if (jToken == null)
{
bsonValue = BsonNull.Value;
}
else if (jToken is JValue jValue)
{
bsonValue = BsonValue.Create(jValue.Value);
}
else
{
bsonValue = BsonDocument.Parse(jToken.ToString());
}
bsonDocument.Add(new BsonElement(newName, bsonValue));
}
return(bsonDocument);
}
public static Dictionary <string, string> Traverse(GenericDataContainer container)
{
var jObject = JObject.Parse(DataEncoder.DecodeToJson(container.Data));
var keyValuePairs = new Dictionary <string, string>
{
{ "Id", container.Id },
{ "OriginalSubmitter", container.OriginalSubmitter },
{ "CreatedTimeUtc", container.CreatedTimeUtc?.ToString("yyyy-MM-dd HH:mm:ss") },
{ "Submitter", container.Submitter },
{ "SubmissionTimeUtc", container.SubmissionTimeUtc.ToString("yyyy-MM-dd HH:mm:ss") }
};
TraverseJObject(jObject, keyValuePairs, "Data");
return(keyValuePairs);
}
private async Task EnsureColumnsExistAsync(string dataType, GenericDataContainer container)
{
var columnNames = GetColumnNamesFromContainer(container);
var existingColumns = await ListColumnsAsync(dataType);
var missingColumns = columnNames.Except(existingColumns).ToList();
if (!missingColumns.Any())
{
return;
}
var columnDefinitions = string.Join(',', missingColumns.Select(columnName => $"{columnName} nvarchar(100) NULL"));
var query = $"ALTER TABLE {dataType} ADD {columnDefinitions}";
await sqlQueryExecutor.ExecuteQueryAsync(query);
}
public BinaryDataObjectSplitterResult Split(GenericDataContainer container)
{
var payload = container.Data.DeepClone().AsBsonDocument;
var bsonValue = payload.GetValue(binaryDataPath);
var base64 = bsonValue.IsString ? bsonValue.AsString : null;
var binaryData = base64 != null?Convert.FromBase64String(base64) : new byte[0];
payload.Set(binaryDataPath, BsonString.Empty);
var containerWithoutData = new GenericDataContainer(
container.Id,
container.OriginalSubmitter,
container.CreatedTimeUtc,
container.Submitter,
container.SubmissionTimeUtc,
container.ApiVersion,
payload);
return(new BinaryDataObjectSplitterResult(containerWithoutData, binaryData));
}
private async Task <GenericDataContainer> FixMissingMetadata(string dataType, string id, byte[] binaryData)
{
// Assumption: Metadata doesn't exist
var blob = binaryDataStorage.GetBlob(dataType, id);
DateTime createdTimeUtc;
DateTime submissionTimeUtc;
if (await blob.ExistsAsync())
{
createdTimeUtc = blob.CreatedTimestampUtc;
submissionTimeUtc = blob.LastModifiedTimestampUtc;
}
else
{
var utcNow = DateTime.UtcNow;
createdTimeUtc = utcNow;
submissionTimeUtc = utcNow;
}
IId obj;
switch (dataType)
{
case nameof(DataBlob):
obj = new DataBlob(id, new byte[0]);
break;
case nameof(Image):
var imageFormat = ImageFormatDetector.Detect(binaryData);
var imageExtension = imageFormat != ImageFormatDetector.ImageFormat.Unknown ? imageFormat.GetFileExtension() : ".bin";
obj = new Image(id, new byte[0], imageExtension);
break;
default:
throw new NotSupportedException($"Cannot fix missing metadata for ID '{id}' of data type '{dataType}'");
}
var container = new GenericDataContainer("unknown", createdTimeUtc, "unknown", submissionTimeUtc, ApiVersion.Current, obj);
await metadataStorage.StoreAsync(dataType, container, false);
return(container);
}
public async Task <StoreResult> StoreAsync(string dataType, GenericDataContainer container, bool overwrite)
{
var tableSetup = tableSetups[dataType];
var exists = await ExistsAsync(dataType, container.Id);
if (exists && !overwrite)
{
throw new DocumentAlreadyExistsException($"Object of type '{dataType}' with ID '{container.Id}' already exists");
}
if (exists)
{
var query = SqlUpdateStatement.CreateFromDataAndTableSetup(container.Data, tableSetup, container.Id);
await tableSetup.QueryExecutor.ExecuteQueryAsync(query);
await metadataStorage.StoreAsync(dataType, container, true);
return(new StoreResult(container.Id, DataModificationType.Replaced, false));
}
else
{
var query = SqlInsertStatement.CreateFromDataAndTableSetup(container.Data, tableSetup);
var id = await tableSetup.QueryExecutor.ExecuteReaderAsync(query).Select(x => x.GetInt32(0)).FirstOrDefaultAsync();
var idReplacedContainer = new GenericDataContainer(
id.ToString(),
container.OriginalSubmitter,
container.CreatedTimeUtc,
container.Submitter,
container.SubmissionTimeUtc,
container.ApiVersion,
container.Data);
await metadataStorage.StoreAsync(dataType, idReplacedContainer, true);
return(new StoreResult(id.ToString(), DataModificationType.Created, false));
}
}
private static GenericDataContainer CloneGenericDataContainer(GenericDataContainer containerWithoutBinaryData)
{
var json = JsonConvert.SerializeObject(containerWithoutBinaryData);
return(JsonConvert.DeserializeObject <GenericDataContainer>(json));
}
private IEnumerable <string> GetColumnNamesFromContainer(GenericDataContainer container)
{
var keyValuePairs = GenericDataContainerTraversal.Traverse(container);
return(keyValuePairs.Keys);
}