public BinaryDataObjectSplitterResult(
     GenericDataContainer containerWithoutBinaryData,
     byte[] binaryData)
 {
     ContainerWithoutBinaryData = containerWithoutBinaryData ?? throw new ArgumentNullException(nameof(containerWithoutBinaryData));
     BinaryData = binaryData ?? throw new ArgumentNullException(nameof(binaryData));
 }
        public GenericDataContainer Reassemble(GenericDataContainer containerWithoutBinaryData, byte[] binaryData)
        {
            var containerWithData = CloneGenericDataContainer(containerWithoutBinaryData);

            containerWithData.Data.Set(binaryDataPath, BsonString.Create(Convert.ToBase64String(binaryData)));
            return(containerWithData);
        }
예제 #3
0
        public async Task <StoreResult> StoreAsync(string dataType, GenericDataContainer container, bool overwrite)
        {
            var id     = container.Id;
            var exists = await ExistsAsync(dataType, id);

            if (exists && !overwrite)
            {
                throw new DocumentAlreadyExistsException($"Object of type '{dataType}' with ID '{id}' already exists");
            }
            var isNewCollection = await IsNewCollection(dataType);

            var binaryDataObjectSplitter = binaryDataObjectSplitters[dataType];
            var splitResult = binaryDataObjectSplitter.Split(container);
            await metadataStorage.StoreAsync(dataType, splitResult.ContainerWithoutBinaryData, true);

            var blobContainer = binaryDataStorage.GetContainer(dataType);
            await blobContainer.CreateIfNotExistsAsync();

            var blob = blobContainer.GetBlob(id);
            await blob.WriteAsync(splitResult.BinaryData);

            var dataModificationType = exists
                ? DataModificationType.Replaced
                : DataModificationType.Created;

            return(new StoreResult(id, dataModificationType, isNewCollection));
        }
예제 #4
0
        public static string CreateFromContainer(string dataType, GenericDataContainer container)
        {
            var keyValuePairs = GenericDataContainerTraversal.Traverse(container);
            var query         = $"UPDATE {dataType} SET ";
            var firstItem     = true;

            foreach (var(key, value) in keyValuePairs)
            {
                if (key.InSet("Id", "CreatedTimeUtc", "OriginalSubmitter"))
                {
                    continue;
                }
                if (!firstItem)
                {
                    query += ", ";
                }
                else
                {
                    firstItem = false;
                }
                if (key == "Id")
                {
                    query += $"{key}={value}";
                }
                else
                {
                    query += $"{key}='{value}'";
                }
            }
            query += $" WHERE Id='{container.Id}'";
            return(query);
        }
        private async Task <GenericDataContainer> BuildContainer(SqlDataReader reader, SqlTableSetup tableSetup)
        {
            var id       = (int)reader[tableSetup.IdColumnName];
            var metadata = await metadataStorage.GetFromIdAsync(tableSetup.DataType, id.ToString());

            if (metadata == null)
            {
                var utcNow = DateTime.UtcNow;
                metadata = new GenericDataContainer(
                    id.ToString(),
                    "unknown",
                    utcNow,
                    "unknown",
                    utcNow,
                    ApiVersion.Current,
                    new BsonDocument());
            }

            var data = BsonDocumentBuilder.BuildFromDelimitedColumns(reader, tableSetup);

            return(new GenericDataContainer(
                       id.ToString(),
                       metadata.OriginalSubmitter,
                       metadata.CreatedTimeUtc,
                       metadata.Submitter,
                       metadata.SubmissionTimeUtc,
                       metadata.ApiVersion,
                       data));
        }
예제 #6
0
        public async Task <StoreResult> StoreAsync(string dataType, GenericDataContainer container, bool overwrite)
        {
            var isNewCollection = await IsNewCollectionAsync(dataType);

            if (isNewCollection)
            {
                await CreateTable(dataType);
            }
            var exists = await ExistsAsync(dataType, container.Id);

            if (exists && !overwrite)
            {
                throw new DocumentAlreadyExistsException($"Object of type '{dataType}' with ID '{container.Id}' already exists");
            }
            await EnsureColumnsExistAsync(dataType, container);

            var query = exists
                ? SqlUpdateStatement.CreateFromContainer(dataType, container)
                : SqlInsertStatement.CreateFromContainer(dataType, container);
            await sqlQueryExecutor.ExecuteQueryAsync(query);

            var modificationType = !exists
                ? DataModificationType.Created
                : DataModificationType.Replaced;

            return(new StoreResult(container.Id, modificationType, isNewCollection));
        }
예제 #7
0
        public async Task <StoreResult> StoreAsync(string dataType, GenericDataContainer container, bool overwrite)
        {
            var isNewCollection = await IsNewCollectionAsync(dataType);

            var collection = rdDataClient.DataDatabase.GetCollection <GenericDataContainer>(dataType);
            var id         = container.Id;

            if (overwrite)
            {
                var replaceResult = await collection.ReplaceOneAsync(x => x.Id == id, container, new ReplaceOptions { IsUpsert = true });

                var dataModificationType = replaceResult.MatchedCount == 0 ? DataModificationType.Created : DataModificationType.Replaced;
                return(new StoreResult(id, dataModificationType, isNewCollection));
            }

            try
            {
                await collection.InsertOneAsync(container);

                return(new StoreResult(id, DataModificationType.Created, isNewCollection));
            }
            catch (MongoWriteException writeException)
            {
                if (writeException.WriteError.Category == ServerErrorCategory.DuplicateKey)
                {
                    throw new DocumentAlreadyExistsException($"Object of type '{dataType}' with ID '{id}' already exists");
                }
                throw;
            }
        }
예제 #8
0
        public static string CreateFromContainer(string dataType, GenericDataContainer container)
        {
            var keyValuePairs = GenericDataContainerTraversal.Traverse(container);
            var query         = $"INSERT INTO {dataType} ({string.Join(", ", keyValuePairs.Keys)}) "
                                + $"VALUES ({string.Join(", ", keyValuePairs.Select(kvp => $"'{kvp.Value}'"))})";

            return(query);
        }
        private BsonDocument MapContainerUsingSelect(GenericDataContainer container, DataApiSqlQuery parsedQuery)
        {
            var containerJObject = JObject.FromObject(container);
            var dataJObject      = JObject.Parse(DataEncoder.DecodeToJson(container.Data));
            var fieldMappings    = parsedQuery.SelectArguments.Split(',').Select(x => x.Trim());
            var bsonDocument     = new BsonDocument();

            foreach (var fieldMapping in fieldMappings)
            {
                var match = Regex.Match(fieldMapping, "^(?<Path>[^\\s]+)(?<Map>\\s+AS\\s+(?<NewName>[^\\s]+))?", RegexOptions.IgnoreCase);
                if (!match.Success)
                {
                    throw new FormatException($"Invalid SELECT statement '{fieldMapping}'");
                }
                var    fieldPath = match.Groups["Path"].Value;
                string newName;
                var    hasMap = match.Groups["Map"].Success;
                if (hasMap)
                {
                    newName = match.Groups["NewName"].Value;
                }
                else
                {
                    newName = fieldPath.Replace('.', '_');
                }
                JToken jToken;
                if (fieldPath.StartsWith("Data."))
                {
                    var token = hasMap ? newName : fieldPath.Substring("Data.".Length);
                    jToken = dataJObject.SelectToken(token);
                }
                else if (fieldPath == "Data")
                {
                    jToken = dataJObject;
                }
                else
                {
                    jToken = containerJObject.SelectToken(fieldPath);
                }
                BsonValue bsonValue;
                if (jToken == null)
                {
                    bsonValue = BsonNull.Value;
                }
                else if (jToken is JValue jValue)
                {
                    bsonValue = BsonValue.Create(jValue.Value);
                }
                else
                {
                    bsonValue = BsonDocument.Parse(jToken.ToString());
                }
                bsonDocument.Add(new BsonElement(newName, bsonValue));
            }
            return(bsonDocument);
        }
예제 #10
0
        public static Dictionary <string, string> Traverse(GenericDataContainer container)
        {
            var jObject       = JObject.Parse(DataEncoder.DecodeToJson(container.Data));
            var keyValuePairs = new Dictionary <string, string>
            {
                { "Id", container.Id },
                { "OriginalSubmitter", container.OriginalSubmitter },
                { "CreatedTimeUtc", container.CreatedTimeUtc?.ToString("yyyy-MM-dd HH:mm:ss") },
                { "Submitter", container.Submitter },
                { "SubmissionTimeUtc", container.SubmissionTimeUtc.ToString("yyyy-MM-dd HH:mm:ss") }
            };

            TraverseJObject(jObject, keyValuePairs, "Data");
            return(keyValuePairs);
        }
예제 #11
0
        private async Task EnsureColumnsExistAsync(string dataType, GenericDataContainer container)
        {
            var columnNames     = GetColumnNamesFromContainer(container);
            var existingColumns = await ListColumnsAsync(dataType);

            var missingColumns = columnNames.Except(existingColumns).ToList();

            if (!missingColumns.Any())
            {
                return;
            }
            var columnDefinitions = string.Join(',', missingColumns.Select(columnName => $"{columnName} nvarchar(100) NULL"));
            var query             = $"ALTER TABLE {dataType} ADD {columnDefinitions}";
            await sqlQueryExecutor.ExecuteQueryAsync(query);
        }
        public BinaryDataObjectSplitterResult Split(GenericDataContainer container)
        {
            var payload    = container.Data.DeepClone().AsBsonDocument;
            var bsonValue  = payload.GetValue(binaryDataPath);
            var base64     = bsonValue.IsString ? bsonValue.AsString : null;
            var binaryData = base64 != null?Convert.FromBase64String(base64) : new byte[0];

            payload.Set(binaryDataPath, BsonString.Empty);
            var containerWithoutData = new GenericDataContainer(
                container.Id,
                container.OriginalSubmitter,
                container.CreatedTimeUtc,
                container.Submitter,
                container.SubmissionTimeUtc,
                container.ApiVersion,
                payload);

            return(new BinaryDataObjectSplitterResult(containerWithoutData, binaryData));
        }
예제 #13
0
        private async Task <GenericDataContainer> FixMissingMetadata(string dataType, string id, byte[] binaryData)
        {
            // Assumption: Metadata doesn't exist

            var      blob = binaryDataStorage.GetBlob(dataType, id);
            DateTime createdTimeUtc;
            DateTime submissionTimeUtc;

            if (await blob.ExistsAsync())
            {
                createdTimeUtc    = blob.CreatedTimestampUtc;
                submissionTimeUtc = blob.LastModifiedTimestampUtc;
            }
            else
            {
                var utcNow = DateTime.UtcNow;
                createdTimeUtc    = utcNow;
                submissionTimeUtc = utcNow;
            }
            IId obj;

            switch (dataType)
            {
            case nameof(DataBlob):
                obj = new DataBlob(id, new byte[0]);
                break;

            case nameof(Image):
                var imageFormat    = ImageFormatDetector.Detect(binaryData);
                var imageExtension = imageFormat != ImageFormatDetector.ImageFormat.Unknown ? imageFormat.GetFileExtension() : ".bin";
                obj = new Image(id, new byte[0], imageExtension);
                break;

            default:
                throw new NotSupportedException($"Cannot fix missing metadata for ID '{id}' of data type '{dataType}'");
            }
            var container = new GenericDataContainer("unknown", createdTimeUtc, "unknown", submissionTimeUtc, ApiVersion.Current, obj);
            await metadataStorage.StoreAsync(dataType, container, false);

            return(container);
        }
        public async Task <StoreResult> StoreAsync(string dataType, GenericDataContainer container, bool overwrite)
        {
            var tableSetup = tableSetups[dataType];
            var exists     = await ExistsAsync(dataType, container.Id);

            if (exists && !overwrite)
            {
                throw new DocumentAlreadyExistsException($"Object of type '{dataType}' with ID '{container.Id}' already exists");
            }
            if (exists)
            {
                var query = SqlUpdateStatement.CreateFromDataAndTableSetup(container.Data, tableSetup, container.Id);
                await tableSetup.QueryExecutor.ExecuteQueryAsync(query);

                await metadataStorage.StoreAsync(dataType, container, true);

                return(new StoreResult(container.Id, DataModificationType.Replaced, false));
            }
            else
            {
                var query = SqlInsertStatement.CreateFromDataAndTableSetup(container.Data, tableSetup);
                var id    = await tableSetup.QueryExecutor.ExecuteReaderAsync(query).Select(x => x.GetInt32(0)).FirstOrDefaultAsync();

                var idReplacedContainer = new GenericDataContainer(
                    id.ToString(),
                    container.OriginalSubmitter,
                    container.CreatedTimeUtc,
                    container.Submitter,
                    container.SubmissionTimeUtc,
                    container.ApiVersion,
                    container.Data);
                await metadataStorage.StoreAsync(dataType, idReplacedContainer, true);

                return(new StoreResult(id.ToString(), DataModificationType.Created, false));
            }
        }
        private static GenericDataContainer CloneGenericDataContainer(GenericDataContainer containerWithoutBinaryData)
        {
            var json = JsonConvert.SerializeObject(containerWithoutBinaryData);

            return(JsonConvert.DeserializeObject <GenericDataContainer>(json));
        }
예제 #16
0
        private IEnumerable <string> GetColumnNamesFromContainer(GenericDataContainer container)
        {
            var keyValuePairs = GenericDataContainerTraversal.Traverse(container);

            return(keyValuePairs.Keys);
        }