public Task ExecuteAsync <TData>(IDbConnection connection, IDataDescriptor descriptor, SqlServerPreBatchStatus status, object userState, BatchSaveStrategy saveStrategy, IEnumerable <TData> objects, long startingAt = 0, int?count = null, IDbTransaction transaction = null, int?commandTimeout = null, CancellationToken cancellationToken = default) { throw new NotImplementedException(); /* * // ReSharper disable once PossibleMultipleEnumeration * var reader = new ObjectReader(typeof(TData), objects, * descriptor.Inserted.Select(x => x.ColumnName).ToArray()); * * var mapping = GenerateBulkCopyMapping(descriptor, reader, connection, transaction, commandTimeout); * using (var bcp = new SqlBulkCopy((SqlConnection) connection, SqlBulkCopyOptions.TableLock, * (SqlTransaction) transaction)) * { * foreach (var column in mapping.DatabaseTableColumns) * bcp.ColumnMappings.Add(new SqlBulkCopyColumnMapping(column, column)); * * // ReSharper disable once PossibleMultipleEnumeration * bcp.BatchSize = count ?? objects.Count(); * bcp.DestinationTableName = $"[{descriptor.Schema ?? DefaultSchema}].[{mapping.DataReaderTable.TableName}]"; * bcp.BulkCopyTimeout = commandTimeout.GetValueOrDefault(); * * await bcp.WriteToServerAsync(reader, cancellationToken); * } */ }
public async Task ExecuteAsync <TData>(IDbConnection connection, IDataDescriptor descriptor, SqliteOptions options, object userState, BatchSaveStrategy saveStrategy, IEnumerable <TData> objects, long startingAt = 0, int?count = null, IDbTransaction transaction = null, int?commandTimeout = null, CancellationToken cancellationToken = default) { if (connection is SqliteConnection db) { switch (saveStrategy) { case BatchSaveStrategy.Insert: if (transaction == null) { transaction = db.BeginTransaction(); } foreach (var @object in objects) { var query = SqlBuilder.Insert(@object, descriptor); await db.ExecuteAsync(query.Sql, query.Parameters, transaction); } break; case BatchSaveStrategy.Upsert: case BatchSaveStrategy.Update: throw new NotImplementedException(); default: throw new ArgumentOutOfRangeException(nameof(saveStrategy), saveStrategy, null); } } }
private async Task <IActionResult> SaveAsync([FromBody] IEnumerable <TObject> objects, long startingAt, int?count = null, BatchSaveStrategy strategy = BatchSaveStrategy.Insert) { var errors = new List <Error>(); objects = objects.Where(x => { if (TryValidateModel(x, $"{x.Id}")) { return(true); } errors.Add(this.ConvertModelStateToError(ErrorEvents.ValidationFailed, ErrorStrings.ValidationFailed)); return(false); }); var result = await _repository.SaveAsync(objects, strategy, startingAt, count); foreach (var error in errors) { result.Errors.Add(error); } // See: https://github.com/Microsoft/api-guidelines/blob/master/Guidelines.md#75-standard-request-headers if (Request.Headers.TryGetValue(HttpHeaders.Prefer, out var prefer) && prefer.ToString() .ToUpperInvariant().Replace(" ", string.Empty).Equals("RETURN=MINIMAL")) { Response.Headers.Add(HttpHeaders.PreferenceApplied, "true"); return(Ok()); } Response.MaybeEnvelope(Request, _apiOptions.CurrentValue.JsonConversion, result.Errors, out var body); return(Ok(body)); }
public async Task <Operation> SaveAsync(IEnumerable <TObject> objects, BatchSaveStrategy strategy, long startingAt = 0, int?count = null) { _db.SetTypeInfo(typeof(TObject)); await _db.Current.CopyAsync(_copy, _descriptor, objects, strategy, startingAt, count); return(Operation.CompletedWithoutErrors); }
public static async Task CopyAsync <T, TOptions>(this IDbConnection connection, IDataBatchOperation <TOptions> batch, IEnumerable <T> stream, BatchSaveStrategy saveStrategy = BatchSaveStrategy.Insert, long startingAt = 0, int?count = null, IDbTransaction transaction = null, int?commandTimeout = null, CancellationToken cancellationToken = default) { await connection.CopyAsync(batch, SimpleDataDescriptor.Create <T>(), stream, saveStrategy, startingAt, count, transaction, commandTimeout, cancellationToken); }
public async Task ExecuteAsync <TData>(IDbConnection connection, IDataDescriptor descriptor, DocumentDbBatchOptions options, object userState, BatchSaveStrategy saveStrategy, IEnumerable <TData> objects, long startingAt = 0, int?count = null, IDbTransaction transaction = null, int?commandTimeout = null, CancellationToken cancellationToken = default) { var client = DocumentDbBulkExecutor.RequisitionBulkCopyClient(connection.GetClient()); var collection = (DocumentCollection)userState; var databaseId = connection.GetDatabaseId(); await DocumentDbBulkExecutor.ExecuteAsync(descriptor, _timestamps, saveStrategy, objects, startingAt, count, client, databaseId, collection); }
public async Task AfterAsync(IDbConnection connection, IDataDescriptor descriptor, SqlServerOptions options, object userState, BatchSaveStrategy saveStrategy, IDbTransaction transaction = null, int?commandTimeout = null) { var database = connection.Database; await connection.ExecuteAsync("USE master"); await connection.ExecuteAsync($"ALTER DATABASE [{database}] SET PAGE_VERIFY {options.PageVerify};"); await connection.ExecuteAsync($"ALTER DATABASE [{database}] SET RECOVERY {options.RecoveryModel}"); await connection.ExecuteAsync($"USE [{database}]"); }
public static async Task CopyAsync <T, TOptions>(this IDbConnection connection, IDataBatchOperation <TOptions> batch, IDataDescriptor descriptor, IEnumerable <T> stream, BatchSaveStrategy saveStrategy = BatchSaveStrategy.Insert, long startingAt = 0, int?count = null, IDbTransaction transaction = null, int?commandTimeout = null, CancellationToken cancellationToken = default) { var before = await batch.BeforeAsync(connection, descriptor, transaction); await batch.ExecuteAsync(connection, descriptor, before.Item1, before.Item2, saveStrategy, stream, startingAt, count, transaction, commandTimeout, cancellationToken); await batch.AfterAsync(connection, descriptor, before.Item1, before.Item2, saveStrategy, transaction, commandTimeout); }
public async Task ExecuteAsync <TData>(IDbConnection connection, IDataDescriptor descriptor, DocumentDbBatchOptions options, object userState, BatchSaveStrategy saveStrategy, IEnumerable <TData> objects, long startingAt = 0, int?count = null, IDbTransaction transaction = null, int?commandTimeout = null, CancellationToken cancellationToken = default) { object ToDocument(TData x) { var @object = new ExpandoObject(); var document = (IDictionary <string, object>)@object; document.Add("DocumentType", typeof(TData).Name); foreach (var property in descriptor.Inserted) { document.Add(property.ColumnName, property.Property.Get(x)); } return(@object); } // ReSharper disable once PossibleMultipleEnumeration count = count ?? objects.Count(); var client = connection.GetClient(); var bulkExecutor = (IBulkExecutor)userState; var batchSize = count.GetValueOrDefault(); switch (saveStrategy) { case BatchSaveStrategy.Insert: case BatchSaveStrategy.Upsert: { // ReSharper disable once PossibleMultipleEnumeration var data = objects.Skip((int)startingAt).Take(batchSize); if (descriptor.Id != null && descriptor.Id.IsIdentity) { var nextValues = await client.GetNextValuesForSequenceAsync(typeof(TData), connection.GetDatabaseId(), _options.Value.CollectionId, batchSize); // ReSharper disable once PossibleMultipleEnumeration data = data.Select((x, i) => { descriptor.Id.Property.Set(x, nextValues.Item1 + i); return(x); }); } if (descriptor.Timestamp != null) { data = data.Select(x => { descriptor.Timestamp?.Property?.Set(x, _timestamps.GetCurrentTime()); return(x); }); } var upsert = saveStrategy == BatchSaveStrategy.Upsert; var batch = data.Select(ToDocument); var response = await bulkExecutor.BulkImportAsync( batch, upsert, false, options.MaxConcurrencyPerPartitionKeyRange, options.MaxInMemorySortingBatchSize, cancellationToken); break; } case BatchSaveStrategy.Update: { // ReSharper disable once PossibleMultipleEnumeration var patch = objects.Select(x => { var id = descriptor.Id.ColumnName; var operations = descriptor.Updated.Select(p => { var type = typeof(SetUpdateOperation <>).MakeGenericType(p.Property.Type); var ctor = type.GetConstructor(new[] { typeof(string), p.Property.Type }); if (ctor == null) { return(null); } var operation = (UpdateOperation)ctor.Invoke(new object[] { p.Property.Name, p.Property.Type }); return(operation); }); return(new UpdateItem(id, id, operations.Where(o => o != null))); }); var response = await bulkExecutor.BulkUpdateAsync(patch, options.MaxConcurrencyPerPartitionKeyRange, options.MaxInMemorySortingBatchSize, cancellationToken); break; } default: throw new ArgumentOutOfRangeException(nameof(saveStrategy), saveStrategy, null); } }
public Task AfterAsync(IDbConnection connection, IDataDescriptor descriptor, DocumentDbBatchOptions options, object userState, BatchSaveStrategy saveStrategy, IDbTransaction transaction = null, int?commandTimeout = null) { return(Task.CompletedTask); }
public virtual async Task SaveAsync(IEnumerable <Person> objects, BatchSaveStrategy strategy, long startingAt = 0, int?count = null) { await _repository.SaveAsync(objects, strategy, startingAt, count); }
public async Task SaveAsync(IEnumerable <TObject> objects, BatchSaveStrategy strategy, long startingAt = 0, int?count = null) { await _saves.SaveAsync(objects, strategy, startingAt, count); }
public static async Task ExecuteAsync <TData>(IDataDescriptor descriptor, IServerTimestampService timestamps, BatchSaveStrategy saveStrategy, IEnumerable <TData> objects, long startingAt, int?count, DocumentClient client, string databaseId, Resource collection) { var offer = client.CreateOfferQuery().Where(o => o.ResourceLink == collection.SelfLink) .AsEnumerable().FirstOrDefault(); var throughput = ((OfferV2)offer)?.Content.OfferThroughput ?? 1000; // ReSharper disable once PossibleMultipleEnumeration count ??= objects.Count(); var batchSize = count.GetValueOrDefault(); switch (saveStrategy) { case BatchSaveStrategy.Insert: { // ReSharper disable once PossibleMultipleEnumeration var data = objects.Skip((int)startingAt).Take(batchSize); if (descriptor.Id != null && descriptor.Id.IsIdentity) { var nextValues = await client.GetNextValuesForSequenceAsync(typeof(TData).Name, databaseId, collection.Id, batchSize); // ReSharper disable once PossibleMultipleEnumeration data = data.Select((x, i) => { descriptor.Id.Property.Set(x, nextValues.Item1 + i); return(x); }); } if (descriptor.Timestamp != null) { var timestamp = timestamps.GetCurrentTime(); data = data.Select(x => { descriptor.Timestamp?.Property?.Set(x, timestamp); return(x); }); } var batch = data.Select(x => { var @object = new ExpandoObject(); var document = (IDictionary <string, object>)@object; foreach (var property in descriptor.Inserted) { document.Add(property.ColumnName, property.Property.Get(x)); } if (!document.ContainsKey("DocumentType")) { document.Add("DocumentType", typeof(TData).Name); } document["id"] = Guid.NewGuid().ToString(); return(@object); }); // set TaskCount = 10 for each 10k RUs, minimum 1, maximum 250 var taskCount = Math.Min(Math.Max(throughput / 1000, 1), 250); var tasks = new List <Task>(); var pageSize = batchSize / taskCount; for (var i = 0; i < taskCount; i++) { // ReSharper disable once PossibleMultipleEnumeration var page = batch.Skip(i * pageSize).Take(pageSize); tasks.Add(InsertDocumentAsync(client, databaseId, collection, page)); } await Task.WhenAll(tasks); break; } case BatchSaveStrategy.Upsert: { throw new NotImplementedException(); } case BatchSaveStrategy.Update: { throw new NotImplementedException(); } default: throw new ArgumentOutOfRangeException(nameof(saveStrategy), saveStrategy, null); } }
private async Task <IActionResult> SaveAsync([FromBody] IEnumerable <Person> objects, long startingAt, int?count = null, BatchSaveStrategy strategy = BatchSaveStrategy.Insert) { var errors = new List <Error>(); objects = objects.Where(x => { if (TryValidateModel(x, $"{x.Id}")) { return(true); } errors.Add(ConvertModelStateToError()); return(false); }); var result = await _repository.SaveAsync(objects, strategy, startingAt, count); foreach (var error in errors) { result.Errors.Add(error); } Response.MaybeEnvelope(Request, _apiOptions.Value, _queryOptions.Value, null, result.Errors, out var body); return(Ok(body)); }