public Task ExecuteAsync <TData>(IDbConnection connection, IDataDescriptor descriptor,
SqlServerPreBatchStatus status,
object userState, BatchSaveStrategy saveStrategy, IEnumerable <TData> objects, long startingAt = 0,
int?count = null,
IDbTransaction transaction = null, int?commandTimeout = null,
CancellationToken cancellationToken = default)
{
throw new NotImplementedException();
/*
* // ReSharper disable once PossibleMultipleEnumeration
* var reader = new ObjectReader(typeof(TData), objects,
* descriptor.Inserted.Select(x => x.ColumnName).ToArray());
*
* var mapping = GenerateBulkCopyMapping(descriptor, reader, connection, transaction, commandTimeout);
* using (var bcp = new SqlBulkCopy((SqlConnection) connection, SqlBulkCopyOptions.TableLock,
* (SqlTransaction) transaction))
* {
* foreach (var column in mapping.DatabaseTableColumns)
* bcp.ColumnMappings.Add(new SqlBulkCopyColumnMapping(column, column));
*
* // ReSharper disable once PossibleMultipleEnumeration
* bcp.BatchSize = count ?? objects.Count();
* bcp.DestinationTableName = $"[{descriptor.Schema ?? DefaultSchema}].[{mapping.DataReaderTable.TableName}]";
* bcp.BulkCopyTimeout = commandTimeout.GetValueOrDefault();
*
* await bcp.WriteToServerAsync(reader, cancellationToken);
* }
*/
}
public async Task ExecuteAsync <TData>(IDbConnection connection, IDataDescriptor descriptor,
SqliteOptions options, object userState,
BatchSaveStrategy saveStrategy, IEnumerable <TData> objects, long startingAt = 0, int?count = null,
IDbTransaction transaction = null, int?commandTimeout = null,
CancellationToken cancellationToken = default)
{
if (connection is SqliteConnection db)
{
switch (saveStrategy)
{
case BatchSaveStrategy.Insert:
if (transaction == null)
{
transaction = db.BeginTransaction();
}
foreach (var @object in objects)
{
var query = SqlBuilder.Insert(@object, descriptor);
await db.ExecuteAsync(query.Sql, query.Parameters, transaction);
}
break;
case BatchSaveStrategy.Upsert:
case BatchSaveStrategy.Update:
throw new NotImplementedException();
default:
throw new ArgumentOutOfRangeException(nameof(saveStrategy), saveStrategy, null);
}
}
}
private async Task <IActionResult> SaveAsync([FromBody] IEnumerable <TObject> objects, long startingAt,
int?count = null, BatchSaveStrategy strategy = BatchSaveStrategy.Insert)
{
var errors = new List <Error>();
objects = objects.Where(x =>
{
if (TryValidateModel(x, $"{x.Id}"))
{
return(true);
}
errors.Add(this.ConvertModelStateToError(ErrorEvents.ValidationFailed, ErrorStrings.ValidationFailed));
return(false);
});
var result = await _repository.SaveAsync(objects, strategy, startingAt, count);
foreach (var error in errors)
{
result.Errors.Add(error);
}
// See: https://github.com/Microsoft/api-guidelines/blob/master/Guidelines.md#75-standard-request-headers
if (Request.Headers.TryGetValue(HttpHeaders.Prefer, out var prefer) && prefer.ToString()
.ToUpperInvariant().Replace(" ", string.Empty).Equals("RETURN=MINIMAL"))
{
Response.Headers.Add(HttpHeaders.PreferenceApplied, "true");
return(Ok());
}
Response.MaybeEnvelope(Request, _apiOptions.CurrentValue.JsonConversion, result.Errors, out var body);
return(Ok(body));
}
public async Task <Operation> SaveAsync(IEnumerable <TObject> objects, BatchSaveStrategy strategy,
long startingAt = 0, int?count = null)
{
_db.SetTypeInfo(typeof(TObject));
await _db.Current.CopyAsync(_copy, _descriptor, objects, strategy, startingAt, count);
return(Operation.CompletedWithoutErrors);
}
public static async Task CopyAsync <T, TOptions>(this IDbConnection connection,
IDataBatchOperation <TOptions> batch,
IEnumerable <T> stream, BatchSaveStrategy saveStrategy = BatchSaveStrategy.Insert, long startingAt = 0,
int?count = null, IDbTransaction transaction = null,
int?commandTimeout = null, CancellationToken cancellationToken = default)
{
await connection.CopyAsync(batch, SimpleDataDescriptor.Create <T>(), stream, saveStrategy, startingAt, count,
transaction, commandTimeout, cancellationToken);
}
public async Task ExecuteAsync <TData>(IDbConnection connection, IDataDescriptor descriptor,
DocumentDbBatchOptions options,
object userState, BatchSaveStrategy saveStrategy, IEnumerable <TData> objects, long startingAt = 0,
int?count = null, IDbTransaction transaction = null,
int?commandTimeout = null, CancellationToken cancellationToken = default)
{
var client = DocumentDbBulkExecutor.RequisitionBulkCopyClient(connection.GetClient());
var collection = (DocumentCollection)userState;
var databaseId = connection.GetDatabaseId();
await DocumentDbBulkExecutor.ExecuteAsync(descriptor, _timestamps, saveStrategy, objects, startingAt, count,
client, databaseId, collection);
}
public async Task AfterAsync(IDbConnection connection, IDataDescriptor descriptor, SqlServerOptions options,
object userState,
BatchSaveStrategy saveStrategy, IDbTransaction transaction = null, int?commandTimeout = null)
{
var database = connection.Database;
await connection.ExecuteAsync("USE master");
await connection.ExecuteAsync($"ALTER DATABASE [{database}] SET PAGE_VERIFY {options.PageVerify};");
await connection.ExecuteAsync($"ALTER DATABASE [{database}] SET RECOVERY {options.RecoveryModel}");
await connection.ExecuteAsync($"USE [{database}]");
}
public static async Task CopyAsync <T, TOptions>(this IDbConnection connection,
IDataBatchOperation <TOptions> batch,
IDataDescriptor descriptor, IEnumerable <T> stream,
BatchSaveStrategy saveStrategy = BatchSaveStrategy.Insert, long startingAt = 0, int?count = null,
IDbTransaction transaction = null,
int?commandTimeout = null, CancellationToken cancellationToken = default)
{
var before = await batch.BeforeAsync(connection, descriptor, transaction);
await batch.ExecuteAsync(connection, descriptor, before.Item1, before.Item2, saveStrategy, stream,
startingAt, count, transaction, commandTimeout, cancellationToken);
await batch.AfterAsync(connection, descriptor, before.Item1, before.Item2, saveStrategy, transaction,
commandTimeout);
}
public async Task ExecuteAsync <TData>(IDbConnection connection, IDataDescriptor descriptor,
DocumentDbBatchOptions options,
object userState, BatchSaveStrategy saveStrategy, IEnumerable <TData> objects, long startingAt = 0,
int?count = null, IDbTransaction transaction = null,
int?commandTimeout = null, CancellationToken cancellationToken = default)
{
object ToDocument(TData x)
{
var @object = new ExpandoObject();
var document = (IDictionary <string, object>)@object;
document.Add("DocumentType", typeof(TData).Name);
foreach (var property in descriptor.Inserted)
{
document.Add(property.ColumnName, property.Property.Get(x));
}
return(@object);
}
// ReSharper disable once PossibleMultipleEnumeration
count = count ?? objects.Count();
var client = connection.GetClient();
var bulkExecutor = (IBulkExecutor)userState;
var batchSize = count.GetValueOrDefault();
switch (saveStrategy)
{
case BatchSaveStrategy.Insert:
case BatchSaveStrategy.Upsert:
{
// ReSharper disable once PossibleMultipleEnumeration
var data = objects.Skip((int)startingAt).Take(batchSize);
if (descriptor.Id != null && descriptor.Id.IsIdentity)
{
var nextValues = await client.GetNextValuesForSequenceAsync(typeof(TData),
connection.GetDatabaseId(), _options.Value.CollectionId, batchSize);
// ReSharper disable once PossibleMultipleEnumeration
data = data.Select((x, i) =>
{
descriptor.Id.Property.Set(x, nextValues.Item1 + i);
return(x);
});
}
if (descriptor.Timestamp != null)
{
data = data.Select(x =>
{
descriptor.Timestamp?.Property?.Set(x, _timestamps.GetCurrentTime());
return(x);
});
}
var upsert = saveStrategy == BatchSaveStrategy.Upsert;
var batch = data.Select(ToDocument);
var response = await bulkExecutor.BulkImportAsync(
batch, upsert, false,
options.MaxConcurrencyPerPartitionKeyRange,
options.MaxInMemorySortingBatchSize, cancellationToken);
break;
}
case BatchSaveStrategy.Update:
{
// ReSharper disable once PossibleMultipleEnumeration
var patch = objects.Select(x =>
{
var id = descriptor.Id.ColumnName;
var operations = descriptor.Updated.Select(p =>
{
var type = typeof(SetUpdateOperation <>).MakeGenericType(p.Property.Type);
var ctor = type.GetConstructor(new[] { typeof(string), p.Property.Type });
if (ctor == null)
{
return(null);
}
var operation =
(UpdateOperation)ctor.Invoke(new object[] { p.Property.Name, p.Property.Type });
return(operation);
});
return(new UpdateItem(id, id, operations.Where(o => o != null)));
});
var response = await bulkExecutor.BulkUpdateAsync(patch,
options.MaxConcurrencyPerPartitionKeyRange,
options.MaxInMemorySortingBatchSize, cancellationToken);
break;
}
default:
throw new ArgumentOutOfRangeException(nameof(saveStrategy), saveStrategy, null);
}
}
public Task AfterAsync(IDbConnection connection, IDataDescriptor descriptor, DocumentDbBatchOptions options,
object userState, BatchSaveStrategy saveStrategy, IDbTransaction transaction = null,
int?commandTimeout = null)
{
return(Task.CompletedTask);
}
public virtual async Task SaveAsync(IEnumerable <Person> objects, BatchSaveStrategy strategy, long startingAt = 0, int?count = null)
{
await _repository.SaveAsync(objects, strategy, startingAt, count);
}
public async Task SaveAsync(IEnumerable <TObject> objects, BatchSaveStrategy strategy, long startingAt = 0,
int?count = null)
{
await _saves.SaveAsync(objects, strategy, startingAt, count);
}
public static async Task ExecuteAsync <TData>(IDataDescriptor descriptor, IServerTimestampService timestamps,
BatchSaveStrategy saveStrategy, IEnumerable <TData> objects,
long startingAt, int?count, DocumentClient client, string databaseId, Resource collection)
{
var offer = client.CreateOfferQuery().Where(o => o.ResourceLink == collection.SelfLink)
.AsEnumerable().FirstOrDefault();
var throughput = ((OfferV2)offer)?.Content.OfferThroughput ?? 1000;
// ReSharper disable once PossibleMultipleEnumeration
count ??= objects.Count();
var batchSize = count.GetValueOrDefault();
switch (saveStrategy)
{
case BatchSaveStrategy.Insert:
{
// ReSharper disable once PossibleMultipleEnumeration
var data = objects.Skip((int)startingAt).Take(batchSize);
if (descriptor.Id != null && descriptor.Id.IsIdentity)
{
var nextValues = await client.GetNextValuesForSequenceAsync(typeof(TData).Name,
databaseId, collection.Id, batchSize);
// ReSharper disable once PossibleMultipleEnumeration
data = data.Select((x, i) =>
{
descriptor.Id.Property.Set(x, nextValues.Item1 + i);
return(x);
});
}
if (descriptor.Timestamp != null)
{
var timestamp = timestamps.GetCurrentTime();
data = data.Select(x =>
{
descriptor.Timestamp?.Property?.Set(x, timestamp);
return(x);
});
}
var batch = data.Select(x =>
{
var @object = new ExpandoObject();
var document = (IDictionary <string, object>)@object;
foreach (var property in descriptor.Inserted)
{
document.Add(property.ColumnName, property.Property.Get(x));
}
if (!document.ContainsKey("DocumentType"))
{
document.Add("DocumentType", typeof(TData).Name);
}
document["id"] = Guid.NewGuid().ToString();
return(@object);
});
// set TaskCount = 10 for each 10k RUs, minimum 1, maximum 250
var taskCount = Math.Min(Math.Max(throughput / 1000, 1), 250);
var tasks = new List <Task>();
var pageSize = batchSize / taskCount;
for (var i = 0; i < taskCount; i++)
{
// ReSharper disable once PossibleMultipleEnumeration
var page = batch.Skip(i * pageSize).Take(pageSize);
tasks.Add(InsertDocumentAsync(client, databaseId, collection, page));
}
await Task.WhenAll(tasks);
break;
}
case BatchSaveStrategy.Upsert:
{
throw new NotImplementedException();
}
case BatchSaveStrategy.Update:
{
throw new NotImplementedException();
}
default:
throw new ArgumentOutOfRangeException(nameof(saveStrategy), saveStrategy, null);
}
}
private async Task <IActionResult> SaveAsync([FromBody] IEnumerable <Person> objects, long startingAt, int?count = null, BatchSaveStrategy strategy = BatchSaveStrategy.Insert)
{
var errors = new List <Error>();
objects = objects.Where(x =>
{
if (TryValidateModel(x, $"{x.Id}"))
{
return(true);
}
errors.Add(ConvertModelStateToError());
return(false);
});
var result = await _repository.SaveAsync(objects, strategy, startingAt, count);
foreach (var error in errors)
{
result.Errors.Add(error);
}
Response.MaybeEnvelope(Request, _apiOptions.Value, _queryOptions.Value, null, result.Errors, out var body);
return(Ok(body));
}
