public static void CopyTo(this ObjectParameterCollection parameters, DbCommand command, Func <string, string> fnGetParamName = null, IBatchRunner runner = null)
{
fnGetParamName = fnGetParamName ?? (paramName => paramName);
runner = runner ?? BatchExtensions.ResolveRunner();
var nullValue = runner.DbNull;
foreach (var objectParameter in parameters)
{
var parameter = command.CreateParameter();
parameter.ParameterName = fnGetParamName(objectParameter.Name);
parameter.Value = objectParameter.Value ?? nullValue;
command.Parameters.Add(parameter);
}
}
/// <summary>
/// 根据条件 删除实体
/// </summary>
/// <param name="doWhere"></param>
/// <returns></returns>
public int DelBySqlWhere(Expression <Func <T, bool> > doWhere)
{
try
{
int iret = -1;
using (DbContext _dbx = new Model.Entities())
{
iret = BatchExtensions.Delete(_dbx.Set <T>().Where(doWhere));
}
return(iret);
}
catch (Exception ex)
{
RecordLog.RecordWarn(ex.ToString());
throw ex;
}
}
internal static void Run()
{
if (string.IsNullOrEmpty(Endpoints))
{
throw new ArgumentException("FhirEndpoints value is empty");
}
endpoints = Endpoints.Split(";", StringSplitOptions.RemoveEmptyEntries).ToList();
var globalPrefix = $"RequestedBlobRange=[{NumberOfBlobsToSkip + 1}-{MaxBlobIndexForImport}]";
Console.WriteLine($"{globalPrefix}: Starting...");
var blobContainerClient = GetContainer(ConnectionString, ContainerName);
var blobs = blobContainerClient.GetBlobs().OrderBy(_ => _.Name).Where(_ => _.Name.EndsWith(".ndjson", StringComparison.OrdinalIgnoreCase)).ToList();
Console.WriteLine($"Found ndjson blobs={blobs.Count} in {ContainerName}.");
var take = MaxBlobIndexForImport == 0 ? blobs.Count : MaxBlobIndexForImport - NumberOfBlobsToSkip;
blobs = blobs.Skip(NumberOfBlobsToSkip).Take(take).ToList();
var swWrites = Stopwatch.StartNew();
var swReport = Stopwatch.StartNew();
var currentBlobRanges = new Tuple <int, int> [ReadThreads];
BatchExtensions.ExecuteInParallelBatches(blobs, ReadThreads, BlobRangeSize, (reader, blobRangeInt) =>
{
var localSw = Stopwatch.StartNew();
var writes = 0L;
var blobRangeIndex = blobRangeInt.Item1;
var blobsInt = blobRangeInt.Item2;
var firstBlob = NumberOfBlobsToSkip + (blobRangeIndex * BlobRangeSize) + 1;
var lastBlob = NumberOfBlobsToSkip + (blobRangeIndex * BlobRangeSize) + blobsInt.Count;
currentBlobRanges[reader] = Tuple.Create(firstBlob, lastBlob);
var prefix = $"Reader={reader}.BlobRange=[{firstBlob}-{lastBlob}]";
var incrementor = new IndexIncrementor(endpoints.Count);
Console.WriteLine($"{prefix}: Starting...");
// 100 below is a compromise between processing with maximum available threads (value 1) and inefficiency in wrapping single resource in a list.
BatchExtensions.ExecuteInParallelBatches(GetLinesInBlobRange(blobsInt, prefix), WriteThreads / ReadThreads, 100, (thread, lineBatch) =>
{
Interlocked.Increment(ref writers);
foreach (var line in lineBatch.Item2)
{
Interlocked.Increment(ref totalWrites);
Interlocked.Increment(ref writes);
PutResource(line, incrementor);
if (swReport.Elapsed.TotalSeconds > ReportingPeriodSec)
{
lock (swReport)
{
if (swReport.Elapsed.TotalSeconds > ReportingPeriodSec)
{
var currWrites = Interlocked.Read(ref totalWrites);
var currReads = Interlocked.Read(ref totalReads);
var minBlob = currentBlobRanges.Min(_ => _.Item1);
var maxBlob = currentBlobRanges.Max(_ => _.Item2);
Console.WriteLine($"{globalPrefix}.WorkingBlobRange=[{minBlob}-{maxBlob}].Readers=[{Interlocked.Read(ref readers)}/{ReadThreads}].Writers=[{Interlocked.Read(ref writers)}].EndPointCalls=[{Interlocked.Read(ref epCalls)}].Waits=[{Interlocked.Read(ref waits)}]: reads={currReads} writes={currWrites} secs={(int)swWrites.Elapsed.TotalSeconds} read-speed={(int)(currReads / swReads.Elapsed.TotalSeconds)} lines/sec write-speed={(int)(currWrites / swWrites.Elapsed.TotalSeconds)} res/sec");
swReport.Restart();
}
}
}
}
Interlocked.Decrement(ref writers);
});
Console.WriteLine($"{prefix}: Completed writes. Total={writes} secs={(int)localSw.Elapsed.TotalSeconds} speed={(int)(writes / localSw.Elapsed.TotalSeconds)} res/sec");
});
Console.WriteLine($"{globalPrefix}.Readers=[{readers}/{ReadThreads}].Writers=[{writers}].EndPointCalls=[{epCalls}].Waits=[{waits}]: total reads={totalReads} total writes={totalWrites} secs={(int)swWrites.Elapsed.TotalSeconds} read-speed={(int)(totalReads / swReads.Elapsed.TotalSeconds)} lines/sec write-speed={(int)(totalWrites / swWrites.Elapsed.TotalSeconds)} res/sec");
}
public virtual void BatchUpdate(Expression <Func <TEntity, bool> > predicate, Expression <Func <TEntity, TEntity> > update)
{
BatchExtensions.Update <TEntity>(this.DbSet.Where(predicate), update);
}
public virtual void DeleteAll()
{
BatchExtensions.Delete <TEntity>(this.DbSet);
}
public virtual void BatchDelete(Expression <Func <TEntity, bool> > predicate)
{
BatchExtensions.Delete <TEntity>(this.DbSet.Where(predicate));
}
