private async Task BatchProcessing(List <DataAsyncWrapper <SaveTransport <PrimaryKey>, bool> > wrapperList)
{
var copySql = copySaveSqlDict.GetOrAdd((await tableInfo.GetTable(DateTimeOffset.UtcNow.ToUnixTimeMilliseconds())).Name,
key => $"copy {key}(stateid,uniqueId,typecode,data,version,timestamp) FROM STDIN (FORMAT BINARY)");
try
{
using (var conn = tableInfo.CreateConnection() as NpgsqlConnection)
{
await conn.OpenAsync();
using (var writer = conn.BeginBinaryImport(copySql))
{
foreach (var wrapper in wrapperList)
{
writer.StartRow();
writer.Write(wrapper.Value.Event.StateId.ToString(), NpgsqlDbType.Varchar);
writer.Write(wrapper.Value.UniqueId, NpgsqlDbType.Varchar);
writer.Write(wrapper.Value.Event.GetType().FullName, NpgsqlDbType.Varchar);
writer.Write(Encoding.Default.GetString(wrapper.Value.BytesTransport.EventBytes), NpgsqlDbType.Jsonb);
writer.Write(wrapper.Value.Event.Base.Version, NpgsqlDbType.Bigint);
writer.Write(wrapper.Value.Event.Base.Timestamp, NpgsqlDbType.Bigint);
}
writer.Complete();
}
}
wrapperList.ForEach(wrap => wrap.TaskSource.TrySetResult(true));
}
catch
{
var saveSql = await GetInsertSql();
using (var conn = tableInfo.CreateConnection())
{
await conn.OpenAsync();
using (var trans = conn.BeginTransaction())
{
try
{
foreach (var wrapper in wrapperList)
{
wrapper.Value.ReturnValue = await conn.ExecuteAsync(saveSql, new
{
StateId = wrapper.Value.Event.StateId.ToString(),
wrapper.Value.UniqueId,
TypeCode = wrapper.Value.Event.Event.GetType().FullName,
Data = Encoding.Default.GetString(wrapper.Value.BytesTransport.EventBytes),
wrapper.Value.Event.Base.Version,
wrapper.Value.Event.Base.Timestamp
}, trans) > 0;
}
trans.Commit();
wrapperList.ForEach(wrap => wrap.TaskSource.TrySetResult(wrap.Value.ReturnValue));
}
catch (Exception e)
{
trans.Rollback();
wrapperList.ForEach(wrap => wrap.TaskSource.TrySetException(e));
}
}
}
}
}
private async Task BatchProcessing(List <DataAsyncWrapper <BatchAppendTransport <PrimaryKey>, bool> > wrapperList)
{
var minTimestamp = wrapperList.Min(t => t.Value.Event.Base.Timestamp);
var maxTimestamp = wrapperList.Max(t => t.Value.Event.Base.Timestamp);
var minTask = config.GetTable(minTimestamp);
if (!minTask.IsCompletedSuccessfully)
{
await minTask;
}
if (minTask.Result.EndTime > maxTimestamp)
{
await BatchCopy(minTask.Result.SubTable, wrapperList);
}
else
{
var groups = (await Task.WhenAll(wrapperList.Select(async t =>
{
var task = config.GetTable(t.Value.Event.Base.Timestamp);
if (!task.IsCompletedSuccessfully)
{
await task;
}
return(task.Result.SubTable, t);
}))).GroupBy(t => t.SubTable);
foreach (var group in groups)
{
await BatchCopy(group.Key, group.Select(t => t.t).ToList());
}
}
async Task BatchCopy(string tableName, List <DataAsyncWrapper <BatchAppendTransport <PrimaryKey>, bool> > list)
{
try
{
var copySql = copySaveSqlDict.GetOrAdd(tableName,
key => $"copy {key}(stateid,uniqueId,typecode,data,version,timestamp) FROM STDIN (FORMAT BINARY)");
using (var conn = config.CreateConnection() as NpgsqlConnection)
{
await conn.OpenAsync();
using (var writer = conn.BeginBinaryImport(copySql))
{
foreach (var wrapper in list)
{
writer.StartRow();
writer.Write(wrapper.Value.Event.StateId.ToString(), NpgsqlDbType.Varchar);
writer.Write(wrapper.Value.UniqueId, NpgsqlDbType.Varchar);
writer.Write(wrapper.Value.Event.GetType().FullName, NpgsqlDbType.Varchar);
writer.Write(Encoding.Default.GetString(wrapper.Value.BytesTransport.EventBytes), NpgsqlDbType.Jsonb);
writer.Write(wrapper.Value.Event.Base.Version, NpgsqlDbType.Bigint);
writer.Write(wrapper.Value.Event.Base.Timestamp, NpgsqlDbType.Bigint);
}
writer.Complete();
}
}
list.ForEach(wrap => wrap.TaskSource.TrySetResult(true));
}
catch (Exception ex)
{
logger.LogError(ex, ex.Message);
var saveSql = saveSqlDict.GetOrAdd(tableName,
key => $"INSERT INTO {key}(stateid,uniqueId,typecode,data,version,timestamp) VALUES(@StateId,@UniqueId,@TypeCode,(@Data)::jsonb,@Version,@Timestamp) ON CONFLICT ON CONSTRAINT {key}_id_unique DO NOTHING");
await BatchInsert(saveSql, wrapperList);
}
}
async Task BatchInsert(string saveSql, List <DataAsyncWrapper <BatchAppendTransport <PrimaryKey>, bool> > list)
{
bool isSuccess = false;
using (var conn = config.CreateConnection())
{
await conn.OpenAsync();
using (var trans = conn.BeginTransaction())
{
try
{
foreach (var wrapper in list)
{
wrapper.Value.ReturnValue = await conn.ExecuteAsync(saveSql, new
{
StateId = wrapper.Value.Event.StateId.ToString(),
wrapper.Value.UniqueId,
TypeCode = wrapper.Value.Event.Event.GetType().FullName,
Data = Encoding.Default.GetString(wrapper.Value.BytesTransport.EventBytes),
wrapper.Value.Event.Base.Version,
wrapper.Value.Event.Base.Timestamp
}, trans) > 0;
}
trans.Commit();
isSuccess = true;
list.ForEach(wrap => wrap.TaskSource.TrySetResult(wrap.Value.ReturnValue));
}
catch
{
trans.Rollback();
}
}
if (!isSuccess)
{
foreach (var wrapper in list)
{
try
{
wrapper.TaskSource.TrySetResult(await conn.ExecuteAsync(saveSql, new
{
StateId = wrapper.Value.Event.StateId.ToString(),
wrapper.Value.UniqueId,
TypeCode = wrapper.Value.Event.Event.GetType().FullName,
Data = Encoding.Default.GetString(wrapper.Value.BytesTransport.EventBytes),
wrapper.Value.Event.Base.Version,
wrapper.Value.Event.Base.Timestamp
}) > 0);
}
catch (Exception ex)
{
wrapper.TaskSource.TrySetException(ex);
}
}
}
}
}
}
