private async Task BatchProcessing(List <DataAsyncWrapper <SaveTransport <PrimaryKey>, bool> > wrapperList) { var copySql = copySaveSqlDict.GetOrAdd((await tableInfo.GetTable(DateTimeOffset.UtcNow.ToUnixTimeMilliseconds())).Name, key => $"copy {key}(stateid,uniqueId,typecode,data,version,timestamp) FROM STDIN (FORMAT BINARY)"); try { using (var conn = tableInfo.CreateConnection() as NpgsqlConnection) { await conn.OpenAsync(); using (var writer = conn.BeginBinaryImport(copySql)) { foreach (var wrapper in wrapperList) { writer.StartRow(); writer.Write(wrapper.Value.Event.StateId.ToString(), NpgsqlDbType.Varchar); writer.Write(wrapper.Value.UniqueId, NpgsqlDbType.Varchar); writer.Write(wrapper.Value.Event.GetType().FullName, NpgsqlDbType.Varchar); writer.Write(Encoding.Default.GetString(wrapper.Value.BytesTransport.EventBytes), NpgsqlDbType.Jsonb); writer.Write(wrapper.Value.Event.Base.Version, NpgsqlDbType.Bigint); writer.Write(wrapper.Value.Event.Base.Timestamp, NpgsqlDbType.Bigint); } writer.Complete(); } } wrapperList.ForEach(wrap => wrap.TaskSource.TrySetResult(true)); } catch { var saveSql = await GetInsertSql(); using (var conn = tableInfo.CreateConnection()) { await conn.OpenAsync(); using (var trans = conn.BeginTransaction()) { try { foreach (var wrapper in wrapperList) { wrapper.Value.ReturnValue = await conn.ExecuteAsync(saveSql, new { StateId = wrapper.Value.Event.StateId.ToString(), wrapper.Value.UniqueId, TypeCode = wrapper.Value.Event.Event.GetType().FullName, Data = Encoding.Default.GetString(wrapper.Value.BytesTransport.EventBytes), wrapper.Value.Event.Base.Version, wrapper.Value.Event.Base.Timestamp }, trans) > 0; } trans.Commit(); wrapperList.ForEach(wrap => wrap.TaskSource.TrySetResult(wrap.Value.ReturnValue)); } catch (Exception e) { trans.Rollback(); wrapperList.ForEach(wrap => wrap.TaskSource.TrySetException(e)); } } } } }
private async Task BatchProcessing(List <DataAsyncWrapper <BatchAppendTransport <PrimaryKey>, bool> > wrapperList) { var minTimestamp = wrapperList.Min(t => t.Value.Event.Base.Timestamp); var maxTimestamp = wrapperList.Max(t => t.Value.Event.Base.Timestamp); var minTask = config.GetTable(minTimestamp); if (!minTask.IsCompletedSuccessfully) { await minTask; } if (minTask.Result.EndTime > maxTimestamp) { await BatchCopy(minTask.Result.SubTable, wrapperList); } else { var groups = (await Task.WhenAll(wrapperList.Select(async t => { var task = config.GetTable(t.Value.Event.Base.Timestamp); if (!task.IsCompletedSuccessfully) { await task; } return(task.Result.SubTable, t); }))).GroupBy(t => t.SubTable); foreach (var group in groups) { await BatchCopy(group.Key, group.Select(t => t.t).ToList()); } } async Task BatchCopy(string tableName, List <DataAsyncWrapper <BatchAppendTransport <PrimaryKey>, bool> > list) { try { var copySql = copySaveSqlDict.GetOrAdd(tableName, key => $"copy {key}(stateid,uniqueId,typecode,data,version,timestamp) FROM STDIN (FORMAT BINARY)"); using (var conn = config.CreateConnection() as NpgsqlConnection) { await conn.OpenAsync(); using (var writer = conn.BeginBinaryImport(copySql)) { foreach (var wrapper in list) { writer.StartRow(); writer.Write(wrapper.Value.Event.StateId.ToString(), NpgsqlDbType.Varchar); writer.Write(wrapper.Value.UniqueId, NpgsqlDbType.Varchar); writer.Write(wrapper.Value.Event.GetType().FullName, NpgsqlDbType.Varchar); writer.Write(Encoding.Default.GetString(wrapper.Value.BytesTransport.EventBytes), NpgsqlDbType.Jsonb); writer.Write(wrapper.Value.Event.Base.Version, NpgsqlDbType.Bigint); writer.Write(wrapper.Value.Event.Base.Timestamp, NpgsqlDbType.Bigint); } writer.Complete(); } } list.ForEach(wrap => wrap.TaskSource.TrySetResult(true)); } catch (Exception ex) { logger.LogError(ex, ex.Message); var saveSql = saveSqlDict.GetOrAdd(tableName, key => $"INSERT INTO {key}(stateid,uniqueId,typecode,data,version,timestamp) VALUES(@StateId,@UniqueId,@TypeCode,(@Data)::jsonb,@Version,@Timestamp) ON CONFLICT ON CONSTRAINT {key}_id_unique DO NOTHING"); await BatchInsert(saveSql, wrapperList); } } async Task BatchInsert(string saveSql, List <DataAsyncWrapper <BatchAppendTransport <PrimaryKey>, bool> > list) { bool isSuccess = false; using (var conn = config.CreateConnection()) { await conn.OpenAsync(); using (var trans = conn.BeginTransaction()) { try { foreach (var wrapper in list) { wrapper.Value.ReturnValue = await conn.ExecuteAsync(saveSql, new { StateId = wrapper.Value.Event.StateId.ToString(), wrapper.Value.UniqueId, TypeCode = wrapper.Value.Event.Event.GetType().FullName, Data = Encoding.Default.GetString(wrapper.Value.BytesTransport.EventBytes), wrapper.Value.Event.Base.Version, wrapper.Value.Event.Base.Timestamp }, trans) > 0; } trans.Commit(); isSuccess = true; list.ForEach(wrap => wrap.TaskSource.TrySetResult(wrap.Value.ReturnValue)); } catch { trans.Rollback(); } } if (!isSuccess) { foreach (var wrapper in list) { try { wrapper.TaskSource.TrySetResult(await conn.ExecuteAsync(saveSql, new { StateId = wrapper.Value.Event.StateId.ToString(), wrapper.Value.UniqueId, TypeCode = wrapper.Value.Event.Event.GetType().FullName, Data = Encoding.Default.GetString(wrapper.Value.BytesTransport.EventBytes), wrapper.Value.Event.Base.Version, wrapper.Value.Event.Base.Timestamp }) > 0); } catch (Exception ex) { wrapper.TaskSource.TrySetException(ex); } } } } } }