internal static IEnumerable <DataAction> FindModified(string tableName, ComparableTable baseTable, ComparableTable comparativeTable)
{
if (!CompareSchema(baseTable, comparativeTable))
{
throw new ArgumentException("Cannot to find modified rows when schema is different.");
}
foreach (var baseRow in baseTable.Rows)
{
if (comparativeTable.Rows.ContainsKey(baseRow.Key) && baseRow.Value != comparativeTable.Rows[baseRow.Key])
{
var compRow = comparativeTable.Rows[baseRow.Key];
int getCellIndex(string header) => Array.FindIndex(comparativeTable.Headers, _ => _ == header);
var columns = baseRow.Value.Cells
.Select((_, i) => _ == compRow.Cells[getCellIndex(baseTable.Headers[i])] ? null : new ColumnData {
Name = baseTable.Headers[i], Data = compRow.Cells[getCellIndex(baseTable.Headers[i])]
})
.Where(_ => _ != null)
.ToArray();
yield return(new UpdateDataAction {
SchemaName = tableName, PrimaryKeyValue = baseRow.Key, Columns = columns
});
}
}
}
public async Task Sync()
{
try
{
var(originDt, targetResp) = await PrepareDataAsync();
if (originDt == null || targetResp == null)
{
return;
}
var ot = ComparableTable.FromDataTable(originDt);
var tt = ComparableTable.FromChainResponse(targetResp);
if (!ComparableTable.CompareSchema(ot, tt))
{
logger.LogWarning($"[{this.option.SyncName}]schema is different, unable to sync. Database side: [{string.Join(",", ot.Headers)}], Chain side: [{string.Join(",", tt.Headers)}]");
return;
}
var actions = new List <DataAction>();
var removed = ComparableTable.FindRemoved(this.option.ChainTableName, tt, ot).ToArray();
var added = ComparableTable.FindAdded(this.option.ChainTableName, tt, ot).ToArray();
var modified = ComparableTable.FindModified(this.option.ChainTableName, tt, ot).ToArray();
actions.AddRange(removed);
actions.AddRange(added);
actions.AddRange(modified);
var actionBatchSize = 10;
var actionBatchList = actions
.Select((val, idx) => new { val, idx })
.GroupBy(_ => _.idx / actionBatchSize)
.Select(g => g.Select(_ => _.val).ToArray())
.ToArray();
var witness = (await GetStatusAsync(this.option.ChainAddress)).Tail.Hash;
foreach (var actionBatch in actionBatchList)
{
await DeployDataAsync(this.option.ChainAddress, this.privateKey, witness, actionBatch);
await Task.Delay(100); // wait for chain to finish create transaction
}
logger.LogInformation($"[{this.option.SyncName}] sync finished, {added.Length} record(s) added, {removed.Length} record(s) removed, {modified.Length} record(s) modified,");
}
catch (Exception ex)
{
logger.LogWarning(ex, $"[{this.option.SyncName}]Exception when doing sync, ignore sync.");
}
}
internal static IEnumerable <DataAction> FindRemoved(string tableName, ComparableTable baseTable, ComparableTable comparativeTable)
{
if (!CompareSchema(baseTable, comparativeTable))
{
throw new ArgumentException("Cannot to find removed rows when schema is different.");
}
foreach (var baseRow in baseTable.Rows)
{
if (!comparativeTable.Rows.ContainsKey(baseRow.Key))
{
yield return(new DeleteDataAction {
SchemaName = tableName, PrimaryKeyValue = baseRow.Key
});
}
}
}
internal static bool CompareSchema(ComparableTable a, ComparableTable b)
{
if (a.PrimaryKeyName != b.PrimaryKeyName)
{
return(false);
}
if (a.Headers.Length != b.Headers.Length)
{
return(false);
}
for (int i = 0; i < a.Headers.Length; i++)
{
if (!b.Headers.Contains(a.Headers[i]))
{
return(false);
}
}
return(true);
}
internal static IEnumerable <DataAction> FindAdded(string tableName, ComparableTable baseTable, ComparableTable comparativeTable)
{
if (!CompareSchema(baseTable, comparativeTable))
{
throw new ArgumentException("Cannot to find added rows when schema is different.");
}
foreach (var compRow in comparativeTable.Rows)
{
if (!baseTable.Rows.ContainsKey(compRow.Key))
{
var columns = compRow.Value.Cells
.Select((_, i) => new ColumnData {
Name = comparativeTable.Headers[i], Data = _
})
.ToArray();
yield return(new InsertDataAction {
SchemaName = tableName, Columns = columns
});
}
}
}
