internal static IEnumerable <DataAction> FindModified(string tableName, ComparableTable baseTable, ComparableTable comparativeTable) { if (!CompareSchema(baseTable, comparativeTable)) { throw new ArgumentException("Cannot to find modified rows when schema is different."); } foreach (var baseRow in baseTable.Rows) { if (comparativeTable.Rows.ContainsKey(baseRow.Key) && baseRow.Value != comparativeTable.Rows[baseRow.Key]) { var compRow = comparativeTable.Rows[baseRow.Key]; int getCellIndex(string header) => Array.FindIndex(comparativeTable.Headers, _ => _ == header); var columns = baseRow.Value.Cells .Select((_, i) => _ == compRow.Cells[getCellIndex(baseTable.Headers[i])] ? null : new ColumnData { Name = baseTable.Headers[i], Data = compRow.Cells[getCellIndex(baseTable.Headers[i])] }) .Where(_ => _ != null) .ToArray(); yield return(new UpdateDataAction { SchemaName = tableName, PrimaryKeyValue = baseRow.Key, Columns = columns }); } } }
public async Task Sync() { try { var(originDt, targetResp) = await PrepareDataAsync(); if (originDt == null || targetResp == null) { return; } var ot = ComparableTable.FromDataTable(originDt); var tt = ComparableTable.FromChainResponse(targetResp); if (!ComparableTable.CompareSchema(ot, tt)) { logger.LogWarning($"[{this.option.SyncName}]schema is different, unable to sync. Database side: [{string.Join(",", ot.Headers)}], Chain side: [{string.Join(",", tt.Headers)}]"); return; } var actions = new List <DataAction>(); var removed = ComparableTable.FindRemoved(this.option.ChainTableName, tt, ot).ToArray(); var added = ComparableTable.FindAdded(this.option.ChainTableName, tt, ot).ToArray(); var modified = ComparableTable.FindModified(this.option.ChainTableName, tt, ot).ToArray(); actions.AddRange(removed); actions.AddRange(added); actions.AddRange(modified); var actionBatchSize = 10; var actionBatchList = actions .Select((val, idx) => new { val, idx }) .GroupBy(_ => _.idx / actionBatchSize) .Select(g => g.Select(_ => _.val).ToArray()) .ToArray(); var witness = (await GetStatusAsync(this.option.ChainAddress)).Tail.Hash; foreach (var actionBatch in actionBatchList) { await DeployDataAsync(this.option.ChainAddress, this.privateKey, witness, actionBatch); await Task.Delay(100); // wait for chain to finish create transaction } logger.LogInformation($"[{this.option.SyncName}] sync finished, {added.Length} record(s) added, {removed.Length} record(s) removed, {modified.Length} record(s) modified,"); } catch (Exception ex) { logger.LogWarning(ex, $"[{this.option.SyncName}]Exception when doing sync, ignore sync."); } }
internal static IEnumerable <DataAction> FindRemoved(string tableName, ComparableTable baseTable, ComparableTable comparativeTable) { if (!CompareSchema(baseTable, comparativeTable)) { throw new ArgumentException("Cannot to find removed rows when schema is different."); } foreach (var baseRow in baseTable.Rows) { if (!comparativeTable.Rows.ContainsKey(baseRow.Key)) { yield return(new DeleteDataAction { SchemaName = tableName, PrimaryKeyValue = baseRow.Key }); } } }
internal static bool CompareSchema(ComparableTable a, ComparableTable b) { if (a.PrimaryKeyName != b.PrimaryKeyName) { return(false); } if (a.Headers.Length != b.Headers.Length) { return(false); } for (int i = 0; i < a.Headers.Length; i++) { if (!b.Headers.Contains(a.Headers[i])) { return(false); } } return(true); }
internal static IEnumerable <DataAction> FindAdded(string tableName, ComparableTable baseTable, ComparableTable comparativeTable) { if (!CompareSchema(baseTable, comparativeTable)) { throw new ArgumentException("Cannot to find added rows when schema is different."); } foreach (var compRow in comparativeTable.Rows) { if (!baseTable.Rows.ContainsKey(compRow.Key)) { var columns = compRow.Value.Cells .Select((_, i) => new ColumnData { Name = comparativeTable.Headers[i], Data = _ }) .ToArray(); yield return(new InsertDataAction { SchemaName = tableName, Columns = columns }); } } }