protected override async Task DoSomethingAsync(NpgsqlConnection connection, CancellationToken token)
{
NpgsqlTransaction transaction = null;
try
{
var nameFormat = new Regex(@"^\d{4}-\d{2}-\d{2}-\d{2}.sql$");
var contents = _fileProvider
.GetDirectoryContents("sql")
.Where(f => nameFormat.IsMatch(f.Name))
.OrderBy(f => f.Name)
.ToArray();
if (!contents.Any())
{
return;
}
var versions = await connection.GetVersionNamesAsync(token);
transaction = connection.BeginTransaction();
foreach (var content in contents)
{
if (versions.Contains(content.Name))
{
continue;
}
string sql;
using (var sr = new StreamReader(content.CreateReadStream()))
{
sql = sr.ReadToEnd();
}
await connection.UpdateDb(sql, token);
await connection.AddVersion(content.Name, token);
}
transaction.CommitAndDispose();
}
catch (OperationCanceledException)
{
transaction.RollbackAndDispose();
}
catch (Exception)
{
transaction.RollbackAndDispose();
throw;
}
}
private async Task BulkSaveAsync(NpgsqlConnection connection, ScraperCashContainer container, ScraperState scraperState, CancellationToken token)
{
NpgsqlTransaction transaction = null;
try
{
var st = DateTime.Now;
transaction = connection.BeginTransaction();
scraperState.BlockId = MoveToMaxConsistentBlockNum(scraperState.BlockId);
var count = container.Count;
await container.CommitAndDispose(connection, scraperState.BlockId, token);
await connection.UpdateServiceStateAsync(ServiceId, JsonConvert.SerializeObject(scraperState), token);
await _blockMiningService.UpdateNodeInfoAsync(connection, token);
transaction.Commit();
_temporaryLogManager.Add(new BulkSaveTemporaryLog(st, count));
}
catch (Exception e)
{
transaction.RollbackAndDispose();
_temporaryLogManager.Add(new BulkSaveTemporaryLog(e));
Logger.LogCritical(e, "BulkSaveAsync");
throw;
}
}
protected override async Task DoSomethingAsync(NpgsqlConnection connection, CancellationToken token)
{
var settings = new TokenCostServiceSettings();
Configuration.GetSection("TokenCostServiceSettings").Bind(settings);
var eosToUsd = await GetEosUsdAsync(settings, token);
var tokenCost = new List <TokenCost>
{
new TokenCost
{
Contract = 6138663591592764928, //eosio.token
TokenName = "EOS",
EosRate = 1,
UsdRate = eosToUsd
}
};
await GetNewDexContractsCostAsync(settings, eosToUsd, tokenCost, token);
await GetDexEosContractsCostAsync(settings, eosToUsd, tokenCost, token);
connection.Open();
NpgsqlTransaction transaction = null;
try
{
transaction = connection.BeginTransaction();
await connection.UpdateTokenCost(tokenCost, token);
transaction.CommitAndDispose();
}
catch (Exception)
{
transaction.RollbackAndDispose();
throw;
}
}
protected override async Task DoSomethingAsync(NpgsqlConnection connection, CancellationToken token)
{
var isAny = await connection.IsDappComExistAsync(token);
if (isAny)
{
return;
}
connection.Close();
var api = new AppSettings();
Configuration.GetSection("DappComApi")
.Bind(api);
var client = new HttpClient();
var dapps = await GetAppIdAsync(client, api.Url, token);
var set = new List <RootObject>(dapps.Count);
for (var i = 0; i < dapps.Count; i++)
{
await Task.Delay(TimeSpan.FromSeconds(5), token);
Logger.LogInformation($"Progress: {i} | {dapps.Count}");
var dapp = dapps[i];
var info = await GetDappInfoOrDefaultAsync(client, $"{api.Url}app/details/{dapp}", token);
if (info != null)
{
set.Add(info);
}
}
NpgsqlTransaction transaction = null;
try
{
connection.Open();
transaction = connection.BeginTransaction();
foreach (var item in set)
{
await InsertOrUpdateDappInfoAsync(connection, item, token);
}
transaction.CommitAndDispose();
connection.Close();
}
catch (HttpRequestException e)
{
transaction.RollbackAndDispose();
Logger.LogWarning(e, nameof(DappComService));
}
catch (Exception e)
{
transaction.RollbackAndDispose();
Logger.LogError(e, nameof(DappComService));
}
}
public static async Task UpdateTransferInfo(this NpgsqlConnection connection, CancellationToken token)
{
var currentState = await GetTransfersInfoLastState(connection, token);
var cmd = $@"SELECT relname
FROM pg_class
WHERE relname SIMILAR TO 'transfer_[\d]{4}_[\d]{2}_[\d]{2}'
{(string.IsNullOrEmpty(currentState) ? string.Empty : "AND relname > (SELECT json FROM public.service_state WHERE service_id = 2 )")}
ORDER BY relname;";
var command = new NpgsqlCommand
{
Connection = connection,
CommandText = cmd
};
var tables = new List <string>();
using (var reader = await command.ExecuteReaderAsync(token))
{
while (reader.Read())
{
tables.Add(reader.GetStringOrDefault(0));
}
}
if (tables.Count < 2)
{
return;
}
foreach (var table in tables.Take(tables.Count - 1))
{
NpgsqlTransaction transaction = null;
try
{
var dts = table.Substring(table.Length - 10);
var dt = DateTime.ParseExact(dts, "yyyy_MM_dd", CultureInfo.InvariantCulture);
//TODO: KOA (SELECT DISTINCT contract FROM public.dapp_contract) - it`s a crutch dapp_contract.contract must be uniq
var upd =
$@"
DELETE FROM transfers_info WHERE ""timestamp"" = '{dt:yyyy-MM-dd}';
INSERT INTO public.transfers_info(""timestamp"", action_contract, token_name, contract, ""sum"", ""count"", ""from_count"", ""to_count"", ""type"")
SELECT t.timestamp, t.action_contract, t.token_name, t.from AS contract, SUM(t.quantity) AS ""sum"", COUNT(*) AS ""count"", COUNT(DISTINCT t.from) AS ""from_count"", COUNT(DISTINCT t.to) AS ""to_count"", 0 AS ""type""
FROM {table} t
JOIN (SELECT DISTINCT contract FROM public.dapp_contract) dc ON dc.contract = t.from
WHERE t.transaction_status = 0
GROUP BY t.timestamp, t.action_contract, t.token_name, t.from
UNION
SELECT t.timestamp, t.action_contract, t.token_name, t.to AS contract, SUM(t.quantity) AS ""sum"", COUNT(*) AS ""count"", COUNT(DISTINCT t.from) AS ""from_count"", COUNT(DISTINCT t.to) AS ""to_count"", 1 AS ""type""
FROM {table} t
JOIN (SELECT DISTINCT contract FROM public.dapp_contract) dc ON dc.contract = t.to
WHERE t.transaction_status = 0
GROUP BY t.timestamp, t.action_contract, t.token_name, t.to
ON CONFLICT (""timestamp"", action_contract, token_name, contract, ""type"") DO NOTHING;
UPDATE service_state SET json='{table}' WHERE service_id = 2 AND json = '{currentState}';
SELECT json FROM service_state WHERE service_id = 2 ";
transaction = connection.BeginTransaction();
var cmdUpdate = new NpgsqlCommand
{
Connection = connection,
CommandText = upd
};
var newTable = string.Empty;
using (var reader = await cmdUpdate.ExecuteReaderAsync(token))
{
while (reader.Read())
{
newTable = reader.GetStringOrDefault(0);
}
}
if (table.Equals(newTable))
{
transaction.CommitAndDispose();
currentState = newTable;
}
else
{
transaction.RollbackAndDispose();
return;
}
}
catch
{
transaction.RollbackAndDispose();
throw;
}
}
}
public static async Task UpdateDelayedTransferAsync(this NpgsqlConnection connection, CancellationToken token)
{
var limit = 10000;
var delSet = new List <TransferAction>();
var container = new ScraperCashContainer();
var get = $@"SELECT t.block_num, t.transaction_id, t.action_num, t.action_contract, t.action_name,
t.transaction_expiration, t.from, t.to, t.quantity, t.token_name, t.memo_utf8,
dt.transaction_status, dt.block_num AS close_block_num, dt.timestamp
FROM public.transfer_1970_01_01 t
JOIN public.delayed_transaction dt ON dt.transaction_id = t.transaction_id
WHERE t.block_num <= dt.block_num
ORDER BY dt.timestamp
LIMIT {limit};";
do
{
NpgsqlTransaction transaction = null;
try
{
delSet.Clear();
container.Clear();
var command = new NpgsqlCommand
{
Connection = connection,
CommandText = get
};
using (var reader = await command.ExecuteReaderAsync(token))
{
while (reader.Read())
{
var itm = new TransferAction
{
BlockNum = reader.GetLongOrDefault(0),
TransactionId = reader.GetBytesOrDefault(1),
ActionNum = reader.GetIntegerOrDefault(2),
ActionContractId = reader.GetULongOrDefault(3),
ActionName = reader.GetStringOrDefault(4),
TransactionExpiration = reader.GetDateTimeOrDefault(5),
FromId = reader.GetULongOrDefault(6),
ToId = reader.GetULongOrDefault(7),
Quantity = reader.GetDecimalOrDefault(8),
TokenName = reader.GetStringOrDefault(9),
MemoUtf8 = reader.GetBytesOrDefault(10),
TransactionStatus = reader.GetNullableStatusEnum(11),
CloseBlockNum = reader.GetLongOrNull(12),
Timestamp = reader.GetDateTimeOrDefault(13),
};
container.AddToBuffer(itm);
delSet.Add(itm);
}
}
if (!delSet.Any())
{
return;
}
transaction = connection.BeginTransaction();
var dComm = new NpgsqlCommand
{
Connection = connection
};
var sb = new StringBuilder();
for (var i = 0; i < delSet.Count; i++)
{
var itm = delSet[i];
sb.AppendLine($"DELETE FROM public.transfer_1970_01_01 WHERE block_num = @p1_{i} AND transaction_id = @p2_{i} AND action_num = @p3_{i};");
dComm.Parameters.AddValue($"p1_{i}", itm.BlockNum);
dComm.Parameters.AddValue($"p2_{i}", itm.TransactionId);
dComm.Parameters.AddValue($"p3_{i}", itm.ActionNum);
}
dComm.CommandText = sb.ToString();
await dComm.ExecuteNonQueryAsync(token);
container.FlushBuffer();
await container.CommitAndDispose(connection, long.MaxValue, token);
transaction.CommitAndDispose();
}
catch
{
transaction.RollbackAndDispose();
throw;
}
} while (delSet.Count == limit);
}
