public static string BuildConnectionString(this XRoadLogsDb sourceDb)
{
var connectionStringBuilder = new NpgsqlConnectionStringBuilder
{
Host = sourceDb.Host,
Port = sourceDb.Port,
Username = sourceDb.Username,
Password = sourceDb.Password,
Database = sourceDb.Database,
};
return(connectionStringBuilder.ConnectionString);
}
public bool AnyRecords(XRoadLogsDb sourceDb, long fromIdInclusive)
{
var connectionString = sourceDb.BuildConnectionString();
using var connection = new NpgsqlConnection(connectionString);
var command = new NpgsqlCommand("select count(*) from logrecord where id >= @from_id_inclusive", connection);
command.Parameters.AddWithValue("from_id_inclusive", NpgsqlDbType.Bigint, fromIdInclusive);
connection.Open();
var result = (long)command.ExecuteScalar();
connection.Close();
return(result > 0);
}
public async Task Collect(XRoadLogsDb targetDb)
{
var lastSucceededCollectorOperation = await _journalService.GetLastSucceededOperationAsync(targetDb.Host);
var isFirstConnection = lastSucceededCollectorOperation == null;
var fromIdInclusive = isFirstConnection ? 0L : lastSucceededCollectorOperation.LastRecordId + 1;
for (var iteration = 0; iteration < _options.CollectorIterationsCount; iteration++)
{
var journal = new JournalRecordBuilder()
.SetXRoadDb(targetDb)
.SetFromIdInclusive(fromIdInclusive)
.SetLimit(_options.Limit);
try
{
var anyRecords = _xRoadDbReader.AnyRecords(targetDb, fromIdInclusive);
if (!anyRecords)
{
//just ignore
break;
}
var collectResult = Collect(targetDb, fromIdInclusive, _options.Limit);
fromIdInclusive = collectResult.LastRecordId + 1;
journal.SetLastRecordId(collectResult.LastRecordId);
journal.SetParsedCount(collectResult.ParsedCount);
await _journalService.AddAsync(journal.Build());
}
catch (Exception exception)
{
journal.SetException(exception);
_logger.LogError(exception, "Error occurred during LogsCollectorService execution");
await _journalService.AddAsync(journal.Build());
break;
}
}
}
private CollectResult Collect(XRoadLogsDb targetDb, long fromIdInclusive, int limit)
{
var collectResult = new CollectResult();
var logsReadResult = _xRoadDbReader.Read(targetDb, fromIdInclusive, limit);
if (!logsReadResult.Any()) //assert logs present
{
throw new ApplicationException();
}
var xRoadRequests = _logsProcessor.Process(logsReadResult);
_requestStore.AddRangeAsync(xRoadRequests).WaitAndUnwrapException();
collectResult.Count = logsReadResult.Count;
collectResult.ParsedCount = xRoadRequests.Count;
collectResult.LastRecordId = logsReadResult.LastRecordId;
return(collectResult);
}
public XRoadLogsReadResult Read(XRoadLogsDb sourceDb, long fromIdInclusive, int limit)
{
_logger.LogInformation("Requesting x-road logs from db: {0}, ids: [{1},{2}]", sourceDb.Host,
fromIdInclusive, limit);
var result = new XRoadLogsReadResult();
var connectionString = sourceDb.BuildConnectionString();
using var connection = new NpgsqlConnection(connectionString);
var command = new NpgsqlCommand(
"select id,queryid,memberclass,membercode,subsystemcode,message,time,attachment,xrequestid,response,discriminator from logrecord "
+ "where id >= @from_id_inclusive "
+ "order by id "
+ "limit @limit", connection
);
command.Parameters.AddWithValue("from_id_inclusive", NpgsqlDbType.Bigint, fromIdInclusive);
command.Parameters.AddWithValue("limit", NpgsqlDbType.Integer, limit);
connection.Open();
using var reader = command.ExecuteReader();
if (!reader.HasRows)
{
return(result);
}
while (reader.Read())
{
var xRoadLog = _mapper.Map <XRoadLog>(reader);
result.Add(xRoadLog);
}
return(result);
}