public async Task Start(bool startImmediately, CancellationToken ct = default(CancellationToken))
{
bool wait = !startImmediately;
while (true)
{
if (wait)
{
await _scheduler.ScheduleOne(Duration.FromMinutes(30), Duration.FromMinutes(20), ct).ConfigureAwait(_taskHelper);
}
wait = true;
try
{
// Fetch for the next settlement period. This should be ready 10 minutes beforehand.
Instant fetchTime = _time.GetCurrentInstant() + Duration.FromMinutes(30);
var data = await _phyBmData.GetAsync(fetchTime.SettlementDate(), fetchTime.SettlementPeriod(), ct).ConfigureAwait(_taskHelper);
await _writer.AppendAsync(data, ct).ConfigureAwait(_taskHelper);
}
catch (Exception e) when(e.Is <IOException>() || e.Is <HttpRequestException>())
{
await _taskHelper.Delay(Duration.FromMinutes(15)).ConfigureAwait(_taskHelper);
}
}
}
public async Task Invoke(HttpContext context, Func <Task> next)
{
var requestInstant = _time.GetCurrentInstant();
await next();
var responseInstant = _time.GetCurrentInstant();
var request = context.Request;
var response = context.Response;
var remoteIp = request.Headers["X-Forwarded-For"].LastOrDefault() ?? context.Connection.RemoteIpAddress.ToString();
var responseDelay = responseInstant - requestInstant;
var line0 = $"{requestInstant.ToString("uuuu-MM-dd HH:mm:ss.fff", null)}, {(int)responseDelay.TotalMilliseconds}";
var line1 = $"{Quote(request.Method)}, {Quote(request.Path)}, {Quote(request.QueryString.ToString())}, {Quote(remoteIp)}";
var line2 = $"{Quote(string.Join(", ", request.Headers["User-Agent"]))}, {response.StatusCode}, {request.ContentLength ?? -1}, {response.ContentLength ?? -1}";
await _appender.AppendLineAsync($"2, {line0}, {line1}, {line2}");
// Version,
// Request time, response delay (ms), req method, req path, req querystring,
// remoteip:port, user agent, response status code, request content length, response content length
}
public static HashSet <string> Seen(ITime time, Db <B1610.Data> b1610Db)
{
lock (s_seenLock)
{
var now = time.GetCurrentInstant();
if (now > s_nextSeenUpdate)
{
s_nextSeenUpdate = now + s_seenUpdateInterval;
s_seen = new HashSet <string>(
b1610Db.ReverseView.Take(20_000).AsEnumerable().Select(x => x.ResourceName)
);
}
return(s_seen);
}
}
private async Task Start0(bool startImmediately, CancellationToken ct)
{
bool wait = !startImmediately;
Duration getOfs = Duration.Zero;
Duration?waitOverride = null;
while (true)
{
if (wait)
{
await _scheduler.ScheduleOne(waitOverride ?? Duration.FromMinutes(2), Duration.FromSeconds(4), ct).ConfigureAwait(_taskHelper);
}
wait = true;
var count = (int)await _reader.CountAsync(ct).ConfigureAwait(_taskHelper);
Instant now = _time.GetCurrentInstant();
Instant from;
if (count > 0)
{
var last = await(await _reader.ReadAsync(count - 1).ConfigureAwait(_taskHelper)).Last().ConfigureAwait(_taskHelper);
from = last.Update + getOfs;
}
else
{
from = now - Duration.FromHours(1);
}
Instant to = from + Duration.FromHours(1);
if (to > now)
{
to = now;
}
var data = await _freq.GetAsync(from + Duration.FromSeconds(1), to, ct).ConfigureAwait(_taskHelper);
if (data.Count == 0 && to < now - Duration.FromDays(1))
{
// If no data, and we're at least a day in the past, then skip forward an hour.
// This is probably because data is missing.
getOfs += Duration.FromHours(1);
waitOverride = Duration.FromSeconds(20);
}
else
{
getOfs = Duration.Zero;
waitOverride = null;
}
await _writer.AppendAsync(data, ct).ConfigureAwait(_taskHelper);
}
}
public bool IsPublished() => _time.GetCurrentInstant() >= PublishTime;
private async Task Start0(CancellationToken ct)
{
while (true)
{
int count = (int)await _reader.CountAsync().ConfigureAwait(_taskHelper);
var items = await(await _reader.ReadAsync(count - 500, ct: ct).ConfigureAwait(_taskHelper)).ToList().ConfigureAwait(_taskHelper);
var lastTime = items.Any() ? items.Max(x => x.SettlementPeriodStart) : _time.GetCurrentInstant() - Duration.FromDays(10);
var fetchTime = lastTime + Duration.FromMinutes(30);
var data = await _b1610.GetAsync(fetchTime.SettlementDate(), fetchTime.SettlementPeriod(), ct).ConfigureAwait(_taskHelper);
if (data.Count == 0)
{
await _scheduler.ScheduleOne(Duration.FromMinutes(30), Duration.FromMinutes(2.5), ct).ConfigureAwait(_taskHelper);
}
else
{
await _writer.AppendAsync(data, ct).ConfigureAwait(_taskHelper);
await _scheduler.ScheduleOne(Duration.FromMinutes(5), Duration.FromMinutes(2.5), ct).ConfigureAwait(_taskHelper);
}
}
}
public Task ScheduleOne(Duration interval, Duration offset, CancellationToken ct = default(CancellationToken)) => _taskHelper.Delay(NextDelay(interval, offset, _time.GetCurrentInstant()), ct);