static async Task <(HttpStatusCode code, HttpResponseMessage message)> GetJournalAsync(DateTime journalDate, Shared.Models.User.Profile user, MSSQLDB db, HttpClient hc, MinioClient minioClient, DiscordWebhook discord)
{
var oldJournalRow = await db.ExecuteListAsync <UserJournal>(
"SELECT TOP 1 * FROM user_journal WHERE user_identifier = @user_identifier AND journal_date = @journal_date",
new SqlParameter("user_identifier", user.UserIdentifier),
new SqlParameter("journal_date", journalDate)
);
if (oldJournalRow.Count > 1)
{
throw new TooManyOldJournalItemsException(journalDate, user.UserIdentifier);
}
var previousRow = oldJournalRow.FirstOrDefault();
if (previousRow?.CompleteEntry ?? false)
{
return(HttpStatusCode.OK, null);
}
var pollicy = Policy <HttpResponseMessage>
.Handle <HttpRequestException>()
.OrResult(r => !r.IsSuccessStatusCode)
.OrResult(r => r.StatusCode == HttpStatusCode.PartialContent)
.WaitAndRetryAsync(100, attempt => TimeSpan.FromSeconds(5));
var journalRequest = await pollicy.ExecuteAsync(() => hc.GetAsync($"/journal/{journalDate.Year}/{journalDate.Month}/{journalDate.Day}"));
var journalContent = await journalRequest.Content.ReadAsStringAsync();
if (!journalRequest.IsSuccessStatusCode || journalRequest.StatusCode == HttpStatusCode.PartialContent)
{
return(journalRequest.StatusCode, journalRequest);
}
var journalRows = journalContent.Trim().Split('\n', StringSplitOptions.RemoveEmptyEntries);
bool updateFileOnS3 = (previousRow?.LastProcessedLineNumber ?? 0) != journalRows.Length && (previousRow?.LastProcessedLine != (journalRows.LastOrDefault() ?? string.Empty)) && journalContent.Trim() != "{}";
if (!string.IsNullOrWhiteSpace(journalContent) && journalContent.Trim() != "{}")
{
var firstValidRow = string.Empty;
foreach (var row in journalRows)
{
try
{
_ = JsonDocument.Parse(row).RootElement;
firstValidRow = row;
break;
}
catch
{
}
}
if (!string.IsNullOrWhiteSpace(firstValidRow))
{
try
{
var row = JsonDocument.Parse(firstValidRow).RootElement;
var apiFileHeader = new
{
Timestamp = row.GetProperty("timestamp").GetString(),
Event = "JournalLimpetFileheader",
Description = "Missing fileheader from cAPI journal"
};
var serializedApiFileHeader = JsonSerializer.Serialize(apiFileHeader, new JsonSerializerOptions {
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
});
serializedApiFileHeader = serializedApiFileHeader.Insert(serializedApiFileHeader.Length - 1, " ").Insert(1, " ");
journalContent =
serializedApiFileHeader +
"\n" +
journalContent;
}
catch (Exception ex)
{
if (ex.ToString().Contains("Json"))
{
var errorMessage = "Line failed: " + firstValidRow;
await SendAdminNotification(discord,
"**[JOURNAL]** JSON Reader Exception while fetching first item",
errorMessage
);
return(HttpStatusCode.InternalServerError, new HttpResponseMessage(HttpStatusCode.InternalServerError)
{
Content = new StringContent("faulty row: " + firstValidRow)
});
}
}
}
}
var journalLineCount = journalContent.Trim().Split('\n', StringSplitOptions.RemoveEmptyEntries).Length;
var journalBytes = ZipManager.Zip(journalContent.Trim());
string fileName = $"{user.UserIdentifier}/journal/{journalDate.Year}/{journalDate.Month.ToString().PadLeft(2, '0')}/{journalDate.Day.ToString().PadLeft(2, '0')}.journal";
if (updateFileOnS3)
{
using (var ms = new MemoryStream(journalBytes))
{
var policy = Policy
.Handle <ConnectionException>()
.WaitAndRetryAsync(new[] {
TimeSpan.FromSeconds(1),
TimeSpan.FromSeconds(2),
TimeSpan.FromSeconds(4),
TimeSpan.FromSeconds(8),
TimeSpan.FromSeconds(16),
});
await policy.ExecuteAsync(() => minioClient.PutObjectAsync("journal-limpet", fileName, ms, ms.Length, "application/gzip"));
}
await SSEActivitySender.SendUserActivityAsync(user.UserIdentifier,
$"Downloaded journals for {journalDate:yyyy-MM-dd}",
$"We downloaded {journalLineCount:N0} lines of journal for this day",
"success"
);
await SSEActivitySender.SendStatsActivityAsync(db);
}
if (previousRow == null)
{
await db.ExecuteNonQueryAsync(@"INSERT INTO user_journal (user_identifier, journal_date, s3_path, last_processed_line, last_processed_line_number, complete_entry, last_update)
VALUES (@user_identifier, @journal_date, @s3_path, @last_processed_line, @last_processed_line_number, @complete_entry, GETUTCDATE())",
new SqlParameter("user_identifier", user.UserIdentifier),
new SqlParameter("journal_date", journalDate),
new SqlParameter("s3_path", fileName),
new SqlParameter("last_processed_line", journalRows.LastOrDefault() ?? string.Empty),
new SqlParameter("last_processed_line_number", journalLineCount),
new SqlParameter("complete_entry", DateTime.UtcNow.Date > journalDate.Date)
);
}
else
{
await db.ExecuteNonQueryAsync(@"UPDATE user_journal SET
last_processed_line = @last_processed_line,
last_processed_line_number = @last_processed_line_number,
complete_entry = @complete_entry,
last_update = GETUTCDATE()
WHERE journal_id = @journal_id AND user_identifier = @user_identifier",
new SqlParameter("journal_id", previousRow.JournalId),
new SqlParameter("user_identifier", user.UserIdentifier),
new SqlParameter("last_processed_line", journalRows.LastOrDefault() ?? string.Empty),
new SqlParameter("last_processed_line_number", journalLineCount),
new SqlParameter("complete_entry", DateTime.UtcNow.Date > journalDate.Date)
);
}
Thread.Sleep(5000);
return(HttpStatusCode.OK, journalRequest);
}
static async Task <(bool failedRequest, bool shouldBail)> TryGetJournalAsync(DiscordWebhook discord, DateTime journalDate, Shared.Models.User.Profile user, MSSQLDB db, HttpClient hc, MinioClient minioClient, PerformContext context)
{
try
{
var res = await GetJournalAsync(journalDate, user, db, hc, minioClient, discord);
int loop_counter = 0;
while (res.code != HttpStatusCode.OK)
{
Thread.Sleep(5000);
var content = await res.message.Content.ReadAsStringAsync();
if (content.Contains("to purchase Elite: Dangerous"))
{
await db.ExecuteNonQueryAsync("UPDATE user_profile SET skip_download = 1 WHERE user_identifier = @user_identifier", new SqlParameter("user_identifier", user.UserIdentifier));
await SendAdminNotification(
discord,
"Failed to download journal, cannot access cAPI",
"User probably has a Epic Games account",
new Dictionary <string, string>
{
{ "Response code", res.code.ToString() },
{ "Content", content },
{ "User identifier", user.UserIdentifier.ToString() },
{ "Journal date", journalDate.ToString("yyyy-MM-dd") }
}
);
return(false, true);
}
if (loop_counter > 10)
{
await SendAdminNotification(
discord,
"Failed to download journal",
"Encountered an error too many times",
new Dictionary <string, string>
{
{ "Response code", res.code.ToString() },
{ "Content", content },
{ "User identifier", user.UserIdentifier.ToString() },
{ "Journal date", journalDate.ToString("yyyy-MM-dd") }
}
);
return(false, false);
}
switch (res.code)
{
case HttpStatusCode.PartialContent:
Thread.Sleep(5000);
res = await GetJournalAsync(journalDate, user, db, hc, minioClient, discord);
break;
}
loop_counter++;
}
}
catch (TooManyOldJournalItemsException ex)
{
await SendAdminNotification(discord,
"**[JOURNAL]** Exception: Too many old journal items",
"The user somehow has duplicates of journals stored for a single date",
new Dictionary <string, string> {
{ "Exception", ex.ToString() },
{ "User identifier", user.UserIdentifier.ToString() }
}
);
return(false, true);
}
catch (Exception ex)
{
var errorMessage = ex.ToString() + "\n\n" + JsonSerializer.Serialize(user, new JsonSerializerOptions()
{
WriteIndented = true
});
await SendAdminNotification(discord,
"**[JOURNAL]** Unhandled exception while downloading journals",
errorMessage
);
return(false, false);
}
return(true, false);
}
