static async Task <(HttpStatusCode code, HttpResponseMessage message)> GetJournalAsync(DateTime journalDate, Shared.Models.User.Profile user, MSSQLDB db, HttpClient hc, MinioClient minioClient, DiscordWebhook discord) { var oldJournalRow = await db.ExecuteListAsync <UserJournal>( "SELECT TOP 1 * FROM user_journal WHERE user_identifier = @user_identifier AND journal_date = @journal_date", new SqlParameter("user_identifier", user.UserIdentifier), new SqlParameter("journal_date", journalDate) ); if (oldJournalRow.Count > 1) { throw new TooManyOldJournalItemsException(journalDate, user.UserIdentifier); } var previousRow = oldJournalRow.FirstOrDefault(); if (previousRow?.CompleteEntry ?? false) { return(HttpStatusCode.OK, null); } var pollicy = Policy <HttpResponseMessage> .Handle <HttpRequestException>() .OrResult(r => !r.IsSuccessStatusCode) .OrResult(r => r.StatusCode == HttpStatusCode.PartialContent) .WaitAndRetryAsync(100, attempt => TimeSpan.FromSeconds(5)); var journalRequest = await pollicy.ExecuteAsync(() => hc.GetAsync($"/journal/{journalDate.Year}/{journalDate.Month}/{journalDate.Day}")); var journalContent = await journalRequest.Content.ReadAsStringAsync(); if (!journalRequest.IsSuccessStatusCode || journalRequest.StatusCode == HttpStatusCode.PartialContent) { return(journalRequest.StatusCode, journalRequest); } var journalRows = journalContent.Trim().Split('\n', StringSplitOptions.RemoveEmptyEntries); bool updateFileOnS3 = (previousRow?.LastProcessedLineNumber ?? 0) != journalRows.Length && (previousRow?.LastProcessedLine != (journalRows.LastOrDefault() ?? string.Empty)) && journalContent.Trim() != "{}"; if (!string.IsNullOrWhiteSpace(journalContent) && journalContent.Trim() != "{}") { var firstValidRow = string.Empty; foreach (var row in journalRows) { try { _ = JsonDocument.Parse(row).RootElement; firstValidRow = row; break; } catch { } } if (!string.IsNullOrWhiteSpace(firstValidRow)) { try { var row = JsonDocument.Parse(firstValidRow).RootElement; var apiFileHeader = new { Timestamp = row.GetProperty("timestamp").GetString(), Event = "JournalLimpetFileheader", Description = "Missing fileheader from cAPI journal" }; var serializedApiFileHeader = JsonSerializer.Serialize(apiFileHeader, new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }); serializedApiFileHeader = serializedApiFileHeader.Insert(serializedApiFileHeader.Length - 1, " ").Insert(1, " "); journalContent = serializedApiFileHeader + "\n" + journalContent; } catch (Exception ex) { if (ex.ToString().Contains("Json")) { var errorMessage = "Line failed: " + firstValidRow; await SendAdminNotification(discord, "**[JOURNAL]** JSON Reader Exception while fetching first item", errorMessage ); return(HttpStatusCode.InternalServerError, new HttpResponseMessage(HttpStatusCode.InternalServerError) { Content = new StringContent("faulty row: " + firstValidRow) }); } } } } var journalLineCount = journalContent.Trim().Split('\n', StringSplitOptions.RemoveEmptyEntries).Length; var journalBytes = ZipManager.Zip(journalContent.Trim()); string fileName = $"{user.UserIdentifier}/journal/{journalDate.Year}/{journalDate.Month.ToString().PadLeft(2, '0')}/{journalDate.Day.ToString().PadLeft(2, '0')}.journal"; if (updateFileOnS3) { using (var ms = new MemoryStream(journalBytes)) { var policy = Policy .Handle <ConnectionException>() .WaitAndRetryAsync(new[] { TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(4), TimeSpan.FromSeconds(8), TimeSpan.FromSeconds(16), }); await policy.ExecuteAsync(() => minioClient.PutObjectAsync("journal-limpet", fileName, ms, ms.Length, "application/gzip")); } await SSEActivitySender.SendUserActivityAsync(user.UserIdentifier, $"Downloaded journals for {journalDate:yyyy-MM-dd}", $"We downloaded {journalLineCount:N0} lines of journal for this day", "success" ); await SSEActivitySender.SendStatsActivityAsync(db); } if (previousRow == null) { await db.ExecuteNonQueryAsync(@"INSERT INTO user_journal (user_identifier, journal_date, s3_path, last_processed_line, last_processed_line_number, complete_entry, last_update) VALUES (@user_identifier, @journal_date, @s3_path, @last_processed_line, @last_processed_line_number, @complete_entry, GETUTCDATE())", new SqlParameter("user_identifier", user.UserIdentifier), new SqlParameter("journal_date", journalDate), new SqlParameter("s3_path", fileName), new SqlParameter("last_processed_line", journalRows.LastOrDefault() ?? string.Empty), new SqlParameter("last_processed_line_number", journalLineCount), new SqlParameter("complete_entry", DateTime.UtcNow.Date > journalDate.Date) ); } else { await db.ExecuteNonQueryAsync(@"UPDATE user_journal SET last_processed_line = @last_processed_line, last_processed_line_number = @last_processed_line_number, complete_entry = @complete_entry, last_update = GETUTCDATE() WHERE journal_id = @journal_id AND user_identifier = @user_identifier", new SqlParameter("journal_id", previousRow.JournalId), new SqlParameter("user_identifier", user.UserIdentifier), new SqlParameter("last_processed_line", journalRows.LastOrDefault() ?? string.Empty), new SqlParameter("last_processed_line_number", journalLineCount), new SqlParameter("complete_entry", DateTime.UtcNow.Date > journalDate.Date) ); } Thread.Sleep(5000); return(HttpStatusCode.OK, journalRequest); }