public static async Task InitializeJournalDownloadersAsync(PerformContext context)
{
context.WriteLine("Looking for journals do download!");
using (var scope = Startup.ServiceProvider.CreateScope())
{
MSSQLDB db = scope.ServiceProvider.GetRequiredService<MSSQLDB>();
IConfiguration configuration = scope.ServiceProvider.GetRequiredService<IConfiguration>();
var usersToDownloadJournalsFor = await db.ExecuteListAsync<Shared.Models.User.Profile>(
@"SELECT *
FROM user_profile
WHERE DATEDIFF(MINUTE, GETUTCDATE(), CAST(JSON_VALUE(user_settings, '$.TokenExpiration') as DATETIMEOFFSET)) > 10
AND last_notification_mail IS NULL
AND skip_download = 0
AND deleted = 0"
);
if (usersToDownloadJournalsFor.Count > 0)
{
await SSEActivitySender.SendGlobalActivityAsync("Fetching journals from Elite", $"Downloading journals for {usersToDownloadJournalsFor.Count} users");
}
foreach (var user in usersToDownloadJournalsFor)
{
if (RedisJobLock.IsLocked($"JournalDownloader.DownloadJournal.{user.UserIdentifier}")) continue;
BackgroundJob.Schedule(() => JournalDownloader.DownloadJournalAsync(user.UserIdentifier, null), TimeSpan.Zero);
}
}
context.WriteLine("All done!");
}
public async Task <JsonResult> GetInfo([FromQuery] string uuid)
{
var user = (await _db.ExecuteListAsync <Profile>(
"SELECT * FROM user_profile WHERE user_identifier = @id",
new SqlParameter("@id", Guid.Parse(uuid))))
.FirstOrDefault();
if (user != null)
{
// This UUID has a user!
return(new JsonResult(user));
}
return(new JsonResult(new
{
success = false,
error = "Missing account"
}));
}
public async Task OnGetAsync()
{
JournalItems = await _db.ExecuteListAsync <JournalListItem>("select * from user_journal where last_processed_line_number > 0 AND user_identifier = @user_identifier ORDER BY journal_date DESC", new SqlParameter("user_identifier", User.Identity.Name));
}
public static async Task RefreshUserTokensAsync(PerformContext context)
{
context.WriteLine("Looking for tokens to refresh!");
using (var scope = Startup.ServiceProvider.CreateScope())
{
MSSQLDB db = scope.ServiceProvider.GetRequiredService <MSSQLDB>();
IConfiguration configuration = scope.ServiceProvider.GetRequiredService <IConfiguration>();
var soonExpiringUsers = await db.ExecuteListAsync <Shared.Models.User.Profile>(
@"SELECT *
FROM user_profile
WHERE DATEDIFF(MINUTE, GETUTCDATE(), CAST(JSON_VALUE(user_settings, '$.TokenExpiration') as DATETIMEOFFSET)) < 10
AND last_notification_mail IS NULL
AND deleted = 0"
);
context.WriteLine($"Found {soonExpiringUsers.Count} user(s) to refresh tokens for");
IHttpClientFactory _hcf = scope.ServiceProvider.GetRequiredService <IHttpClientFactory>();
var hc = _hcf.CreateClient();
foreach (var user in soonExpiringUsers.WithProgress(context))
{
var res = await hc.PostAsync("https://auth.frontierstore.net/token",
new FormUrlEncodedContent(new Dictionary <string, string>
{
{ "grant_type", "refresh_token" },
{ "refresh_token", user.UserSettings.RefreshToken.ToString() },
{ "client_id", configuration["EliteDangerous:ClientId"] }
})
);
if (!res.IsSuccessStatusCode)
{
// The user is not authorized to perform more automatic refreshes of the token
// Send notification to the user that they need to re-login if they want to keep getting their journals stored
if (!string.IsNullOrWhiteSpace(user.NotificationEmail))
{
await SendLoginNotificationMethod.SendLoginNotification(db, configuration, user);
}
}
else
{
// We managed to grab a new token, lets save it!
var tokenInfo = JsonSerializer.Deserialize <OAuth2Response>(await res.Content.ReadAsStringAsync());
var settings = new Settings
{
AuthToken = tokenInfo.AccessToken,
TokenExpiration = DateTimeOffset.UtcNow.AddSeconds(tokenInfo.ExpiresIn),
RefreshToken = tokenInfo.RefreshToken,
FrontierProfile = user.UserSettings.FrontierProfile
};
// Update user with new token info
await db.ExecuteNonQueryAsync("UPDATE user_profile SET user_settings = @settings, last_notification_mail = NULL WHERE user_identifier = @userIdentifier",
new SqlParameter("@settings", JsonSerializer.Serialize(settings)),
new SqlParameter("@userIdentifier", user.UserIdentifier)
);
}
}
}
}
public async Task <List <EDSystemData> > GetSystemDataFromSystemNameAsync(string systemName)
{
return(await _db.ExecuteListAsync <EDSystemData>("SELECT * FROM EliteSystem WHERE StarSystem = @starSystem", new SqlParameter("starSystem", systemName)));
}
static async Task <(HttpStatusCode code, HttpResponseMessage message)> GetJournalAsync(DateTime journalDate, Shared.Models.User.Profile user, MSSQLDB db, HttpClient hc, MinioClient minioClient, DiscordWebhook discord)
{
var oldJournalRow = await db.ExecuteListAsync <UserJournal>(
"SELECT TOP 1 * FROM user_journal WHERE user_identifier = @user_identifier AND journal_date = @journal_date",
new SqlParameter("user_identifier", user.UserIdentifier),
new SqlParameter("journal_date", journalDate)
);
if (oldJournalRow.Count > 1)
{
throw new TooManyOldJournalItemsException(journalDate, user.UserIdentifier);
}
var previousRow = oldJournalRow.FirstOrDefault();
if (previousRow?.CompleteEntry ?? false)
{
return(HttpStatusCode.OK, null);
}
var pollicy = Policy <HttpResponseMessage>
.Handle <HttpRequestException>()
.OrResult(r => !r.IsSuccessStatusCode)
.OrResult(r => r.StatusCode == HttpStatusCode.PartialContent)
.WaitAndRetryAsync(100, attempt => TimeSpan.FromSeconds(5));
var journalRequest = await pollicy.ExecuteAsync(() => hc.GetAsync($"/journal/{journalDate.Year}/{journalDate.Month}/{journalDate.Day}"));
var journalContent = await journalRequest.Content.ReadAsStringAsync();
if (!journalRequest.IsSuccessStatusCode || journalRequest.StatusCode == HttpStatusCode.PartialContent)
{
return(journalRequest.StatusCode, journalRequest);
}
var journalRows = journalContent.Trim().Split('\n', StringSplitOptions.RemoveEmptyEntries);
bool updateFileOnS3 = (previousRow?.LastProcessedLineNumber ?? 0) != journalRows.Length && (previousRow?.LastProcessedLine != (journalRows.LastOrDefault() ?? string.Empty)) && journalContent.Trim() != "{}";
if (!string.IsNullOrWhiteSpace(journalContent) && journalContent.Trim() != "{}")
{
var firstValidRow = string.Empty;
foreach (var row in journalRows)
{
try
{
_ = JsonDocument.Parse(row).RootElement;
firstValidRow = row;
break;
}
catch
{
}
}
if (!string.IsNullOrWhiteSpace(firstValidRow))
{
try
{
var row = JsonDocument.Parse(firstValidRow).RootElement;
var apiFileHeader = new
{
Timestamp = row.GetProperty("timestamp").GetString(),
Event = "JournalLimpetFileheader",
Description = "Missing fileheader from cAPI journal"
};
var serializedApiFileHeader = JsonSerializer.Serialize(apiFileHeader, new JsonSerializerOptions {
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
});
serializedApiFileHeader = serializedApiFileHeader.Insert(serializedApiFileHeader.Length - 1, " ").Insert(1, " ");
journalContent =
serializedApiFileHeader +
"\n" +
journalContent;
}
catch (Exception ex)
{
if (ex.ToString().Contains("Json"))
{
var errorMessage = "Line failed: " + firstValidRow;
await SendAdminNotification(discord,
"**[JOURNAL]** JSON Reader Exception while fetching first item",
errorMessage
);
return(HttpStatusCode.InternalServerError, new HttpResponseMessage(HttpStatusCode.InternalServerError)
{
Content = new StringContent("faulty row: " + firstValidRow)
});
}
}
}
}
var journalLineCount = journalContent.Trim().Split('\n', StringSplitOptions.RemoveEmptyEntries).Length;
var journalBytes = ZipManager.Zip(journalContent.Trim());
string fileName = $"{user.UserIdentifier}/journal/{journalDate.Year}/{journalDate.Month.ToString().PadLeft(2, '0')}/{journalDate.Day.ToString().PadLeft(2, '0')}.journal";
if (updateFileOnS3)
{
using (var ms = new MemoryStream(journalBytes))
{
var policy = Policy
.Handle <ConnectionException>()
.WaitAndRetryAsync(new[] {
TimeSpan.FromSeconds(1),
TimeSpan.FromSeconds(2),
TimeSpan.FromSeconds(4),
TimeSpan.FromSeconds(8),
TimeSpan.FromSeconds(16),
});
await policy.ExecuteAsync(() => minioClient.PutObjectAsync("journal-limpet", fileName, ms, ms.Length, "application/gzip"));
}
await SSEActivitySender.SendUserActivityAsync(user.UserIdentifier,
$"Downloaded journals for {journalDate:yyyy-MM-dd}",
$"We downloaded {journalLineCount:N0} lines of journal for this day",
"success"
);
await SSEActivitySender.SendStatsActivityAsync(db);
}
if (previousRow == null)
{
await db.ExecuteNonQueryAsync(@"INSERT INTO user_journal (user_identifier, journal_date, s3_path, last_processed_line, last_processed_line_number, complete_entry, last_update)
VALUES (@user_identifier, @journal_date, @s3_path, @last_processed_line, @last_processed_line_number, @complete_entry, GETUTCDATE())",
new SqlParameter("user_identifier", user.UserIdentifier),
new SqlParameter("journal_date", journalDate),
new SqlParameter("s3_path", fileName),
new SqlParameter("last_processed_line", journalRows.LastOrDefault() ?? string.Empty),
new SqlParameter("last_processed_line_number", journalLineCount),
new SqlParameter("complete_entry", DateTime.UtcNow.Date > journalDate.Date)
);
}
else
{
await db.ExecuteNonQueryAsync(@"UPDATE user_journal SET
last_processed_line = @last_processed_line,
last_processed_line_number = @last_processed_line_number,
complete_entry = @complete_entry,
last_update = GETUTCDATE()
WHERE journal_id = @journal_id AND user_identifier = @user_identifier",
new SqlParameter("journal_id", previousRow.JournalId),
new SqlParameter("user_identifier", user.UserIdentifier),
new SqlParameter("last_processed_line", journalRows.LastOrDefault() ?? string.Empty),
new SqlParameter("last_processed_line_number", journalLineCount),
new SqlParameter("complete_entry", DateTime.UtcNow.Date > journalDate.Date)
);
}
Thread.Sleep(5000);
return(HttpStatusCode.OK, journalRequest);
}