public static async Task InitializeJournalDownloadersAsync(PerformContext context)
{
context.WriteLine("Looking for journals do download!");
using (var scope = Startup.ServiceProvider.CreateScope())
{
MSSQLDB db = scope.ServiceProvider.GetRequiredService<MSSQLDB>();
IConfiguration configuration = scope.ServiceProvider.GetRequiredService<IConfiguration>();
var usersToDownloadJournalsFor = await db.ExecuteListAsync<Shared.Models.User.Profile>(
@"SELECT *
FROM user_profile
WHERE DATEDIFF(MINUTE, GETUTCDATE(), CAST(JSON_VALUE(user_settings, '$.TokenExpiration') as DATETIMEOFFSET)) > 10
AND last_notification_mail IS NULL
AND skip_download = 0
AND deleted = 0"
);
if (usersToDownloadJournalsFor.Count > 0)
{
await SSEActivitySender.SendGlobalActivityAsync("Fetching journals from Elite", $"Downloading journals for {usersToDownloadJournalsFor.Count} users");
}
foreach (var user in usersToDownloadJournalsFor)
{
if (RedisJobLock.IsLocked($"JournalDownloader.DownloadJournal.{user.UserIdentifier}")) continue;
BackgroundJob.Schedule(() => JournalDownloader.DownloadJournalAsync(user.UserIdentifier, null), TimeSpan.Zero);
}
}
context.WriteLine("All done!");
}
public async Task <IActionResult> Authenticate()
{
var code = Request.Query["code"];
var state = Request.Query["state"];
if (!_memoryCache.TryGetValue("frontierLogin-" + HttpContext.Connection.RemoteIpAddress.ToString(), out string storedState))
{
return(BadRequest("Could not find login token, try again"));
}
if (state != storedState)
{
return(Unauthorized("Invalid state, please relogin"));
}
var redirectUrl = string.Format("{0}://{1}{2}", Request.Scheme, Request.Host, Url.Content("~/api/journal/authenticate"));
using var c = new HttpClient();
var formData = new Dictionary <string, string>();
formData.Add("grant_type", "authorization_code");
formData.Add("code", code);
formData.Add("client_id", _configuration["EliteDangerous:ClientId"]);
formData.Add("client_secret", _configuration["EliteDangerous:ClientSecret"]);
formData.Add("state", state);
formData.Add("redirect_uri", redirectUrl);
var result = await c.PostAsync("https://auth.frontierstore.net/token", new FormUrlEncodedContent(formData));
var tokenInfo = JsonSerializer.Deserialize <OAuth2Response>(await result.Content.ReadAsStringAsync());
if (result.IsSuccessStatusCode)
{
c.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", tokenInfo.AccessToken);
result = await c.GetAsync("https://auth.frontierstore.net/me");
var profile = JsonSerializer.Deserialize <FrontierProfile>(await result.Content.ReadAsStringAsync());
var settings = new Settings
{
AuthToken = tokenInfo.AccessToken,
TokenExpiration = DateTimeOffset.UtcNow.AddSeconds(tokenInfo.ExpiresIn),
RefreshToken = tokenInfo.RefreshToken,
FrontierProfile = profile
};
// Move this so a service later
var matchingUser = (await _db.ExecuteListAsync <Profile>(@"
SELECT *
FROM user_profile
WHERE JSON_VALUE(user_settings, '$.FrontierProfile.customer_id') = @customerId",
new SqlParameter("customerId", profile.CustomerId))
).FirstOrDefault();
if (matchingUser != null)
{
// Update user with new token info
await _db.ExecuteNonQueryAsync("UPDATE user_profile SET user_settings = @settings, last_notification_mail = NULL, skip_download = 0 WHERE user_identifier = @userIdentifier",
new SqlParameter("settings", JsonSerializer.Serialize(settings)),
new SqlParameter("userIdentifier", matchingUser.UserIdentifier)
);
matchingUser.UserSettings = settings;
}
else
{
// Create new user
matchingUser = await _db.ExecuteSingleRowAsync <Profile>("INSERT INTO user_profile (user_settings) OUTPUT INSERTED.* VALUES (@settings)",
new SqlParameter("settings", JsonSerializer.Serialize(settings))
);
var userCount = await _db.ExecuteScalarAsync <long>("SELECT COUNT_BIG(user_identifier) FROM user_profile WHERE deleted = 0");
await SSEActivitySender.SendGlobalActivityAsync("A new user has registered!", $"We now have {userCount:N0} users registered!");
await SSEActivitySender.SendStatsActivityAsync(_db);
BackgroundJob.Enqueue(() => JournalDownloader.DownloadJournalAsync(matchingUser.UserIdentifier, null));
}
var claims = new List <Claim>()
{
new Claim(ClaimTypes.Name, matchingUser.UserIdentifier.ToString())
};
var claimsIdentity = new ClaimsIdentity(claims, CookieAuthenticationDefaults.AuthenticationScheme);
var authProperties = new AuthenticationProperties
{
AllowRefresh = true,
IsPersistent = false,
IssuedUtc = DateTimeOffset.UtcNow
};
await HttpContext.SignInAsync(CookieAuthenticationDefaults.AuthenticationScheme, new ClaimsPrincipal(claimsIdentity), authProperties);
return(LocalRedirect("~/Index"));
}
else
{
return(new JsonResult(await result.Content.ReadAsStringAsync()));
}
}
public static async Task <(int errorCode, string resultContent, TimeSpan executionTime, bool sentData)> UploadJournalItemToEDSM(HttpClient hc, string journalRow, Guid userIdentifier, EDSMIntegrationSettings edsmSettings, EDGameState gameState, StarSystemChecker starSystemChecker)
{
var element = JsonDocument.Parse(journalRow).RootElement;
if (!element.TryGetProperty("event", out JsonElement journalEvent))
{
return(303, string.Empty, TimeSpan.Zero, false);
}
Stopwatch sw = new Stopwatch();
sw.Start();
try
{
element = await SetGamestateProperties(element, gameState, edsmSettings.CommanderName.Trim(), starSystemChecker);
if (System.Enum.TryParse(typeof(IgnoredEvents), journalEvent.GetString(), false, out _))
{
return(304, string.Empty, TimeSpan.Zero, false);
}
if (!gameState.SendEvents)
{
return(104, string.Empty, TimeSpan.Zero, false);
}
var formContent = new MultipartFormDataContent();
var json = JsonSerializer.Serialize(element, new JsonSerializerOptions()
{
WriteIndented = true
});
formContent.Add(new StringContent(edsmSettings.CommanderName.Trim()), "commanderName");
formContent.Add(new StringContent(edsmSettings.ApiKey), "apiKey");
formContent.Add(new StringContent("Journal Limpet"), "fromSoftware");
formContent.Add(new StringContent(SharedSettings.VersionNumber), "fromSoftwareVersion");
formContent.Add(new StringContent(json), "message");
await SSEActivitySender.SendUserLogDataAsync(userIdentifier, new { fromIntegration = "EDSM", data = element });
var policy = Policy
.HandleResult <HttpResponseMessage>(res => !res.IsSuccessStatusCode)
.Or <HttpRequestException>()
.WaitAndRetryAsync(30, retryCount => TimeSpan.FromSeconds(retryCount * 5));
var status = await policy.ExecuteAsync(() => hc.PostAsync("https://www.edsm.net/api-journal-v1", formContent));
var postResponseBytes = await status.Content.ReadAsByteArrayAsync();
var postResponse = System.Text.Encoding.UTF8.GetString(postResponseBytes);
if (!status.IsSuccessStatusCode)
{
return((int)status.StatusCode, postResponse, TimeSpan.FromSeconds(30), true);
}
var resp = JsonSerializer.Deserialize <EDSMApiResponse>(postResponse);
sw.Stop();
return(resp.ResultCode, postResponse, sw.Elapsed, true);
}
catch (InvalidTimestampException)
{
return(206, string.Empty, TimeSpan.FromMilliseconds(100), false);
}
}
private static async Task <bool> SendEventBatch(Guid userIdentifier, PerformContext context, IConfiguration configuration, DiscordWebhook discordClient, HttpClient hc, string lastLine, UserJournal journalItem, bool loggingEnabled, IntegrationJournalData ijd, List <Dictionary <string, object> > journalEvents)
{
var breakJournal = false;
await SSEActivitySender.SendUserLogDataAsync(userIdentifier, new { fromIntegration = "Canonn R&D", data = journalEvents });
var json = JsonSerializer.Serialize(journalEvents, new JsonSerializerOptions()
{
WriteIndented = true
});
if (loggingEnabled)
{
context.WriteLine($"Sent event:\n{json}");
}
var res = await SendEventsToCanonn(hc, configuration, json, context);
switch (res.errorCode)
{
// These codes are OK
case 200:
break;
// We're sending too many requests at once, let's break out of the loop and wait until next batch
case 429:
breakJournal = true;
context.WriteLine("We're sending stuff too quickly, breaking out of the loop");
context.WriteLine(lastLine);
context.WriteLine(res.resultContent);
await discordClient.SendMessageAsync("**[Canonn R&D Upload]** Rate limited by Canonn", new List <DiscordWebhookEmbed>
{
new DiscordWebhookEmbed
{
Description = res.resultContent,
Fields = new Dictionary <string, string>()
{
{ "User identifier", userIdentifier.ToString() },
{ "Last line", lastLine },
{ "Journal", journalItem.S3Path },
{ "Current GameState", JsonSerializer.Serialize(ijd.CurrentGameState, new JsonSerializerOptions {
WriteIndented = true
}) }
}.Select(k => new DiscordWebhookEmbedField {
Name = k.Key, Value = k.Value
}).ToList()
}
});
await Task.Delay(30000);
break;
// Exceptions and debug
case 500: // Exception: %%
case 501: // %%
case 502: // Broken gateway
case 503:
breakJournal = true;
context.WriteLine("We got an error from the service, breaking off!");
context.WriteLine(lastLine);
context.WriteLine(res.resultContent);
await discordClient.SendMessageAsync("**[Canonn R&D Upload]** Error from the API", new List <DiscordWebhookEmbed>
{
new DiscordWebhookEmbed
{
Description = res.resultContent,
Fields = new Dictionary <string, string>()
{
{ "User identifier", userIdentifier.ToString() },
{ "Last line", lastLine },
{ "Journal", journalItem.S3Path },
{ "Current GameState", JsonSerializer.Serialize(ijd.CurrentGameState, new JsonSerializerOptions {
WriteIndented = true
}) }
}.Select(k => new DiscordWebhookEmbedField {
Name = k.Key, Value = k.Value
}).ToList()
}
});
break;
default:
breakJournal = true;
context.WriteLine("We got an error from the service, breaking off!");
context.WriteLine(lastLine);
context.WriteLine(res.resultContent);
await discordClient.SendMessageAsync("**[Canonn R&D Upload]** Unhandled response code", new List <DiscordWebhookEmbed>
{
new DiscordWebhookEmbed
{
Description = res.resultContent,
Fields = new Dictionary <string, string>()
{
{ "User identifier", userIdentifier.ToString() },
{ "Last line", lastLine },
{ "Journal", journalItem.S3Path },
{ "Current GameState", JsonSerializer.Serialize(ijd.CurrentGameState, new JsonSerializerOptions {
WriteIndented = true
}) }
}.Select(k => new DiscordWebhookEmbedField {
Name = k.Key, Value = k.Value
}).ToList()
}
});
break;
}
return(breakJournal);
}
static async Task <(HttpStatusCode code, HttpResponseMessage message)> GetJournalAsync(DateTime journalDate, Shared.Models.User.Profile user, MSSQLDB db, HttpClient hc, MinioClient minioClient, DiscordWebhook discord)
{
var oldJournalRow = await db.ExecuteListAsync <UserJournal>(
"SELECT TOP 1 * FROM user_journal WHERE user_identifier = @user_identifier AND journal_date = @journal_date",
new SqlParameter("user_identifier", user.UserIdentifier),
new SqlParameter("journal_date", journalDate)
);
if (oldJournalRow.Count > 1)
{
throw new TooManyOldJournalItemsException(journalDate, user.UserIdentifier);
}
var previousRow = oldJournalRow.FirstOrDefault();
if (previousRow?.CompleteEntry ?? false)
{
return(HttpStatusCode.OK, null);
}
var pollicy = Policy <HttpResponseMessage>
.Handle <HttpRequestException>()
.OrResult(r => !r.IsSuccessStatusCode)
.OrResult(r => r.StatusCode == HttpStatusCode.PartialContent)
.WaitAndRetryAsync(100, attempt => TimeSpan.FromSeconds(5));
var journalRequest = await pollicy.ExecuteAsync(() => hc.GetAsync($"/journal/{journalDate.Year}/{journalDate.Month}/{journalDate.Day}"));
var journalContent = await journalRequest.Content.ReadAsStringAsync();
if (!journalRequest.IsSuccessStatusCode || journalRequest.StatusCode == HttpStatusCode.PartialContent)
{
return(journalRequest.StatusCode, journalRequest);
}
var journalRows = journalContent.Trim().Split('\n', StringSplitOptions.RemoveEmptyEntries);
bool updateFileOnS3 = (previousRow?.LastProcessedLineNumber ?? 0) != journalRows.Length && (previousRow?.LastProcessedLine != (journalRows.LastOrDefault() ?? string.Empty)) && journalContent.Trim() != "{}";
if (!string.IsNullOrWhiteSpace(journalContent) && journalContent.Trim() != "{}")
{
var firstValidRow = string.Empty;
foreach (var row in journalRows)
{
try
{
_ = JsonDocument.Parse(row).RootElement;
firstValidRow = row;
break;
}
catch
{
}
}
if (!string.IsNullOrWhiteSpace(firstValidRow))
{
try
{
var row = JsonDocument.Parse(firstValidRow).RootElement;
var apiFileHeader = new
{
Timestamp = row.GetProperty("timestamp").GetString(),
Event = "JournalLimpetFileheader",
Description = "Missing fileheader from cAPI journal"
};
var serializedApiFileHeader = JsonSerializer.Serialize(apiFileHeader, new JsonSerializerOptions {
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
});
serializedApiFileHeader = serializedApiFileHeader.Insert(serializedApiFileHeader.Length - 1, " ").Insert(1, " ");
journalContent =
serializedApiFileHeader +
"\n" +
journalContent;
}
catch (Exception ex)
{
if (ex.ToString().Contains("Json"))
{
var errorMessage = "Line failed: " + firstValidRow;
await SendAdminNotification(discord,
"**[JOURNAL]** JSON Reader Exception while fetching first item",
errorMessage
);
return(HttpStatusCode.InternalServerError, new HttpResponseMessage(HttpStatusCode.InternalServerError)
{
Content = new StringContent("faulty row: " + firstValidRow)
});
}
}
}
}
var journalLineCount = journalContent.Trim().Split('\n', StringSplitOptions.RemoveEmptyEntries).Length;
var journalBytes = ZipManager.Zip(journalContent.Trim());
string fileName = $"{user.UserIdentifier}/journal/{journalDate.Year}/{journalDate.Month.ToString().PadLeft(2, '0')}/{journalDate.Day.ToString().PadLeft(2, '0')}.journal";
if (updateFileOnS3)
{
using (var ms = new MemoryStream(journalBytes))
{
var policy = Policy
.Handle <ConnectionException>()
.WaitAndRetryAsync(new[] {
TimeSpan.FromSeconds(1),
TimeSpan.FromSeconds(2),
TimeSpan.FromSeconds(4),
TimeSpan.FromSeconds(8),
TimeSpan.FromSeconds(16),
});
await policy.ExecuteAsync(() => minioClient.PutObjectAsync("journal-limpet", fileName, ms, ms.Length, "application/gzip"));
}
await SSEActivitySender.SendUserActivityAsync(user.UserIdentifier,
$"Downloaded journals for {journalDate:yyyy-MM-dd}",
$"We downloaded {journalLineCount:N0} lines of journal for this day",
"success"
);
await SSEActivitySender.SendStatsActivityAsync(db);
}
if (previousRow == null)
{
await db.ExecuteNonQueryAsync(@"INSERT INTO user_journal (user_identifier, journal_date, s3_path, last_processed_line, last_processed_line_number, complete_entry, last_update)
VALUES (@user_identifier, @journal_date, @s3_path, @last_processed_line, @last_processed_line_number, @complete_entry, GETUTCDATE())",
new SqlParameter("user_identifier", user.UserIdentifier),
new SqlParameter("journal_date", journalDate),
new SqlParameter("s3_path", fileName),
new SqlParameter("last_processed_line", journalRows.LastOrDefault() ?? string.Empty),
new SqlParameter("last_processed_line_number", journalLineCount),
new SqlParameter("complete_entry", DateTime.UtcNow.Date > journalDate.Date)
);
}
else
{
await db.ExecuteNonQueryAsync(@"UPDATE user_journal SET
last_processed_line = @last_processed_line,
last_processed_line_number = @last_processed_line_number,
complete_entry = @complete_entry,
last_update = GETUTCDATE()
WHERE journal_id = @journal_id AND user_identifier = @user_identifier",
new SqlParameter("journal_id", previousRow.JournalId),
new SqlParameter("user_identifier", user.UserIdentifier),
new SqlParameter("last_processed_line", journalRows.LastOrDefault() ?? string.Empty),
new SqlParameter("last_processed_line_number", journalLineCount),
new SqlParameter("complete_entry", DateTime.UtcNow.Date > journalDate.Date)
);
}
Thread.Sleep(5000);
return(HttpStatusCode.OK, journalRequest);
}
public static async Task DownloadJournalAsync(Guid userIdentifier, PerformContext context)
{
using (var rlock = new RedisJobLock($"JournalDownloader.DownloadJournal.{userIdentifier}"))
{
if (!rlock.TryTakeLock())
{
return;
}
context.WriteLine($"Looking for journals for user {userIdentifier}");
using (var scope = Startup.ServiceProvider.CreateScope())
{
MSSQLDB db = scope.ServiceProvider.GetRequiredService <MSSQLDB>();
IConfiguration configuration = scope.ServiceProvider.GetRequiredService <IConfiguration>();
MinioClient minioClient = scope.ServiceProvider.GetRequiredService <MinioClient>();
var discordClient = scope.ServiceProvider.GetRequiredService <DiscordWebhook>();
var user = await db.ExecuteSingleRowAsync <Profile>(
@"SELECT * FROM user_profile WHERE user_identifier = @user_identifier AND last_notification_mail IS NULL AND deleted = 0 AND skip_download = 0",
new SqlParameter("user_identifier", userIdentifier)
);
if (user == null)
{
return;
}
var authToken = user.UserSettings.AuthToken;
var hc = SharedSettings.GetHttpClient(scope);
hc.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", authToken);
hc.BaseAddress = new Uri("https://companion.orerve.net");
var profile = await GetProfileAsync(hc);
var profileJson = await profile.Content.ReadAsStringAsync();
if (!profile.IsSuccessStatusCode || string.IsNullOrWhiteSpace(profileJson))
{
context.WriteLine($"Invalid statuscode: {profile.StatusCode} {profile.ReasonPhrase}");
bool resetAuth = false;
switch (profile.StatusCode)
{
case HttpStatusCode.BadRequest:
// User does not potentially own the game
resetAuth = true;
break;
case HttpStatusCode.Unauthorized:
// Invalid token (or Epic games)
resetAuth = true;
break;
}
if (string.IsNullOrWhiteSpace(profileJson))
{
resetAuth = true;
}
if (resetAuth)
{
await SSEActivitySender.SendUserActivityAsync(user.UserIdentifier,
"Could not authorize you",
"Sorry, but there seems to be something wrong with your account. Please contact us so we can try and figure out what's wrong!"
);
context.WriteLine("Bailing out early, user doesn't own Elite or has issues with cAPI auth");
if (!string.IsNullOrWhiteSpace(user.NotificationEmail))
{
context.WriteLine("User cannot be fetched, asking them to reauthenticate");
await SendLoginNotificationMethod.SendLoginNotification(db, configuration, user);
}
}
return;
}
var profileData = JsonSerializer.Deserialize <EliteProfile>(profileJson);
context.WriteLine(profileJson);
if (profileJson == "{}")
{
if (!string.IsNullOrWhiteSpace(user.NotificationEmail))
{
context.WriteLine("User cannot be fetched, asking them to reauthenticate");
await SendLoginNotificationMethod.SendLoginNotification(db, configuration, user);
}
return;
}
context.WriteLine($"Downloading journals for {profileData.Commander.Name}");
DateTime journalDate = DateTime.Today.AddDays(-25);
await SSEActivitySender.SendUserActivityAsync(user.UserIdentifier,
"Downloading your journals",
"We're beginning to download your journals now, a few notifications may pop up."
);
while (journalDate.Date != DateTime.Today)
{
context.WriteLine($"Fetching data for {journalDate.ToString("yyyy-MM-dd")}");
var req = await TryGetJournalAsync(discordClient, journalDate, user, db, hc, minioClient, context);
if (req.shouldBail)
{
// Failed to get loop journal
context.WriteLine($"Bailing because of errors");
return;
}
journalDate = journalDate.AddDays(1);
}
context.WriteLine($"Fetching data for {journalDate.ToString("yyyy-MM-dd")}");
var reqOut = await TryGetJournalAsync(discordClient, journalDate, user, db, hc, minioClient, context);
if (reqOut.shouldBail)
{
// Failed to get loop journal
context.WriteLine($"Bailing because of errors");
return;
}
if (user.SendToEDDN && !RedisJobLock.IsLocked($"EDDNUserUploader.UploadAsync.{user.UserIdentifier}"))
{
var userJournals = await db.ExecuteScalarAsync <long>(
"SELECT COUNT_BIG(journal_id) FROM user_journal WHERE user_identifier = @user_identifier AND sent_to_eddn = 0 AND last_processed_line_number >= sent_to_eddn_line",
new SqlParameter("user_identifier", userIdentifier)
);
if (userJournals > 0)
{
context.WriteLine($"Sending {userJournals} journals to EDDN");
BackgroundJob.Enqueue(() => EDDNUserUploader.UploadAsync(user.UserIdentifier, profileData.Commander.Name, null));
}
}
if (user.IntegrationSettings.ContainsKey("EDSM") && user.IntegrationSettings["EDSM"].GetTypedObject <EDSMIntegrationSettings>().Enabled)
{
var userJournals = await db.ExecuteScalarAsync <long>(
"SELECT COUNT_BIG(journal_id) FROM user_journal WHERE user_identifier = @user_identifier AND ISNULL(JSON_VALUE(integration_data, '$.EDSM.lastSentLineNumber'), '0') < last_processed_line_number AND last_processed_line_number > 0 AND ISNULL(JSON_VALUE(integration_data, '$.EDSM.fullySent'), 'false') = 'false'",
new SqlParameter("user_identifier", userIdentifier)
);
if (userJournals > 0)
{
if (!RedisJobLock.IsLocked($"EDSMUserUploader.UploadAsync.{user.UserIdentifier}"))
{
context.WriteLine($"Sending {userJournals} journals to EDSM");
BackgroundJob.Enqueue(() => EDSMUserUploader.UploadAsync(user.UserIdentifier, null));
}
}
}
if (user.IntegrationSettings.ContainsKey("Canonn R&D") && user.IntegrationSettings["Canonn R&D"].GetTypedObject <CanonnRDIntegrationSettings>().Enabled)
{
var userJournals = await db.ExecuteScalarAsync <long>(
"SELECT COUNT_BIG(journal_id) FROM user_journal WHERE user_identifier = @user_identifier AND ISNULL(JSON_VALUE(integration_data, '$.\"Canonn R\\u0026D\".lastSentLineNumber'), '0') <= last_processed_line_number AND last_processed_line_number > 0 AND ISNULL(JSON_VALUE(integration_data, '$.\"Canonn R\\u0026D\".fullySent'), 'false') = 'false'",
new SqlParameter("user_identifier", userIdentifier)
);
if (userJournals > 0)
{
if (!RedisJobLock.IsLocked($"CanonnRDUserUploader.UploadAsync.{user.UserIdentifier}"))
{
context.WriteLine($"Sending {userJournals} journals to Canonn");
BackgroundJob.Enqueue(() => CanonnRDUserUploader.UploadAsync(user.UserIdentifier, profileData.Commander.Name, null));
}
}
}
context.WriteLine("All done!");
}
}
}
