public async Task <IEnumerable <Ban> > GetBansAsync(int page = 1)
{
var request = new RestRequest($"bans/{RecordsPerPage}/{page}", Method.GET, DataFormat.Json);
var response = await Client.ExecuteAsync(request);
if (response.StatusCode != HttpStatusCode.OK)
{
FailedRequest(response);
}
var toReturn = new List <Ban>();
var content = JsonSerializer.Deserialize <Dictionary <string, JsonElement> >(response.Content);
foreach (var ban in content["value"].GetProperty("bans").EnumerateArray())
{
// Need to get both the expiration as well as the unbanned time as they can differ
DateTime?expiration = null;
var unbannedTime = ban.GetProperty("unbannedTime").GetString();
var expireTime = ban.GetProperty("banExpireTime").GetString();
if (unbannedTime != null)
{
expiration = DateTime.Parse(unbannedTime);
}
if (expireTime != null)
{
expiration ??= DateTime.Parse(expireTime);
}
// Get ban
var toAdd = new Ban()
{
BannedOn = DateTime.Parse(ban.GetProperty("banApplyTime").GetString()),
BannedBy = ban.GetProperty("adminCkey").GetString(),
BanType = ban.GetProperty("role")[0].GetString().ToLower() == "server"
? BanType.Server
: BanType.Job,
Expires = expiration,
CKey = ban.GetProperty("bannedCkey").GetString(),
Reason = ban.GetProperty("reason").GetString(),
SourceNavigation = BanSource
};
// Add jobs if relevant
if (toAdd.BanType == BanType.Job)
{
toAdd.AddJobRange(ban.GetProperty("role").EnumerateArray().Select(x => x.GetString()));
}
// Specify UTC
toAdd.BannedOn = DateTime.SpecifyKind(toAdd.BannedOn, DateTimeKind.Utc);
if (toAdd.Expires.HasValue)
{
toAdd.Expires = DateTime.SpecifyKind(toAdd.Expires.Value, DateTimeKind.Utc);
}
toReturn.Add(toAdd);
}
return(toReturn);
}
internal async Task <IEnumerable <Ban> > GetBansAsync(int page = 1)
{
var request =
new RestRequest("bans", Method.GET, DataFormat.Json).AddQueryParameter("page", page.ToString());
var response = await Client.ExecuteAsync(request);
if (response.StatusCode != HttpStatusCode.OK)
{
FailedRequest(response);
}
var toReturn = new List <Ban>();
var content = JsonSerializer.Deserialize <JsonElement>(response.Content);
foreach (var b in content.GetProperty("data").EnumerateArray())
{
var expiryString = b.GetProperty("unbanned_datetime").GetString() ??
b.GetProperty("expiration_time").GetString();
var toAdd = new Ban()
{
BannedOn = DateTime.SpecifyKind(DateTime.Parse(b.GetProperty("bantime").GetString()),
DateTimeKind.Utc),
BannedBy = b.GetProperty("a_ckey").GetString(),
UnbannedBy = b.GetProperty("unbanned_ckey").GetString(),
BanType = b.GetProperty("roles").EnumerateArray().Select(x => x.GetString()).Contains("Server")
? BanType.Server
: BanType.Job,
Expires = expiryString == null
? null
: DateTime.SpecifyKind(DateTime.Parse(expiryString), DateTimeKind.Utc),
CKey = b.GetProperty("ckey").GetString(),
Reason = b.GetProperty("reason").GetString(),
BanID = b.GetProperty("id").GetInt32().ToString(),
SourceNavigation = ParseBanSource(b.GetProperty("server_name").GetString())
};
if (toAdd.BanType == BanType.Job)
{
toAdd.AddJobRange(b.GetProperty("roles").EnumerateArray().Select(x => x.GetString()));
}
if (b.GetProperty("global_ban").GetInt32() == 1)
{
toAdd.AddAttribute(BanAttribute.BeeStationGlobal);
}
toReturn.Add(toAdd);
}
return(toReturn);
}
public async Task <IEnumerable <Ban> > GetBansAsync(int page = 1)
{
var request = new RestRequest($"bans/{RECORDS_PER_PAGE}/{page}", Method.GET, DataFormat.Json);
var response = await _client.ExecuteAsync(request);
if (response.StatusCode != System.Net.HttpStatusCode.OK)
{
_logger.LogError($"Fulpstation website returned a non-200 HTTP response code: {response.StatusCode}, aborting parse.");
throw new BanSourceUnavailableException($"Fulpstation website returned a non-200 HTTP response code: {response.StatusCode}, aborting parse.");
}
var toReturn = new List <Ban>();
var content = JsonSerializer.Deserialize <Dictionary <string, JsonElement> >(response.Content);
foreach (var ban in content["value"].GetProperty("bans").EnumerateArray())
{
// Get ban
var toAdd = new Ban()
{
BannedOn = DateTime.Parse(ban.GetProperty("banApplyTime").GetString()),
BannedBy = ban.GetProperty("adminCkey").GetString(),
BanType = ban.GetProperty("role")[0].GetString().ToLower() == "server" ? BanType.Server : BanType.Job,
Expires = ban.GetProperty("banExpireTime").GetString() == null ? (DateTime?)null : DateTime.Parse(ban.GetProperty("banExpireTime").GetString()),
CKey = ban.GetProperty("bannedCkey").GetString(),
Reason = ban.GetProperty("reason").GetString(),
SourceNavigation = _banSource
};
// Add jobs if relevant
if (toAdd.BanType == BanType.Job)
{
toAdd.AddJobRange(ban.GetProperty("role").EnumerateArray().Select(x => x.GetString()));
}
// Specify UTC
toAdd.BannedOn = DateTime.SpecifyKind(toAdd.BannedOn, DateTimeKind.Utc);
if (toAdd.Expires.HasValue)
{
toAdd.Expires = DateTime.SpecifyKind(toAdd.Expires.Value, DateTimeKind.Utc);
}
toReturn.Add(toAdd);
}
return(toReturn);
}
private async Task <IEnumerable <Ban> > GetBansAsync(int page = 1)
{
var request = new RestRequest("bans", Method.GET, DataFormat.Json)
.AddQueryParameter("json", "1")
.AddQueryParameter("page", page.ToString())
.AddQueryParameter("amount", "1000");
var response = await Client.ExecuteAsync(request);
if (response.StatusCode != HttpStatusCode.OK)
{
FailedRequest(response);
}
var toReturn = new List <Ban>();
var content = JsonSerializer.Deserialize <IEnumerable <Dictionary <string, JsonElement> > >(response.Content);
foreach (var b in content)
{
var expiryString = b["unbanned_datetime"].GetString() ?? b["expiration_time"].GetString();
var toAdd = new Ban()
{
BannedOn = DateTime.SpecifyKind(DateTime.Parse(b["bantime"].GetString()), DateTimeKind.Utc),
BannedBy = b["a_ckey"].GetString(),
UnbannedBy = b["unbanned_ckey"].GetString(),
BanType = b["roles"].EnumerateArray().Select(x => x.GetString()).Contains("Server")
? BanType.Server
: BanType.Job,
Expires = expiryString == null ? null : DateTime.SpecifyKind(DateTime.Parse(expiryString), DateTimeKind.Utc),
CKey = b["ckey"].GetString(),
Reason = b["reason"].GetString(),
BanID = b["id"].GetInt32().ToString(),
SourceNavigation = BanSource
};
if (toAdd.BanType == BanType.Job)
{
toAdd.AddJobRange(b["roles"].EnumerateArray().Select(x => x.GetString()));
}
toReturn.Add(toAdd);
}
return(toReturn);
}
public async Task <IEnumerable <Ban> > GetBansAsync(int page = 1)
{
var request = new RestRequest("bans", Method.GET, DataFormat.Json).AddQueryParameter("json", "1").AddQueryParameter("page", page.ToString());
var response = await _client.ExecuteAsync(request);
if (response.StatusCode != System.Net.HttpStatusCode.OK)
{
_logger.LogError($"Beestation website returned a non-200 HTTP response code: {response.StatusCode}, aborting parse.");
throw new BanSourceUnavailableException($"Beestation website returned a non-200 HTTP response code: {response.StatusCode}, aborting parse.");
}
var toReturn = new List <Ban>();
var content = JsonSerializer.Deserialize <IEnumerable <Dictionary <string, JsonElement> > >(response.Content);
foreach (var b in content)
{
var toAdd = new Ban()
{
BannedOn = DateTime.Parse(b["ban_date"].GetString()).ToUniversalTime(),
BannedBy = b["banner"].GetString(),
BanType = ParseBanType(b["type"].GetString()),
Expires = b["unban_date"].GetString() == null ? (DateTime?)null : DateTime.Parse(b["unban_date"].GetString()).ToUniversalTime(),
CKey = b["user"].GetString(),
Reason = b["reason"].GetString(),
BanID = b["id"].GetInt32().ToString(),
SourceNavigation = ParseBanSource(b["server"].GetString())
};
if (toAdd.BanType == BanType.Job)
{
toAdd.AddJobRange(b["job"].EnumerateArray().Select(x => x.GetString()));
}
toReturn.Add(toAdd);
}
return(toReturn);
}
public void Equals_SameBanDifferentJobOrder_ReturnTrue()
{
var banA = new Ban()
{
Id = 12,
BanType = BanType.Job
};
banA.AddJobRange(new[] { "detective", "head of security", "security officer", "warden" });
var banB = new Ban()
{
Id = 0,
BanType = BanType.Job
};
banB.AddJobRange(new[] { "head of security", "warden", "detective", "security officer" });
var comparer = BanEqualityComparer.Instance;
Assert.True(comparer.Equals(banA, banB), "Two bans with the same jobbans in different orders should be equal");
Assert.True(comparer.GetHashCode(banA) == comparer.GetHashCode(banB), "Two bans with the same jobbans in different orders should be equal");
}
// TODO: cleanup
public async Task <IEnumerable <Ban> > GetBansAsync()
{
var toReturn = new List <Ban>();
var config = AngleSharp.Configuration.Default.WithDefaultLoader();
var context = BrowsingContext.New(config);
var document = await context.OpenAsync("https://ss13.moe/index.php/bans");
if (document.StatusCode != HttpStatusCode.OK)
{
_logger.LogError(
$"Source website returned a non-200 HTTP response code. Url: \"{document.Url}\", code: {document.StatusCode}");
throw new BanSourceUnavailableException(
$"Source website returned a non-200 HTTP response code. Url: \"{document.Url}\", code: {document.StatusCode}", document.TextContent);
}
var tables = document.QuerySelectorAll("form > table > tbody");
var banTable = tables[0];
var jobTable = tables[1];
for (var i = 1; i < banTable.Children.Length; i++)
{
var cursor = banTable.Children[i];
var ckey = cursor.Children[0].Children[0].TextContent.Trim();
DateTimeOffset date = DateTime.SpecifyKind(
cursor.Children[0].Children[0].GetAttribute("title") == "0000-00-00 00:00:00"
? DateTime.MinValue
: DateTime.Parse(cursor.Children[0].Children[0].GetAttribute("title").Trim()),
DateTimeKind.Utc);
var reason = cursor.Children[1].TextContent.Trim();
var bannedBy = cursor.Children[2].TextContent.Trim();
var expiresText = cursor.Children[3].TextContent.Trim();
DateTimeOffset?expires = null;
if (DateTime.TryParse(expiresText, out var d))
{
expires = DateTime.SpecifyKind(d, DateTimeKind.Utc);
}
toReturn.Add(new Ban()
{
CKey = ckey,
BannedOn = date.UtcDateTime,
BannedBy = bannedBy,
Reason = reason,
Expires = expires.HasValue ? expires.Value.UtcDateTime : (DateTime?)null,
BanType = BanType.Server,
SourceNavigation = BanSource
});
}
for (var i = 1; i < jobTable.Children.Length; i++)
{
var cursor = jobTable.Children[i];
var bannedDetails = cursor.Children[0];
var ckey = bannedDetails.Children[0].TextContent.Trim();
DateTimeOffset date = DateTime.SpecifyKind(
cursor.Children[0].Children[0].GetAttribute("title") == "0000-00-00 00:00:00"
? DateTime.MinValue
: DateTime.Parse(cursor.Children[0].Children[0].GetAttribute("title").Trim()),
DateTimeKind.Utc);
var jobDetails = cursor.QuerySelector(".clmJobs");
var jobs = jobDetails.QuerySelectorAll("a").Select(x => x.TextContent.Trim()).Distinct();
var reason = cursor.Children[2].TextContent.Trim();
var bannedBy = cursor.Children[3].TextContent.Trim();
var expiresText = cursor.Children[4].TextContent.Trim();
DateTimeOffset?expires = null;
if (DateTime.TryParse(expiresText, out var d))
{
expires = DateTime.SpecifyKind(d, DateTimeKind.Utc);
}
var toAdd = new Ban()
{
CKey = ckey,
BanType = BanType.Job,
Reason = reason,
BannedBy = bannedBy,
BannedOn = date.UtcDateTime,
Expires = expires?.UtcDateTime,
SourceNavigation = BanSource
};
toAdd.AddJobRange(jobs);
toReturn.Add(toAdd);
}
return(toReturn);
}
/// <summary>
/// Attempts to fetch and process bans from the source for the ban parser.
/// </summary>
/// <param name="context">The job execution context provided by Quartz' scheduler</param>
/// <returns>A task for the asynchronous work</returns>
public virtual async Task ParseBans(IJobExecutionContext context)
{
_logger.LogInformation($"Beginning ban parsing");
// Get stored bans from the database
IEnumerable <Ban> storedBans = null;
try
{
storedBans = await _dbContext.Bans
.Where(x => Sources.Keys.Contains(x.SourceNavigation.Name))
.Include(x => x.JobBans)
.Include(x => x.SourceNavigation)
.ToListAsync();
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to get stored ban data from database, encountered exception");
throw new JobExecutionException(ex, false);
}
// Get bans from the source
var isCompleteRefresh = context.MergedJobDataMap.GetBoolean("completeRefresh") || storedBans.Count() == 0;
IEnumerable <Ban> bans = null;
try
{
bans = await(isCompleteRefresh ? FetchAllBansAsync() : FetchNewBansAsync());
}
catch (BanSourceUnavailableException)
{
return;
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to get ban data from source, encountered exception during fetch");
throw new JobExecutionException(ex, false);
}
// Assign proper sources
bans = await AssignBanSources(bans);
// Check for ban updates
var inserted = 0;
var updated = 0;
foreach (var b in bans)
{
// Enssure the CKey is actually canonical
b.MakeKeysCanonical();
// Attempt to find matching bans in the database
Ban matchedBan = null;
if (SourceSupportsBanIDs)
{
matchedBan = storedBans.FirstOrDefault(x =>
b.Source == x.Source &&
b.BanID == x.BanID);
}
else
{
matchedBan = storedBans.FirstOrDefault(x =>
b.Source == x.Source &&
b.BannedOn == x.BannedOn &&
b.BanType == x.BanType &&
b.CKey == x.CKey &&
b.BannedBy == x.BannedBy &&
(b.BanType == BanType.Server ||
b.JobBans.SetEquals(x.JobBans)));
}
// Update ban if an existing one is found
if (matchedBan != null)
{
// Check for a difference in date time, unbans, or reason
if (matchedBan.Reason != b.Reason || matchedBan.Expires != b.Expires || matchedBan.UnbannedBy != b.UnbannedBy)
{
matchedBan.Reason = b.Reason;
matchedBan.Expires = b.Expires;
matchedBan.UnbannedBy = b.UnbannedBy;
updated++;
}
// Check for a difference in recorded jobbans
if (b.BanType == BanType.Job && !b.JobBans.SetEquals(matchedBan.JobBans))
{
matchedBan.JobBans = new HashSet <JobBan>(JobBanEqualityComparer.Instance);
matchedBan.AddJobRange(b.JobBans.Select(x => x.Job));
updated++;
}
}
// Otherwise add insert a new ban
else
{
inserted++;
_dbContext.Bans.Add(b);
}
}
// Insert new changes
_logger.LogInformation($"Inserting {inserted} new bans, updating {updated} modified bans...");
await _dbContext.SaveChangesAsync();
// Delete any missing bans if we're doing a complete refresh
if (isCompleteRefresh)
{
var bansHashed = new HashSet <Ban>(bans, BanEqualityComparer.Instance);
var missingBans = storedBans.Except(bansHashed, BanEqualityComparer.Instance).ToList();
if (bansHashed.Count == 0 && missingBans.Count > 1)
{
throw new Exception("Failed to find any bans for source, aborting removal phase of ban " +
"parsing to avoid dumping entire set of bans");
}
// Apply deletions
_logger.LogInformation(missingBans.Count > 0 ? $"Removing {missingBans.Count} deleted bans..."
: "Found no deleted bans to remove");
if (missingBans.Count > 0)
{
_dbContext.RemoveRange(missingBans);
await _dbContext.SaveChangesAsync();
}
}
_logger.LogInformation("Completed ban parsing.");
}
