public async Task <ActionResult <QueueStat> > GetJobQueueStat(
[FromServices] RurikawaDb db,
[FromServices] RedisService redis,
[FromServices] JsonSerializerOptions jsonSerializerOptions
)
{
var red = await redis.GetDatabase();
var judgerStat = await red.StringGetAsync(QUEUE_STAT_CACHE_KEY);
if (!judgerStat.IsNullOrEmpty)
{
return(new ContentResult()
{
Content = (string)judgerStat,
StatusCode = 200,
ContentType = "application/json"
});
}
// TODO: Use redis to track jobs count?
var jobCount = await JudgerCoordinatorService.QueuedCriteria(db.Jobs).CountAsync();
var stat = new QueueStat {
QueuedJobs = jobCount
};
await red.StringSetAsync(
QUEUE_STAT_CACHE_KEY,
JsonSerializer.Serialize(stat, jsonSerializerOptions),
expiry : TimeSpan.FromSeconds(10));
return(stat);
}
protected async Task <Job?> GetLastUndispatchedJobFromDatabase(RurikawaDb db)
{
var res = await db.Jobs.Where(j => j.Stage == JobStage.Queued)
.OrderBy(j => j.Id)
.FirstOrDefaultAsync();
return(res);
}
protected async Task <Job?> GetLastUndispatchedJobFromDatabase(RurikawaDb db)
{
var res = await QueuedCriteria(db.Jobs)
.OrderBy(j => j.Id)
.FirstOrDefaultAsync();
return(res);
}
public async Task <IActionResult> DownloadSuite(
[FromRoute] FlowSnake suite,
[FromServices] RurikawaDb db)
{
var test_suite = await db.TestSuites.SingleOrDefaultAsync(s => s.Id == suite);
return(Redirect($"/api/v1/file/{test_suite.PackageFileId}"));
}
protected async Task <List <Job> > GetUndispatchedJobsFromDatabase(RurikawaDb db, int count)
{
var res = await QueuedCriteria(db.Jobs)
.OrderBy(j => j.Id)
.Take(count)
.ToListAsync();
return(res);
}
public AccountController(
ILogger <JudgerApiController> logger,
RurikawaDb db,
AccountService accountService
)
{
this.logger = logger;
this.db = db;
this.accountService = accountService;
}
public DashboardController(
ILogger <JudgerApiController> logger,
RurikawaDb db,
SingleBucketFileStorageService fs,
JsonSerializerOptions?jsonOptions)
{
this.logger = logger;
this.db = db;
this.jsonOptions = jsonOptions;
}
public AccountService(
RurikawaDb db,
AuthInfo authInfo,
JsonSerializerOptions jsonSerializerOptions,
ILogger <AccountService> logger)
{
this.db = db;
this.authInfo = authInfo;
this.jsonSerializerOptions = jsonSerializerOptions;
this.logger = logger;
}
public TestApiController(
ILogger <JudgerApiController> logger,
RurikawaDb db,
DbService dbService,
SingleBucketFileStorageService fs,
JsonSerializerOptions?jsonOptions)
{
this.logger = logger;
this.db = db;
this.dbService = dbService;
this.fs = fs;
this.jsonOptions = jsonOptions;
}
public async Task <JudgerStat> GetJudgerStat(
[FromServices] JudgerCoordinatorService coordinatorService,
[FromServices] RurikawaDb db)
{
var judgerCount = await db.Judgers.CountAsync();
var(connectedCount, runningCount) = await coordinatorService.GetConnectedJudgerInfo();
return(new JudgerStat
{
Count = judgerCount,
Connected = connectedCount,
Running = runningCount
});
}
public async Task <ActionResult> EstimateDumpSuiteJobDumpCount(
[FromRoute] FlowSnake suiteId,
[FromServices] RurikawaDb db)
{
var res = await db.Jobs.FromSqlInterpolated($@"
select
distinct on (account)
*
from jobs
order by account, id desc
").Where((job) => job.TestSuite == suiteId)
.CountAsync();
return(Ok(res));
}
public async Task <ActionResult> DumpProfiles([FromServices] RurikawaDb db)
{
var ptr = db.Profiles.AsAsyncEnumerable();
Response.ContentType = "application/csv";
Response.StatusCode = 200;
await Response.StartAsync();
int flushInterval = 50;
await Task.Run(async() => {
// write to body of response
using var sw = new StreamWriter(new StreamAsyncAdaptor(Response.Body));
await using var swGuard = sw.ConfigureAwait(false);
var csvWriter = new CsvWriter(sw);
csvWriter.QuoteAllFields = true;
csvWriter.WriteField("username");
csvWriter.WriteField("studentId");
csvWriter.WriteField("email");
csvWriter.NextRecord();
int counter = 0;
await foreach (var val in ptr)
{
if (counter % flushInterval == 0)
{
await sw.FlushAsync();
}
csvWriter.WriteField(val.Username);
csvWriter.WriteField(val.StudentId);
csvWriter.WriteField(val.Email);
csvWriter.NextRecord();
counter++;
}
await sw.FlushAsync();
});
return(new EmptyResult());
}
public async Task <ActionResult <JudgerStat> > GetJudgerStat(
[FromServices] JudgerCoordinatorService coordinatorService,
[FromServices] RurikawaDb db,
[FromServices] RedisService redis,
[FromServices] JsonSerializerOptions jsonSerializerOptions)
{
var red = await redis.GetDatabase();
var judgerStat = await red.StringGetAsync(JUDGER_STAT_CACHE_KEY);
if (!judgerStat.IsNullOrEmpty)
{
return(new ContentResult()
{
Content = (string)judgerStat,
StatusCode = 200,
ContentType = "application/json"
});
}
var judgerCount = await db.Judgers.CountAsync();
var(connectedCount, runningCount) = await coordinatorService.GetConnectedJudgerInfo();
var stat = new JudgerStat {
Count = judgerCount,
Connected = connectedCount,
Running = runningCount
};
await red.StringSetAsync(
JUDGER_STAT_CACHE_KEY,
JsonSerializer.Serialize(stat, jsonSerializerOptions),
expiry : TimeSpan.FromSeconds(10));
return(stat);
}
public async Task <ActionResult> DumpSuiteJobs(
[FromRoute] FlowSnake suiteId,
[FromServices] RurikawaDb db)
{
var suite = await dbService.GetTestSuite(suiteId);
if (suite == null)
{
return(NotFound());
}
var columns = suite.TestGroups
.SelectMany(group => group.Value.Select(value => value.Name))
.ToList();
var ptr = db.Jobs.FromSqlInterpolated($@"
select
distinct on (account)
*
from jobs
where test_suite = {suiteId.Num}
order by account, id desc
").AsAsyncEnumerable();
Response.StatusCode = 200;
Response.ContentType = "application/csv";
const int flushInterval = 100;
// write to body of response
var sw = new StreamWriter(Response.Body);
var csvWriter = new CsvWriter(sw);
csvWriter.QuoteAllFields = true;
csvWriter.WriteField("id");
csvWriter.WriteField("account");
csvWriter.WriteField("repo");
csvWriter.WriteField("revision");
csvWriter.WriteField("stage");
csvWriter.WriteField("result_kind");
foreach (var col in columns)
{
csvWriter.WriteField(col);
}
csvWriter.NextRecord();
int counter = 0;
await foreach (var val in ptr)
{
if (counter % flushInterval == 0)
{
await sw.FlushAsync();
}
WriteJobInfo(csvWriter, val, columns);
counter++;
}
await sw.FlushAsync();
return(new EmptyResult());
}
public ProfileService(RurikawaDb db)
{
this.db = db;
}
public DbService(RurikawaDb db)
{
this.db = db;
}
public JudgerService(ILogger <JudgerService> logger, AccountService accountService, RurikawaDb db)
{
this.logger = logger;
this.accountService = accountService;
this.db = db;
}
public async Task <ActionResult> DumpSuiteAllJobs(
[FromRoute] FlowSnake suiteId,
[FromServices] RurikawaDb db)
{
var suite = await dbService.GetTestSuite(suiteId);
if (suite == null)
{
return(NotFound());
}
var columns = suite.TestGroups
.SelectMany(group => group.Value.Select(value => value.Name))
.ToList();
Response.ContentType = "application/csv";
Response.StatusCode = 200;
Response.Headers.Add("Content-Disposition", $"inline; filename=\"{suiteId}.all-jobs.csv\"");
await Response.StartAsync();
// write to body of response
using var sw = new StreamWriter(new StreamAsyncAdaptor(Response.Body));
await using var swGuard = sw.ConfigureAwait(false);
var csvWriter = new CsvWriter(sw);
const int batchSize = 100;
var startId = FlowSnake.MaxValue;
await WriteJobHeaders(csvWriter, columns);
while (true)
{
var batch = db.Jobs
.OrderByDescending(job => job.Id)
.Where((job) => job.TestSuite == suiteId && job.Id < startId)
.Take(batchSize)
.Join(
db.Profiles,
(job) => job.Account,
(profile) => profile.Username,
(job, profile) =>
new JobDumpEntry {
Job = job, StudentId = profile.StudentId
})
.ToList();
if (batch.Count == 0)
{
break;
}
startId = batch.Last().Job.Id;
await Task.Run(() => {
foreach (var val in batch)
{
WriteJobInfo(csvWriter, val, columns);
}
});
await sw.FlushAsync();
}
return(new EmptyResult());
}
