public async void ItWrapsTheSchemaInSourceMetadata()
{
var config = new WorkerConfiguration {
PublishUrl = "http://datadock.io/"
};
var proc = new ImportSchemaProcessor(config, _mockSchemeStore.Object, _mockFileStore.Object);
WithJsonSchema(@"{ '@context': 'http://www.w3.org/ns/csvw' }");
#pragma warning disable 618
var job = new JobInfo
#pragma warning restore 618
{
JobType = JobType.SchemaCreate,
UserId = "kal",
OwnerId = "datadock",
RepositoryId = "test",
SchemaFileId = "schemaFileId"
};
await proc.ProcessJob(job, new UserAccount(), _mockProgressLog.Object);
_mockSchemeStore.Verify(s => s.CreateOrUpdateSchemaRecordAsync(It.Is <SchemaInfo>(p =>
p.OwnerId.Equals("datadock") &&
p.RepositoryId.Equals("test") &&
p.Schema.ContainsKey("@context") &&
(p.Schema["@context"] as JValue).Value <string>().Equals("http://www.w3.org/ns/csvw"))));
}
public async Task ItMakesUrlPropertiesRelative()
{
var config = new WorkerConfiguration {
PublishUrl = "http://datadock.io/"
};
var proc = new ImportSchemaProcessor(config, _mockSchemeStore.Object, _mockFileStore.Object);
WithJsonSchema(@"{
'@context': 'http://www.w3.org/ns/csvw',
'url': 'http://datadock.io/datadock/test/id/dataset/mydataset',
'dc:title': 'http://datadock.io/datadock/test/foo',
'dc:license': 'https://creativecommons.org/publicdomain/zero/1.0/',
'aboutUrl': 'http://datadock.io/datadock/test/id/resource/mydataset/monument_record_no/{monument_record_no}',
'tableSchema': {
'columns': [
{
'name' : 'foo',
'propertyUrl': 'http://datadock.io/datadock/test/id/definition/foo',
'valueUrl' : 'http://datadock.io/datadock/test/id/bar/{bar}'
}
]
}
}");
#pragma warning disable 618
var job = new JobInfo
#pragma warning restore 618
{
JobType = JobType.SchemaCreate, UserId = "kal", OwnerId = "datadock", RepositoryId = "test",
SchemaFileId = "schemaFileId"
};
await proc.ProcessJob(job, new UserAccount(), _mockProgressLog.Object);
_mockSchemeStore.Verify(s => s.CreateOrUpdateSchemaRecordAsync(It.Is <SchemaInfo>(p =>
(p.Schema["aboutUrl"] as JValue).Value <string>().Equals("id/resource/mydataset/monument_record_no/{monument_record_no}"))));
_mockSchemeStore.Verify(s => s.CreateOrUpdateSchemaRecordAsync(It.Is <SchemaInfo>(p =>
(p.Schema["dc:title"] as JValue).Value <string>().Equals("http://datadock.io/datadock/test/foo"))));
_mockSchemeStore.Verify(s => s.CreateOrUpdateSchemaRecordAsync(It.Is <SchemaInfo>(p =>
(p.Schema["tableSchema"]["columns"][0]["propertyUrl"] as JValue).Value <string>()
.Equals("id/definition/foo"))));
_mockSchemeStore.Verify(s => s.CreateOrUpdateSchemaRecordAsync(It.Is <SchemaInfo>(p =>
(p.Schema["tableSchema"]["columns"][0]["valueUrl"] as JValue).Value <string>()
.Equals("id/bar/{bar}"))));
}
private async Task ProcessJob(IJobStore jobStore, JobInfo jobInfo)
{
var progressLogFactory = Services.GetRequiredService <IProgressLogFactory>();
var progressLog = await progressLogFactory.MakeProgressLogForJobAsync(jobInfo);
var logStore = Services.GetService <ILogStore>();
try
{
var jobLogger = Log.ForContext("JobId", jobInfo.JobId);
jobLogger.Information("Processing Job {JobId} - {JobType}", jobInfo.JobId, jobInfo.JobType);
jobLogger.Debug("Retrieving user account for {UserId}", jobInfo.UserId);
var userRepo = Services.GetRequiredService <IUserStore>();
var userAccount = await userRepo.GetUserAccountAsync(jobInfo.UserId);
// TODO: Should encapsulate this logic plus basic job info validation into its own processor factory class (issue #83)
jobLogger.Debug("Creating job processor for job type {JobType}", jobInfo.JobType);
IDataDockProcessor processor;
switch (jobInfo.JobType)
{
case JobType.Import:
{
var cmdProcessorFactory = Services.GetRequiredService <IGitCommandProcessorFactory>();
processor = new ImportJobProcessor(
Services.GetRequiredService <WorkerConfiguration>(),
cmdProcessorFactory.MakeGitCommandProcessor(progressLog),
Services.GetRequiredService <IGitHubClientFactory>(),
Services.GetRequiredService <IDatasetStore>(),
Services.GetRequiredService <IFileStore>(),
Services.GetRequiredService <IOwnerSettingsStore>(),
Services.GetRequiredService <IRepoSettingsStore>(),
Services.GetRequiredService <IDataDockRepositoryFactory>(),
Services.GetRequiredService <IDataDockUriService>());
break;
}
case JobType.Delete:
{
var cmdProcessorFactory = Services.GetRequiredService <IGitCommandProcessorFactory>();
processor = new DeleteDatasetProcessor(
Services.GetRequiredService <WorkerConfiguration>(),
cmdProcessorFactory.MakeGitCommandProcessor(progressLog),
Services.GetRequiredService <IGitHubClientFactory>(),
Services.GetRequiredService <IOwnerSettingsStore>(),
Services.GetRequiredService <IRepoSettingsStore>(),
Services.GetRequiredService <IDatasetStore>(),
Services.GetRequiredService <IDataDockRepositoryFactory>());
break;
}
case JobType.SchemaCreate:
processor = new ImportSchemaProcessor(
Services.GetRequiredService <WorkerConfiguration>(),
Services.GetRequiredService <ISchemaStore>(),
Services.GetRequiredService <IFileStore>());
break;
case JobType.SchemaDelete:
processor = new DeleteSchemaProcessor(Services.GetRequiredService <ISchemaStore>());
break;
default:
throw new WorkerException($"Could not process job of type {jobInfo.JobType}");
}
// Log start
jobLogger.Debug("Start job processor");
jobInfo.StartedAt = DateTime.UtcNow;
jobInfo.CurrentStatus = JobStatus.Running;
await jobStore.UpdateJobInfoAsync(jobInfo);
progressLog.UpdateStatus(JobStatus.Running, "Job processing started");
await processor.ProcessJob(jobInfo, userAccount, progressLog);
// Log end
jobLogger.Information("Job processing completed");
var logId = await logStore.AddLogAsync(jobInfo.OwnerId, jobInfo.RepositoryId, jobInfo.JobId,
progressLog.GetLogText());
jobInfo.CurrentStatus = JobStatus.Completed;
jobInfo.CompletedAt = DateTime.UtcNow;
jobInfo.LogId = logId;
await jobStore.UpdateJobInfoAsync(jobInfo);
progressLog.UpdateStatus(jobInfo.CurrentStatus, "Job completed");
}
catch (Exception ex)
{
if (ex is WorkerException wex)
{
progressLog.UpdateStatus(JobStatus.Failed, wex.Message);
Log.Error(wex, "WorkerException raised for job {JobId}", jobInfo.JobId);
}
else
{
Log.Error(ex, "Job processing failed for job {JobId}", jobInfo.JobId);
}
var logId = await logStore.AddLogAsync(jobInfo.OwnerId, jobInfo.RepositoryId, jobInfo.JobId,
progressLog.GetLogText());
jobInfo.LogId = logId;
jobInfo.CurrentStatus = JobStatus.Failed;
jobInfo.CompletedAt = DateTime.UtcNow;
await jobStore.UpdateJobInfoAsync(jobInfo);
progressLog.UpdateStatus(JobStatus.Failed, "Job processing failed");
}
}
