public async void ItWrapsTheSchemaInSourceMetadata() { var config = new WorkerConfiguration { PublishUrl = "http://datadock.io/" }; var proc = new ImportSchemaProcessor(config, _mockSchemeStore.Object, _mockFileStore.Object); WithJsonSchema(@"{ '@context': 'http://www.w3.org/ns/csvw' }"); #pragma warning disable 618 var job = new JobInfo #pragma warning restore 618 { JobType = JobType.SchemaCreate, UserId = "kal", OwnerId = "datadock", RepositoryId = "test", SchemaFileId = "schemaFileId" }; await proc.ProcessJob(job, new UserAccount(), _mockProgressLog.Object); _mockSchemeStore.Verify(s => s.CreateOrUpdateSchemaRecordAsync(It.Is <SchemaInfo>(p => p.OwnerId.Equals("datadock") && p.RepositoryId.Equals("test") && p.Schema.ContainsKey("@context") && (p.Schema["@context"] as JValue).Value <string>().Equals("http://www.w3.org/ns/csvw")))); }
public async Task ItMakesUrlPropertiesRelative() { var config = new WorkerConfiguration { PublishUrl = "http://datadock.io/" }; var proc = new ImportSchemaProcessor(config, _mockSchemeStore.Object, _mockFileStore.Object); WithJsonSchema(@"{ '@context': 'http://www.w3.org/ns/csvw', 'url': 'http://datadock.io/datadock/test/id/dataset/mydataset', 'dc:title': 'http://datadock.io/datadock/test/foo', 'dc:license': 'https://creativecommons.org/publicdomain/zero/1.0/', 'aboutUrl': 'http://datadock.io/datadock/test/id/resource/mydataset/monument_record_no/{monument_record_no}', 'tableSchema': { 'columns': [ { 'name' : 'foo', 'propertyUrl': 'http://datadock.io/datadock/test/id/definition/foo', 'valueUrl' : 'http://datadock.io/datadock/test/id/bar/{bar}' } ] } }"); #pragma warning disable 618 var job = new JobInfo #pragma warning restore 618 { JobType = JobType.SchemaCreate, UserId = "kal", OwnerId = "datadock", RepositoryId = "test", SchemaFileId = "schemaFileId" }; await proc.ProcessJob(job, new UserAccount(), _mockProgressLog.Object); _mockSchemeStore.Verify(s => s.CreateOrUpdateSchemaRecordAsync(It.Is <SchemaInfo>(p => (p.Schema["aboutUrl"] as JValue).Value <string>().Equals("id/resource/mydataset/monument_record_no/{monument_record_no}")))); _mockSchemeStore.Verify(s => s.CreateOrUpdateSchemaRecordAsync(It.Is <SchemaInfo>(p => (p.Schema["dc:title"] as JValue).Value <string>().Equals("http://datadock.io/datadock/test/foo")))); _mockSchemeStore.Verify(s => s.CreateOrUpdateSchemaRecordAsync(It.Is <SchemaInfo>(p => (p.Schema["tableSchema"]["columns"][0]["propertyUrl"] as JValue).Value <string>() .Equals("id/definition/foo")))); _mockSchemeStore.Verify(s => s.CreateOrUpdateSchemaRecordAsync(It.Is <SchemaInfo>(p => (p.Schema["tableSchema"]["columns"][0]["valueUrl"] as JValue).Value <string>() .Equals("id/bar/{bar}")))); }
private async Task ProcessJob(IJobStore jobStore, JobInfo jobInfo) { var progressLogFactory = Services.GetRequiredService <IProgressLogFactory>(); var progressLog = await progressLogFactory.MakeProgressLogForJobAsync(jobInfo); var logStore = Services.GetService <ILogStore>(); try { var jobLogger = Log.ForContext("JobId", jobInfo.JobId); jobLogger.Information("Processing Job {JobId} - {JobType}", jobInfo.JobId, jobInfo.JobType); jobLogger.Debug("Retrieving user account for {UserId}", jobInfo.UserId); var userRepo = Services.GetRequiredService <IUserStore>(); var userAccount = await userRepo.GetUserAccountAsync(jobInfo.UserId); // TODO: Should encapsulate this logic plus basic job info validation into its own processor factory class (issue #83) jobLogger.Debug("Creating job processor for job type {JobType}", jobInfo.JobType); IDataDockProcessor processor; switch (jobInfo.JobType) { case JobType.Import: { var cmdProcessorFactory = Services.GetRequiredService <IGitCommandProcessorFactory>(); processor = new ImportJobProcessor( Services.GetRequiredService <WorkerConfiguration>(), cmdProcessorFactory.MakeGitCommandProcessor(progressLog), Services.GetRequiredService <IGitHubClientFactory>(), Services.GetRequiredService <IDatasetStore>(), Services.GetRequiredService <IFileStore>(), Services.GetRequiredService <IOwnerSettingsStore>(), Services.GetRequiredService <IRepoSettingsStore>(), Services.GetRequiredService <IDataDockRepositoryFactory>(), Services.GetRequiredService <IDataDockUriService>()); break; } case JobType.Delete: { var cmdProcessorFactory = Services.GetRequiredService <IGitCommandProcessorFactory>(); processor = new DeleteDatasetProcessor( Services.GetRequiredService <WorkerConfiguration>(), cmdProcessorFactory.MakeGitCommandProcessor(progressLog), Services.GetRequiredService <IGitHubClientFactory>(), Services.GetRequiredService <IOwnerSettingsStore>(), Services.GetRequiredService <IRepoSettingsStore>(), Services.GetRequiredService <IDatasetStore>(), Services.GetRequiredService <IDataDockRepositoryFactory>()); break; } case JobType.SchemaCreate: processor = new ImportSchemaProcessor( Services.GetRequiredService <WorkerConfiguration>(), Services.GetRequiredService <ISchemaStore>(), Services.GetRequiredService <IFileStore>()); break; case JobType.SchemaDelete: processor = new DeleteSchemaProcessor(Services.GetRequiredService <ISchemaStore>()); break; default: throw new WorkerException($"Could not process job of type {jobInfo.JobType}"); } // Log start jobLogger.Debug("Start job processor"); jobInfo.StartedAt = DateTime.UtcNow; jobInfo.CurrentStatus = JobStatus.Running; await jobStore.UpdateJobInfoAsync(jobInfo); progressLog.UpdateStatus(JobStatus.Running, "Job processing started"); await processor.ProcessJob(jobInfo, userAccount, progressLog); // Log end jobLogger.Information("Job processing completed"); var logId = await logStore.AddLogAsync(jobInfo.OwnerId, jobInfo.RepositoryId, jobInfo.JobId, progressLog.GetLogText()); jobInfo.CurrentStatus = JobStatus.Completed; jobInfo.CompletedAt = DateTime.UtcNow; jobInfo.LogId = logId; await jobStore.UpdateJobInfoAsync(jobInfo); progressLog.UpdateStatus(jobInfo.CurrentStatus, "Job completed"); } catch (Exception ex) { if (ex is WorkerException wex) { progressLog.UpdateStatus(JobStatus.Failed, wex.Message); Log.Error(wex, "WorkerException raised for job {JobId}", jobInfo.JobId); } else { Log.Error(ex, "Job processing failed for job {JobId}", jobInfo.JobId); } var logId = await logStore.AddLogAsync(jobInfo.OwnerId, jobInfo.RepositoryId, jobInfo.JobId, progressLog.GetLogText()); jobInfo.LogId = logId; jobInfo.CurrentStatus = JobStatus.Failed; jobInfo.CompletedAt = DateTime.UtcNow; await jobStore.UpdateJobInfoAsync(jobInfo); progressLog.UpdateStatus(JobStatus.Failed, "Job processing failed"); } }