public void PublishGovMetadata(Record record) { if (record.Publication.Target?.Gov?.Publishable == true) { var attempt = new PublicationAttempt { DateUtc = Clock.NowUtc }; try { Logger.Info("Starting publish to data.gov.uk"); uploadRecordService.UpdateGovPublishAttempt(record, attempt); db.SaveChanges(); var redactedRecord = recordRedactor.RedactRecord(record); // this isn't saved back to the db govService.UploadGeminiXml(Helpers.RemoveCollectionFromId(redactedRecord)); govService.UpdateDguIndex(Helpers.RemoveCollectionFromId(redactedRecord)); uploadRecordService.UpdateGovPublishSuccess(record, attempt); Logger.Info("Publish to data.gov.uk completed successfully"); } catch (Exception ex) { attempt.Message = ex.Message + (ex.InnerException != null ? ex.InnerException.Message : ""); Logger.Error($"DGU metadata transfer failed for record with GUID={record.Id}", ex); throw; } } else { Logger.Info("Data.gov.uk not defined as a target publishing destination"); } }
private void UploadRecord(Record record) { var attempt = new PublicationAttempt { DateUtc = Clock.NowUtc }; uploadRecordService.UpdateLastAttempt(record, attempt); db.SaveChanges(); bool alternativeResources = record.Publication != null && record.Publication.OpenData != null && record.Publication.OpenData.Resources != null && record.Publication.OpenData.Resources.Any(); bool corpulent = record.Gemini.Keywords.Any(k => k.Vocab == "http://vocab.jncc.gov.uk/metadata-admin" && k.Value == "Corpulent"); try { if (alternativeResources) { // upload the alternative resources; don't touch the resource locator uploadHelper.UploadAlternativeResources(record); } else { if (corpulent) { // set the resource locator to the download request page; don't upload any resources if (record.Gemini.ResourceLocator.IsBlank()) // arguably should always do it actually { uploadRecordService.UpdateTheResourceLocatorToBeTheOpenDataDownloadPage(record); } } else if (record.Gemini.ResourceLocator.IsBlank() || record.Gemini.ResourceLocator.Contains(uploadHelper.GetHttpRootUrl())) { // "normal" case - if the resource locator is blank or already data.jncc.gov.uk // upload the resource pointed at by record.Path, and update the resource locator to match uploadHelper.UploadDataFile(record.Id, record.Path); string dataHttpPath = uploadHelper.GetHttpRootUrl() + "/" + GetUnrootedDataPath(record.Id, record.Path); uploadRecordService.UpdateResourceLocatorToMatchMainDataFile(record, dataHttpPath); } else { // do nothing - don't change the resource locator, don't upload anything Logger.Info("Deferring to existing resource locator - not uploading."); } } uploadHelper.UploadMetadataDocument(record); uploadHelper.UploadWafIndexDocument(record); uploadRecordService.UpdateLastSuccess(record, attempt); } catch (WebException ex) { string message = ex.Message + (ex.InnerException != null ? ex.InnerException.Message : ""); attempt.Message = message; Logger.Error("Upload failed for record with GUID=" + record.Id, ex); } // commit the changes - to both the record (resource locator may have changed) and the attempt object db.SaveChanges(); }
public void UpdateGovPublishSuccess(Record record, PublicationAttempt attempt) { record.Publication.Target.Gov.LastSuccess = attempt; UpdateMetadataDate(record, attempt.DateUtc); var recordServiceResult = Upsert(record, db, validator); if (!recordServiceResult.Success) { var e = new Exception("Error while saving upload changes."); e.LogAndThrow(Logger); } }
private void PublishDataFiles(Record record) { var attempt = new PublicationAttempt { DateUtc = Clock.NowUtc }; try { uploadRecordService.UpdateDataPublishAttempt(record, attempt); db.SaveChanges(); var recordId = Helpers.RemoveCollection(record.Id); dataService.CreateDataRollback(recordId); var resources = record.Resources.Copy(); // don't want to save if any of them fail if (resources != null) { Logger.Info($"Found {record.Resources.Count} publishable resources"); foreach (var resource in resources) { if (Helpers.IsFileResource(resource)) { Logger.Info($"Resource {resource.Path} is a file - starting upload process"); dataService.UploadDataFile(recordId, resource.Path); string dataHttpPath = env.HTTP_ROOT_URL + GetUnrootedDataPath(recordId, resource.Path); resource.PublishedUrl = dataHttpPath; } else { Logger.Info($"Resource {resource.Path} is a URL - no file to upload"); resource.PublishedUrl = null; // make sure this is "clean" if no data file was uploaded } } } uploadRecordService.UpdateDataPublishSuccess(record, resources, attempt); } catch (Exception ex) { attempt.Message = ex.Message + (ex.InnerException != null ? ex.InnerException.Message : ""); Logger.Error($"Data transfer failed for record with GUID={record.Id}", ex); throw; } //todo: save changes here in finally }
public void UpdateHubPublishAttempt(Record record, PublicationAttempt attempt) { if (record.Publication.Target.Hub == null) { record.Publication.Target.Hub = new HubPublicationInfo(); } record.Publication.Target.Hub.LastAttempt = attempt; UpdateMetadataDate(record, attempt.DateUtc); var recordServiceResult = Upsert(record, db, validator); if (!recordServiceResult.Success) { var e = new Exception("Error while saving upload changes."); e.LogAndThrow(Logger); } }
public void PublishHubMetadata(Record record) { if (record.Publication.Target?.Hub?.Publishable == true) { var attempt = new PublicationAttempt { DateUtc = Clock.NowUtc }; Record redactedRecord; try { Logger.Info("Starting publish to Resource hub"); uploadRecordService.UpdateHubPublishAttempt(record, attempt); db.SaveChanges(); redactedRecord = recordRedactor.RedactRecord(record); // this isn't saved back to the db hubService.Publish(redactedRecord); var url = env.HUB_ASSETS_BASE_URL + Helpers.RemoveCollection(record.Id); uploadRecordService.UpdateHubPublishSuccess(record, url, attempt); // successfully published to the hub at this stage redactedRecord.Publication.Target.Hub.Url = url; Logger.Info("Publish to Resource hub completed successfully"); } catch (Exception ex) { attempt.Message = ex.Message + (ex.InnerException != null ? ex.InnerException.Message : ""); Logger.Error($"Could not save record to Resource hub database, GUID={record.Id}", ex); throw; } } else { Logger.Info("Resource hub not defined as a target publishing destination"); } }