public ValidationResult<Record> Validate(Record record) { var result = new ValidationResult<Record>(); ValidatePath(record, result); ValidateTitle(record, result); ValidateKeywords(record, result); ValidateDatasetReferenceDate(record, result); ValidateTemporalExtent(record, result); ValidateTopicCategory(record, result); ValidateResourceLocator(record, result); ValidateResponsibleOrganisation(record, result); ValidateMetadataPointOfContact(record, result); ValidateResourceType(record, result); ValidateSecurityInvariants(record, result); ValidatePublishableInvariants(record, result); ValidateBoundingBox(record, result); ValidateJnccSpecificRules(record, result); if (record.Validation == Validation.Gemini) { PerformGeminiValidation(record, result); } return result; }
public RecordServiceResult Update(Record record) { if (record.ReadOnly) throw new InvalidOperationException("Cannot update a read-only record."); return Upsert(record); }
public void versioning_should_work() { // guid keys are problematic for raven versioning // so here's a test that show that it now works // this test mutates data so we don't use the ResusableDocumentStore... IDocumentStore store = new InMemoryDatabaseHelper().Create(); Guid id = Guid.Parse("f7b444f7-76f3-47a4-b8d8-cc204d400728"); using (IDocumentSession db = store.OpenSession()) { var record = new Record {Id = id, Gemini = Library.Example()}; db.Store(record); db.SaveChanges(); } using (IDocumentSession db = store.OpenSession()) { var record = db.Load<Record>(id); record.Notes = "i'm updating this record!"; db.SaveChanges(); } using (IDocumentSession db = store.OpenSession()) { var record = db.Load<Record>(id); Record[] revisions = db.Advanced.GetRevisionsFor<Record>(db.Advanced.GetDocumentId(record), 0, 10); revisions.Count().Should().Be(2); revisions.Select(r => r.Revision).Should().ContainInOrder(new[] {1, 2}); } }
public void ExportRecord(Record record, TextWriter writer) { var csv = new CsvWriter(writer); TopcatMapping.ApplyStandardTopcatCsvConfiguration(csv.Configuration); csv.WriteRecord(record); }
private Record Clone(Record record) { var clonedRecord = record.Copy(); clonedRecord.Id = Guid.Empty; clonedRecord.Path = String.Empty; clonedRecord.Gemini.Title = String.Empty; return clonedRecord; }
public void should_count_records_with_duplicate_titles() { var service = new RecordService(Db, new RecordValidator()); var record1 = new Record().With(r => { r.Id = new Guid("7ce85158-f6f9-491d-902e-b3f2c8bb5264"); r.Path = @"X:\path\to\duplicate\record\1"; r.Gemini = new Metadata().With(m => { m.Title = "This is a duplicate record"; m.Keywords.Add(new MetadataKeyword { Vocab = "http://vocab.jncc.gov.uk/jncc-domain", Value = "Marine" }); m.Keywords.Add(new MetadataKeyword { Vocab = "http://vocab.jncc.gov.uk/jncc-category", Value = "Example" }); }); }); var record2 = new Record().With(r => { r.Id = new Guid("afb4ebbf-4286-47ed-b09f-a4d40af139e1"); r.Path = @"X:\path\to\duplicate\record\2"; r.Gemini = new Metadata().With(m => { m.Title = "This is a duplicate record"; m.Keywords.Add(new MetadataKeyword { Vocab = "http://vocab.jncc.gov.uk/jncc-domain", Value = "Marine" }); m.Keywords.Add(new MetadataKeyword { Vocab = "http://vocab.jncc.gov.uk/jncc-category", Value = "Example" }); }); }); service.Insert(record1); service.Insert(record2); Db.SaveChanges(); RavenUtility.WaitForIndexing(Db); var results = Db.Query<RecordsWithDuplicateTitleCheckerIndex.Result, RecordsWithDuplicateTitleCheckerIndex>() .Where(x => x.Count > 1) .Take(100) .ToList(); // looks like we have some duplicates created by the seeder! results.Count.Should().BeInRange(1, 10); // perhaps prevent more duplicate titles being seeded in the future! results.Should().Contain(r => r.Title == "This is a duplicate record"); }
internal RecordServiceResult Upsert(Record record) { CorrectlyOrderKeywords(record); StandardiseUnconditionalUseConstraints(record); UpdateMetadataDateToNow(record); SetMetadataPointOfContactRoleToOnlyAllowedValue(record); var validation = validator.Validate(record); if (!validation.Errors.Any()) { PerformDenormalizations(record); db.Store(record); } return new RecordServiceResult { Record = record, Validation = validation, }; }
void ValidatePublishableInvariants(Record record, ValidationResult<Record> result) { // disabled until publishing mechanism created // publishable_records_must_have_a_resource_locator // if (record.Status == Status.Publishable && record.Gemini.ResourceLocator.IsBlank()) // { // result.Errors.Add("Publishable records must have a resource locator", // r => r.Status, r => r.Gemini.ResourceLocator); // } }
void ValidateResourceLocator(Record record, ValidationResult<Record> result) { // resource_locator_must_be_a_well_formed_http_url if (record.Gemini.ResourceLocator.IsNotBlank()) { Uri url; if (Uri.TryCreate(record.Gemini.ResourceLocator, UriKind.Absolute, out url)) { if (url.Scheme != Uri.UriSchemeHttp && url.Scheme != Uri.UriSchemeHttps) { result.Errors.Add("Resource locator must be an http url", r => r.Gemini.ResourceLocator); } } else { result.Errors.Add("Resource locator must be a valid url", r => r.Gemini.ResourceLocator); } } }
void ValidateMetadataPointOfContact(Record record, ValidationResult<Record> result) { // metadata_point_of_contact_role_must_be_an_allowed_role var role = record.Gemini.MetadataPointOfContact.Role; if (role.IsNotBlank() && !ResponsiblePartyRoles.Allowed.Contains(role)) { result.Errors.Add(String.Format("Metadata Point of Contact Role '{0}' is not valid", role), r => r.Gemini.MetadataPointOfContact.Role); } }
void ValidatePath(Record record, ValidationResult<Record> result) { // path_must_not_be_blank if (record.Path.IsBlank()) { result.Errors.Add("Path must not be blank", r => r.Path); } else // (let's not add additional errors if it's just that it's blank) { // path_must_be_an_acceptable_kind // currently, a file system path or paul's experimental OGR connection string Uri uri; // allow OGR connection strings (experimental) if (record.Path.StartsWith("PG:")) { if (!Regex.IsMatch(record.Path, "PG:\".+\"")) { result.Errors.Add("Path doesn't appear to be a valid OGR connection string"); } } else if (Uri.TryCreate(record.Path, UriKind.Absolute, out uri)) { if (uri.Scheme != Uri.UriSchemeFile) { result.Errors.Add("Path must be a file system path", r => r.Path); } } else { result.Errors.Add("Path is invalid. Normally should be a file system path", r => r.Path); } } }
void ValidateJnccSpecificRules(Record record, ValidationResult<Record> result) { var meshGuiKeywords = record.Gemini.Keywords.Where(k => k.Vocab == "http://vocab.jncc.gov.uk/mesh-gui").ToList(); if (meshGuiKeywords.Count > 1) result.Errors.Add("More than one MESH GUI keyword isn't allowed", r => r.Gemini.Keywords); if (meshGuiKeywords.Any(k => !Regex.IsMatch(k.Value, "^GB[0-9]{6}$"))) result.Errors.Add("MESH GUI not valid", r => r.Gemini.Keywords); }
void ValidateKeywords(Record record, ValidationResult<Record> ValidationResult) { var jnccDomainKeywords = record.Gemini.Keywords.Where(k => k.Vocab == "http://vocab.jncc.gov.uk/jncc-domain"); var jnccCategoryKeywords = record.Gemini.Keywords.Where(k => k.Vocab == "http://vocab.jncc.gov.uk/jncc-category"); if (!jnccDomainKeywords.Any()) { ValidationResult.Errors.Add(String.Format("Must specify a JNCC Domain keyword"), r => r.Gemini.Keywords); } if (!jnccCategoryKeywords.Any()) { ValidationResult.Errors.Add(String.Format("Must specify a JNCC Category keyword"), r => r.Gemini.Keywords); } //No blank keywords if (record.Gemini.Keywords.Any(k => String.IsNullOrWhiteSpace(k.Value))) { ValidationResult.Errors.Add( String.Format("Keywords cannot be blank"), r => r.Gemini.Keywords); } }
void ValidateTitle(Record record, ValidationResult<Record> result) { // title_must_not_be_blank if (record.Gemini.Title.IsBlank()) { result.Errors.Add("Title must not be blank", r => r.Gemini.Title); } // title must be reasonable length if (record.Gemini.Title != null && (record.Gemini.Title.Length > 200)) { if (record.Gemini.ResourceType == "publication" && record.Gemini.Title.Length > 250) { result.Errors.Add("Title is too long. 250 characters or less, please", r => r.Gemini.Title); } else if (record.Gemini.ResourceType != "publication") { result.Errors.Add("Title is too long. 200 characters or less, please", r => r.Gemini.Title); } } }
void AddOverseasTerritoriesRecord() { var record = new Record().With(r => { r.Id = new Guid("d836cd57-7c94-43b6-931b-3d63a58e3541"); r.Path = @"X:\path\to\overseas\territories\data"; r.Gemini = new Metadata().With(m => { m.Title = "A simple Overseas Territories example record"; m.Keywords.Add(new MetadataKeyword { Vocab = "http://vocab.jncc.gov.uk/jncc-domain", Value = "Marine" }); m.Keywords.Add(new MetadataKeyword { Vocab = "http://vocab.jncc.gov.uk/jncc-category", Value = "Overseas Territories" }); m.ResourceType = "dataset"; }); }); recordService.Insert(record); }
public void should_give_new_record_a_new_guid() { var record = new Record { Path = @"X:\some\path", Gemini = Library.Blank().With(m => m.Title = "Some new record!") }; var rsr = RecordServiceResult.SuccessfulResult.With(r => r.Record = record); var service = Mock.Of<IRecordService>(s => s.Insert(It.IsAny<Record>()) == rsr); var controller = new RecordsController(service, Mock.Of<IDocumentSession>(), new TestUserContext()); var result = controller.Post(record); result.Record.Id.Should().NotBeEmpty(); }
void ValidateSecurityInvariants(Record record, ValidationResult<Record> result) { // sensitive_records_must_have_limitations_on_public_access if (record.Security != Security.Official && record.Gemini.LimitationsOnPublicAccess.IsBlank()) { result.Errors.Add("Non-open records must describe their limitations on public access", r => r.Security, r => r.Gemini.LimitationsOnPublicAccess); } }
void UploadAlternativeResources(Record record) { // check no duplicate filenames after webifying var fileNames = from r in record.Publication.OpenData.Resources let fileName = WebificationUtility.ToUrlFriendlyString(Path.GetFileName(r.Path)) group r by fileName; if (fileNames.Count() != record.Publication.OpenData.Resources.Count) throw new Exception("There are duplicate resource file names (after webifying) for this record."); // upload the resources foreach (var r in record.Publication.OpenData.Resources) { UploadDataFile(record.Id, r.Path); } }
public void load_gemini_records() { string dir = @"C:\Users\PETMON\Downloads\nbn-gemini2"; XNamespace gmd = "http://www.isotc211.org/2005/gmd"; var q = (from f in Directory.GetFiles(dir, "*.xml") where File.ReadLines(f).Any() let xml = XDocument.Load(f) let p = new { North = (decimal) xml.Descendants(gmd + "northBoundLatitude").Single(), South = (decimal) xml.Descendants(gmd + "southBoundLatitude").Single(), East = (decimal) xml.Descendants(gmd + "eastBoundLongitude").Single(), West = (decimal) xml.Descendants(gmd + "westBoundLongitude").Single(), } where p.North < 61 && p.South > 50 && p.East < 2 && p.West > -12 // ignore bad data select new { File = Path.GetFileName(f), Wkt = BoundingBoxUtility.ToWkt(new BoundingBox { North = p.North, South = p.South, East = p.East, West = p.West, }) }) .ToList(); Console.WriteLine(q.Count()); q.ForEach(Console.WriteLine); IDocumentStore store = new DocumentStore {Url = RavenUrl}.Initialize(); using (IDocumentSession db = store.OpenSession()) { foreach (var x in q) { var item = new Record { Gemini = new Metadata { Title = x.File, }, Wkt = x.Wkt, }; db.Store(item); } db.SaveChanges(); } IndexCreation.CreateIndexes(typeof (Record).Assembly, store); }
void UploadTheMetadataDocument(Record record, bool alternativeResources) { var doc = new global::Catalogue.Gemini.Encoding.XmlEncoder().Create(record.Id, record.Gemini); if (alternativeResources) { // mung (mutate) the metadata doc so data.gov.uk knows about the resources var onlineResources = record.Publication.OpenData.Resources .Select(r => new OnlineResource { Name = WebificationUtility.ToUrlFriendlyString(Path.GetFileName(r.Path)), Url = config.HttpRootUrl + "/" + GetUnrootedDataPath(record.Id, r.Path) }).ToList(); global::Catalogue.Gemini.Encoding.XmlEncoder.ReplaceDigitalTransferOptions(doc, onlineResources); } var s = new MemoryStream(); doc.Save(s); var metaXmlDoc = s.ToArray(); string metaPath = String.Format("waf/{0}.xml", record.Id); string metaFtpPath = config.FtpRootUrl + "/" + metaPath; ftpClient.UploadBytes(metaFtpPath, metaXmlDoc); }
void ValidateResourceType(Record record, ValidationResult<Record> result) { // resource type must be a valid Gemini resource type if not blank var resourceType = record.Gemini.ResourceType; if (resourceType.IsNotBlank() && !ResourceTypes.Allowed.Contains(resourceType)) { result.Errors.Add(String.Format("Resource Type '{0}' is not valid", resourceType), r => r.Gemini.ResourceType); } }
void UpdateTheWafIndexDocument(Record record) { string indexDocFtpPath = String.Format("{0}/waf/index.html", config.FtpRootUrl); string indexDocHtml = ftpClient.DownloadString(indexDocFtpPath); var doc = XDocument.Parse(indexDocHtml); var body = doc.Root.Element("body"); var newLink = new XElement("a", new XAttribute("href", record.Id + ".xml"), record.Gemini.Title); var existingLinks = body.Elements("a").ToList(); existingLinks.Remove(); var newLinks = existingLinks .Concat(new [] { newLink }) .GroupBy(a => a.Attribute("href").Value) .Select(g => g.First()); // poor man's DistinctBy body.Add(newLinks); ftpClient.UploadString(indexDocFtpPath, doc.ToString()); }
void ValidateResponsibleOrganisation(Record record, ValidationResult<Record> result) { // responsible_organisation_role_must_be_an_allowed_role var role = record.Gemini.ResponsibleOrganisation.Role; if (role.IsNotBlank() && !ResponsiblePartyRoles.Allowed.Contains(role)) { result.Errors.Add(String.Format("Responsible Organisation Role '{0}' is not valid", role), r => r.Gemini.ResponsibleOrganisation.Role); } }
void AddHumanActivitiesRecord() { var record = new Record().With(r => { r.Id = new Guid("d8b438dc-4cd3-4d4f-9fa7-1160ea2336fd"); r.Path = @"X:\path\to\human\activities\data"; r.Gemini = new Metadata().With(m => { m.Title = "A simple Human Activity example record"; m.Keywords.Add(new MetadataKeyword { Vocab = "http://vocab.jncc.gov.uk/jncc-domain", Value = "Marine" }); m.Keywords.Add(new MetadataKeyword { Vocab = "http://vocab.jncc.gov.uk/jncc-category", Value = "Human Activities" }); m.ResourceType = "dataset"; }); }); recordService.Insert(record); }
void ValidateTemporalExtent(Record record, ValidationResult<Record> result) { var begin = record.Gemini.TemporalExtent.Begin; var end = record.Gemini.TemporalExtent.End; // temporal_extent_must_be_valid_dates if (begin.IsNotBlank() && !IsValidDate(begin)) { result.Errors.Add("Temporal Extent (Begin) is not a valid date", r => r.Gemini.TemporalExtent.Begin); } if (end.IsNotBlank() && !IsValidDate(end)) { result.Errors.Add("Temporal Extent (End) is not a valid date", r => r.Gemini.TemporalExtent.End); } // todo ensure End is after Begin }
void ValidateTopicCategory(Record record, ValidationResult<Record> result) { // topic_category_must_be_valid var s = record.Gemini.TopicCategory; if (s.IsNotBlank() && !TopicCategories.Values.Any(c => c.Name == s)) { result.Errors.Add(String.Format("Topic Category '{0}' is not valid", record.Gemini.TopicCategory), r => r.Gemini.TopicCategory); } }
void ValidateBoundingBox(Record record, ValidationResult<Record> result) { if (!BoundingBoxUtility.IsBlank(record.Gemini.BoundingBox)) { if (!BoundingBoxUtility.IsValid(record.Gemini.BoundingBox)) result.Errors.Add("Invalid bounding box", r => r.Gemini.BoundingBox); } }
void UpdateTheResourceLocatorToBeTheOpenDataDownloadPage(Record record) { // this is a big dataset so just link to a webpage string jnccWebDownloadPage = "http://jncc.defra.gov.uk/opendata"; record.Gemini.ResourceLocator = jnccWebDownloadPage; Console.WriteLine("ResourceLocator updated to point to open data request webpage."); }
void PerformGeminiValidation(Record record, ValidationResult<Record> result) { // structured to match the gemini doc // 1 title is validated at basic level // 2 alternative title not used as optional // 3 Dataset language, conditional - data resource contains textual information // lets assume all data resources contain text // data_type is enum so can't be null, will default to eng // 4 abstract is mandatory if (record.Gemini.Abstract.IsBlank()) { result.Errors.Add("Abstract must be provided" + GeminiSuffix, r => r.Gemini.Abstract); } // 5 topic_category_must_not_be_blank if (record.Gemini.TopicCategory.IsBlank()) { result.Errors.Add(String.Format("Topic Category must be provided" + GeminiSuffix), r => r.Gemini.TopicCategory); } // 6 keywords mandatory if (record.Gemini.Keywords.Count == 0) { result.Errors.Add("Keywords must be provided" + GeminiSuffix, r => r.Gemini.Keywords); } // 7 temporal extent is mandatory - at least Begin must be provided if (record.Gemini.TemporalExtent.Begin.IsBlank()) { result.Errors.Add("Temporal Extent must be provided" + GeminiSuffix, r => r.Gemini.TemporalExtent.Begin); } // 8 DatasetReferenceDate mandatory if (record.Gemini.DatasetReferenceDate.IsBlank()) { result.Errors.Add("Dataset Reference Date must be provided" + GeminiSuffix, r => r.Gemini.DatasetReferenceDate); } // 10 Lineage is mandatory if (record.Gemini.Lineage.IsBlank()) { result.Errors.Add("Lineage must be provided" + GeminiSuffix, r => r.Gemini.Lineage); } // 15 extent is optional and not used // 16 Vertical extent information is optional and not used // 17 Spatial reference system is optional // 18 Spatial resolution, where it can be specified it should - so its optional // 19 resource location, conditional // when online access is availble, should be a valid url // when do not yet perform a get request and get a 200 response, the only true way to validate a url if (record.Gemini.ResourceLocator.IsNotBlank()) { Uri url; if (Uri.TryCreate(record.Gemini.ResourceLocator, UriKind.Absolute, out url)) { if (url.Scheme != Uri.UriSchemeHttp) { result.Errors.Add("Resource locator must be an http url", r => r.Gemini.ResourceLocator); } } else { result.Errors.Add("Resource locator must be a valid url", r => r.Gemini.ResourceLocator); } } // 21 DataFormat optional // 23 reponsible Organisation if (record.Gemini.ResponsibleOrganisation.Email.IsBlank()) { result.Errors.Add("Email address for responsible organisation must be provided" + GeminiSuffix, r => r.Gemini.ResponsibleOrganisation.Email); } if (record.Gemini.ResponsibleOrganisation.Name.IsBlank()) { result.Errors.Add("Name of responsible organisation must be provided" + GeminiSuffix, r => r.Gemini.ResponsibleOrganisation.Name); } if (record.Gemini.ResponsibleOrganisation.Role.IsBlank()) { result.Errors.Add("Role of responsible organisation must be provided" + GeminiSuffix, r => r.Gemini.ResponsibleOrganisation.Role); } // 24 frequency of update is optional // 25 limitations on publci access is mandatory if (record.Gemini.LimitationsOnPublicAccess.IsBlank()) { result.Errors.Add("Limitations On Public Access must be provided" + GeminiSuffix, r => r.Gemini.LimitationsOnPublicAccess); } // 26 use constraints are mandatory if (record.Gemini.UseConstraints.IsBlank()) { result.Errors.Add("Use Constraints must be provided (if there are none, leave as 'no conditions apply')", r => r.Gemini.UseConstraints); } // 27 Additional information source is optional // 30 metadatadate is mandatory if (record.Gemini.MetadataDate.Equals(DateTime.MinValue)) { result.Errors.Add("A metadata reference date must be provided" + GeminiSuffix, r => r.Gemini.MetadataDate); } // 33 Metadatalanguage // 35 Point of contacts // org name and email contact mandatory if (record.Gemini.MetadataPointOfContact.Email.IsBlank()) { result.Errors.Add("A metadata point of contact email address must be provided" + GeminiSuffix, r => r.Gemini.MetadataPointOfContact.Email); } if (record.Gemini.MetadataPointOfContact.Name.IsBlank()) { result.Errors.Add("A metadata point of contact organisation name must be provided" + GeminiSuffix, r => r.Gemini.MetadataPointOfContact.Name); } if (record.Gemini.MetadataPointOfContact.Role != "pointOfContact") { result.Errors.Add("The metadata point of contact role must be 'pointOfContact'" + GeminiSuffix, r => r.Gemini.MetadataPointOfContact.Name); } // 36 Unique resource identifier // 39 resource type is mandatory if (record.Gemini.ResourceType.IsBlank()) { result.Errors.Add("A resource type must be provided" + GeminiSuffix, r => r.Gemini.ResourceType); } // 40 Keywords from controlled vocabularys must be defined, they cannot be added. //ValidateControlledKeywords(record, recordValidationResult<Record>); // Conformity, required if claiming conformity to INSPIRE // not yet implemented // Equivalent scale, optional // we're going to try to squash gemini and non-geographic iso metadata together in the same validation if (record.Gemini.ResourceType != "nonGeographicDataset") { // BoundingBox // mandatory // valid todo if (BoundingBoxUtility.IsBlank(record.Gemini.BoundingBox)) { result.Errors.Add("A bounding box must be supplied", r => r.Gemini.BoundingBox); } } }
void ValidateDatasetReferenceDate(Record record, ValidationResult<Record> result) { // dataset_reference_date_must_be_valid_date if (!String.IsNullOrWhiteSpace(record.Gemini.DatasetReferenceDate) && !IsValidDate(record.Gemini.DatasetReferenceDate)) { result.Errors.Add("Dataset reference date is not a valid date", r => r.Gemini.DatasetReferenceDate); } }