public ValidationResult<Record> Validate(Record record)
{
var result = new ValidationResult<Record>();
ValidatePath(record, result);
ValidateTitle(record, result);
ValidateKeywords(record, result);
ValidateDatasetReferenceDate(record, result);
ValidateTemporalExtent(record, result);
ValidateTopicCategory(record, result);
ValidateResourceLocator(record, result);
ValidateResponsibleOrganisation(record, result);
ValidateMetadataPointOfContact(record, result);
ValidateResourceType(record, result);
ValidateSecurityInvariants(record, result);
ValidatePublishableInvariants(record, result);
ValidateBoundingBox(record, result);
ValidateJnccSpecificRules(record, result);
if (record.Validation == Validation.Gemini)
{
PerformGeminiValidation(record, result);
}
return result;
}
public RecordServiceResult Update(Record record)
{
if (record.ReadOnly)
throw new InvalidOperationException("Cannot update a read-only record.");
return Upsert(record);
}
public void versioning_should_work()
{
// guid keys are problematic for raven versioning
// so here's a test that show that it now works
// this test mutates data so we don't use the ResusableDocumentStore...
IDocumentStore store = new InMemoryDatabaseHelper().Create();
Guid id = Guid.Parse("f7b444f7-76f3-47a4-b8d8-cc204d400728");
using (IDocumentSession db = store.OpenSession())
{
var record = new Record {Id = id, Gemini = Library.Example()};
db.Store(record);
db.SaveChanges();
}
using (IDocumentSession db = store.OpenSession())
{
var record = db.Load<Record>(id);
record.Notes = "i'm updating this record!";
db.SaveChanges();
}
using (IDocumentSession db = store.OpenSession())
{
var record = db.Load<Record>(id);
Record[] revisions = db.Advanced.GetRevisionsFor<Record>(db.Advanced.GetDocumentId(record), 0, 10);
revisions.Count().Should().Be(2);
revisions.Select(r => r.Revision).Should().ContainInOrder(new[] {1, 2});
}
}
public void ExportRecord(Record record, TextWriter writer)
{
var csv = new CsvWriter(writer);
TopcatMapping.ApplyStandardTopcatCsvConfiguration(csv.Configuration);
csv.WriteRecord(record);
}
private Record Clone(Record record)
{
var clonedRecord = record.Copy();
clonedRecord.Id = Guid.Empty;
clonedRecord.Path = String.Empty;
clonedRecord.Gemini.Title = String.Empty;
return clonedRecord;
}
public void should_count_records_with_duplicate_titles()
{
var service = new RecordService(Db, new RecordValidator());
var record1 = new Record().With(r =>
{
r.Id = new Guid("7ce85158-f6f9-491d-902e-b3f2c8bb5264");
r.Path = @"X:\path\to\duplicate\record\1";
r.Gemini = new Metadata().With(m =>
{
m.Title = "This is a duplicate record";
m.Keywords.Add(new MetadataKeyword { Vocab = "http://vocab.jncc.gov.uk/jncc-domain", Value = "Marine" });
m.Keywords.Add(new MetadataKeyword { Vocab = "http://vocab.jncc.gov.uk/jncc-category", Value = "Example" });
});
});
var record2 = new Record().With(r =>
{
r.Id = new Guid("afb4ebbf-4286-47ed-b09f-a4d40af139e1");
r.Path = @"X:\path\to\duplicate\record\2";
r.Gemini = new Metadata().With(m =>
{
m.Title = "This is a duplicate record";
m.Keywords.Add(new MetadataKeyword { Vocab = "http://vocab.jncc.gov.uk/jncc-domain", Value = "Marine" });
m.Keywords.Add(new MetadataKeyword { Vocab = "http://vocab.jncc.gov.uk/jncc-category", Value = "Example" });
});
});
service.Insert(record1);
service.Insert(record2);
Db.SaveChanges();
RavenUtility.WaitForIndexing(Db);
var results = Db.Query<RecordsWithDuplicateTitleCheckerIndex.Result, RecordsWithDuplicateTitleCheckerIndex>()
.Where(x => x.Count > 1)
.Take(100)
.ToList();
// looks like we have some duplicates created by the seeder!
results.Count.Should().BeInRange(1, 10); // perhaps prevent more duplicate titles being seeded in the future!
results.Should().Contain(r => r.Title == "This is a duplicate record");
}
internal RecordServiceResult Upsert(Record record)
{
CorrectlyOrderKeywords(record);
StandardiseUnconditionalUseConstraints(record);
UpdateMetadataDateToNow(record);
SetMetadataPointOfContactRoleToOnlyAllowedValue(record);
var validation = validator.Validate(record);
if (!validation.Errors.Any())
{
PerformDenormalizations(record);
db.Store(record);
}
return new RecordServiceResult
{
Record = record,
Validation = validation,
};
}
void ValidatePublishableInvariants(Record record, ValidationResult<Record> result)
{
// disabled until publishing mechanism created
// publishable_records_must_have_a_resource_locator
// if (record.Status == Status.Publishable && record.Gemini.ResourceLocator.IsBlank())
// {
// result.Errors.Add("Publishable records must have a resource locator",
// r => r.Status, r => r.Gemini.ResourceLocator);
// }
}
void ValidateResourceLocator(Record record, ValidationResult<Record> result)
{
// resource_locator_must_be_a_well_formed_http_url
if (record.Gemini.ResourceLocator.IsNotBlank())
{
Uri url;
if (Uri.TryCreate(record.Gemini.ResourceLocator, UriKind.Absolute, out url))
{
if (url.Scheme != Uri.UriSchemeHttp && url.Scheme != Uri.UriSchemeHttps)
{
result.Errors.Add("Resource locator must be an http url",
r => r.Gemini.ResourceLocator);
}
}
else
{
result.Errors.Add("Resource locator must be a valid url",
r => r.Gemini.ResourceLocator);
}
}
}
void ValidateMetadataPointOfContact(Record record, ValidationResult<Record> result)
{
// metadata_point_of_contact_role_must_be_an_allowed_role
var role = record.Gemini.MetadataPointOfContact.Role;
if (role.IsNotBlank() && !ResponsiblePartyRoles.Allowed.Contains(role))
{
result.Errors.Add(String.Format("Metadata Point of Contact Role '{0}' is not valid", role),
r => r.Gemini.MetadataPointOfContact.Role);
}
}
void ValidatePath(Record record, ValidationResult<Record> result)
{
// path_must_not_be_blank
if (record.Path.IsBlank())
{
result.Errors.Add("Path must not be blank", r => r.Path);
}
else // (let's not add additional errors if it's just that it's blank)
{
// path_must_be_an_acceptable_kind
// currently, a file system path or paul's experimental OGR connection string
Uri uri;
// allow OGR connection strings (experimental)
if (record.Path.StartsWith("PG:"))
{
if (!Regex.IsMatch(record.Path, "PG:\".+\""))
{
result.Errors.Add("Path doesn't appear to be a valid OGR connection string");
}
}
else if (Uri.TryCreate(record.Path, UriKind.Absolute, out uri))
{
if (uri.Scheme != Uri.UriSchemeFile)
{
result.Errors.Add("Path must be a file system path", r => r.Path);
}
}
else
{
result.Errors.Add("Path is invalid. Normally should be a file system path", r => r.Path);
}
}
}
void ValidateJnccSpecificRules(Record record, ValidationResult<Record> result)
{
var meshGuiKeywords = record.Gemini.Keywords.Where(k => k.Vocab == "http://vocab.jncc.gov.uk/mesh-gui").ToList();
if (meshGuiKeywords.Count > 1)
result.Errors.Add("More than one MESH GUI keyword isn't allowed", r => r.Gemini.Keywords);
if (meshGuiKeywords.Any(k => !Regex.IsMatch(k.Value, "^GB[0-9]{6}$")))
result.Errors.Add("MESH GUI not valid", r => r.Gemini.Keywords);
}
void ValidateKeywords(Record record, ValidationResult<Record> ValidationResult)
{
var jnccDomainKeywords = record.Gemini.Keywords.Where(k => k.Vocab == "http://vocab.jncc.gov.uk/jncc-domain");
var jnccCategoryKeywords = record.Gemini.Keywords.Where(k => k.Vocab == "http://vocab.jncc.gov.uk/jncc-category");
if (!jnccDomainKeywords.Any())
{
ValidationResult.Errors.Add(String.Format("Must specify a JNCC Domain keyword"),
r => r.Gemini.Keywords);
}
if (!jnccCategoryKeywords.Any())
{
ValidationResult.Errors.Add(String.Format("Must specify a JNCC Category keyword"),
r => r.Gemini.Keywords);
}
//No blank keywords
if (record.Gemini.Keywords.Any(k => String.IsNullOrWhiteSpace(k.Value)))
{
ValidationResult.Errors.Add(
String.Format("Keywords cannot be blank"),
r => r.Gemini.Keywords);
}
}
void ValidateTitle(Record record, ValidationResult<Record> result)
{
// title_must_not_be_blank
if (record.Gemini.Title.IsBlank())
{
result.Errors.Add("Title must not be blank", r => r.Gemini.Title);
}
// title must be reasonable length
if (record.Gemini.Title != null && (record.Gemini.Title.Length > 200))
{
if (record.Gemini.ResourceType == "publication" && record.Gemini.Title.Length > 250)
{
result.Errors.Add("Title is too long. 250 characters or less, please", r => r.Gemini.Title);
}
else if (record.Gemini.ResourceType != "publication")
{
result.Errors.Add("Title is too long. 200 characters or less, please", r => r.Gemini.Title);
}
}
}
void AddOverseasTerritoriesRecord()
{
var record = new Record().With(r =>
{
r.Id = new Guid("d836cd57-7c94-43b6-931b-3d63a58e3541");
r.Path = @"X:\path\to\overseas\territories\data";
r.Gemini = new Metadata().With(m =>
{
m.Title = "A simple Overseas Territories example record";
m.Keywords.Add(new MetadataKeyword { Vocab = "http://vocab.jncc.gov.uk/jncc-domain", Value = "Marine" });
m.Keywords.Add(new MetadataKeyword { Vocab = "http://vocab.jncc.gov.uk/jncc-category", Value = "Overseas Territories" });
m.ResourceType = "dataset";
});
});
recordService.Insert(record);
}
public void should_give_new_record_a_new_guid()
{
var record = new Record
{
Path = @"X:\some\path",
Gemini = Library.Blank().With(m => m.Title = "Some new record!")
};
var rsr = RecordServiceResult.SuccessfulResult.With(r => r.Record = record);
var service = Mock.Of<IRecordService>(s => s.Insert(It.IsAny<Record>()) == rsr);
var controller = new RecordsController(service, Mock.Of<IDocumentSession>(), new TestUserContext());
var result = controller.Post(record);
result.Record.Id.Should().NotBeEmpty();
}
void ValidateSecurityInvariants(Record record, ValidationResult<Record> result)
{
// sensitive_records_must_have_limitations_on_public_access
if (record.Security != Security.Official && record.Gemini.LimitationsOnPublicAccess.IsBlank())
{
result.Errors.Add("Non-open records must describe their limitations on public access",
r => r.Security,
r => r.Gemini.LimitationsOnPublicAccess);
}
}
void UploadAlternativeResources(Record record)
{
// check no duplicate filenames after webifying
var fileNames = from r in record.Publication.OpenData.Resources
let fileName = WebificationUtility.ToUrlFriendlyString(Path.GetFileName(r.Path))
group r by fileName;
if (fileNames.Count() != record.Publication.OpenData.Resources.Count)
throw new Exception("There are duplicate resource file names (after webifying) for this record.");
// upload the resources
foreach (var r in record.Publication.OpenData.Resources)
{
UploadDataFile(record.Id, r.Path);
}
}
public void load_gemini_records()
{
string dir = @"C:\Users\PETMON\Downloads\nbn-gemini2";
XNamespace gmd = "http://www.isotc211.org/2005/gmd";
var q = (from f in Directory.GetFiles(dir, "*.xml")
where File.ReadLines(f).Any()
let xml = XDocument.Load(f)
let p = new
{
North = (decimal) xml.Descendants(gmd + "northBoundLatitude").Single(),
South = (decimal) xml.Descendants(gmd + "southBoundLatitude").Single(),
East = (decimal) xml.Descendants(gmd + "eastBoundLongitude").Single(),
West = (decimal) xml.Descendants(gmd + "westBoundLongitude").Single(),
}
where p.North < 61 && p.South > 50 && p.East < 2 && p.West > -12
// ignore bad data
select new
{
File = Path.GetFileName(f),
Wkt = BoundingBoxUtility.ToWkt(new BoundingBox
{
North = p.North,
South = p.South,
East = p.East,
West = p.West,
})
})
.ToList();
Console.WriteLine(q.Count());
q.ForEach(Console.WriteLine);
IDocumentStore store = new DocumentStore {Url = RavenUrl}.Initialize();
using (IDocumentSession db = store.OpenSession())
{
foreach (var x in q)
{
var item = new Record
{
Gemini = new Metadata
{
Title = x.File,
},
Wkt = x.Wkt,
};
db.Store(item);
}
db.SaveChanges();
}
IndexCreation.CreateIndexes(typeof (Record).Assembly, store);
}
void UploadTheMetadataDocument(Record record, bool alternativeResources)
{
var doc = new global::Catalogue.Gemini.Encoding.XmlEncoder().Create(record.Id, record.Gemini);
if (alternativeResources)
{
// mung (mutate) the metadata doc so data.gov.uk knows about the resources
var onlineResources = record.Publication.OpenData.Resources
.Select(r => new OnlineResource
{
Name = WebificationUtility.ToUrlFriendlyString(Path.GetFileName(r.Path)),
Url = config.HttpRootUrl + "/" + GetUnrootedDataPath(record.Id, r.Path)
}).ToList();
global::Catalogue.Gemini.Encoding.XmlEncoder.ReplaceDigitalTransferOptions(doc, onlineResources);
}
var s = new MemoryStream();
doc.Save(s);
var metaXmlDoc = s.ToArray();
string metaPath = String.Format("waf/{0}.xml", record.Id);
string metaFtpPath = config.FtpRootUrl + "/" + metaPath;
ftpClient.UploadBytes(metaFtpPath, metaXmlDoc);
}
void ValidateResourceType(Record record, ValidationResult<Record> result)
{
// resource type must be a valid Gemini resource type if not blank
var resourceType = record.Gemini.ResourceType;
if (resourceType.IsNotBlank() && !ResourceTypes.Allowed.Contains(resourceType))
{
result.Errors.Add(String.Format("Resource Type '{0}' is not valid", resourceType),
r => r.Gemini.ResourceType);
}
}
void UpdateTheWafIndexDocument(Record record)
{
string indexDocFtpPath = String.Format("{0}/waf/index.html", config.FtpRootUrl);
string indexDocHtml = ftpClient.DownloadString(indexDocFtpPath);
var doc = XDocument.Parse(indexDocHtml);
var body = doc.Root.Element("body");
var newLink = new XElement("a", new XAttribute("href", record.Id + ".xml"), record.Gemini.Title);
var existingLinks = body.Elements("a").ToList();
existingLinks.Remove();
var newLinks = existingLinks
.Concat(new [] { newLink })
.GroupBy(a => a.Attribute("href").Value)
.Select(g => g.First()); // poor man's DistinctBy
body.Add(newLinks);
ftpClient.UploadString(indexDocFtpPath, doc.ToString());
}
void ValidateResponsibleOrganisation(Record record, ValidationResult<Record> result)
{
// responsible_organisation_role_must_be_an_allowed_role
var role = record.Gemini.ResponsibleOrganisation.Role;
if (role.IsNotBlank() && !ResponsiblePartyRoles.Allowed.Contains(role))
{
result.Errors.Add(String.Format("Responsible Organisation Role '{0}' is not valid", role),
r => r.Gemini.ResponsibleOrganisation.Role);
}
}
void AddHumanActivitiesRecord()
{
var record = new Record().With(r =>
{
r.Id = new Guid("d8b438dc-4cd3-4d4f-9fa7-1160ea2336fd");
r.Path = @"X:\path\to\human\activities\data";
r.Gemini = new Metadata().With(m =>
{
m.Title = "A simple Human Activity example record";
m.Keywords.Add(new MetadataKeyword { Vocab = "http://vocab.jncc.gov.uk/jncc-domain", Value = "Marine" });
m.Keywords.Add(new MetadataKeyword { Vocab = "http://vocab.jncc.gov.uk/jncc-category", Value = "Human Activities" });
m.ResourceType = "dataset";
});
});
recordService.Insert(record);
}
void ValidateTemporalExtent(Record record, ValidationResult<Record> result)
{
var begin = record.Gemini.TemporalExtent.Begin;
var end = record.Gemini.TemporalExtent.End;
// temporal_extent_must_be_valid_dates
if (begin.IsNotBlank() && !IsValidDate(begin))
{
result.Errors.Add("Temporal Extent (Begin) is not a valid date", r => r.Gemini.TemporalExtent.Begin);
}
if (end.IsNotBlank() && !IsValidDate(end))
{
result.Errors.Add("Temporal Extent (End) is not a valid date", r => r.Gemini.TemporalExtent.End);
}
// todo ensure End is after Begin
}
void ValidateTopicCategory(Record record, ValidationResult<Record> result)
{
// topic_category_must_be_valid
var s = record.Gemini.TopicCategory;
if (s.IsNotBlank() && !TopicCategories.Values.Any(c => c.Name == s))
{
result.Errors.Add(String.Format("Topic Category '{0}' is not valid", record.Gemini.TopicCategory),
r => r.Gemini.TopicCategory);
}
}
void ValidateBoundingBox(Record record, ValidationResult<Record> result)
{
if (!BoundingBoxUtility.IsBlank(record.Gemini.BoundingBox))
{
if (!BoundingBoxUtility.IsValid(record.Gemini.BoundingBox))
result.Errors.Add("Invalid bounding box", r => r.Gemini.BoundingBox);
}
}
void UpdateTheResourceLocatorToBeTheOpenDataDownloadPage(Record record)
{
// this is a big dataset so just link to a webpage
string jnccWebDownloadPage = "http://jncc.defra.gov.uk/opendata";
record.Gemini.ResourceLocator = jnccWebDownloadPage;
Console.WriteLine("ResourceLocator updated to point to open data request webpage.");
}
void PerformGeminiValidation(Record record, ValidationResult<Record> result)
{
// structured to match the gemini doc
// 1 title is validated at basic level
// 2 alternative title not used as optional
// 3 Dataset language, conditional - data resource contains textual information
// lets assume all data resources contain text
// data_type is enum so can't be null, will default to eng
// 4 abstract is mandatory
if (record.Gemini.Abstract.IsBlank())
{
result.Errors.Add("Abstract must be provided" + GeminiSuffix, r => r.Gemini.Abstract);
}
// 5 topic_category_must_not_be_blank
if (record.Gemini.TopicCategory.IsBlank())
{
result.Errors.Add(String.Format("Topic Category must be provided" + GeminiSuffix),
r => r.Gemini.TopicCategory);
}
// 6 keywords mandatory
if (record.Gemini.Keywords.Count == 0)
{
result.Errors.Add("Keywords must be provided" + GeminiSuffix, r => r.Gemini.Keywords);
}
// 7 temporal extent is mandatory - at least Begin must be provided
if (record.Gemini.TemporalExtent.Begin.IsBlank())
{
result.Errors.Add("Temporal Extent must be provided" + GeminiSuffix,
r => r.Gemini.TemporalExtent.Begin);
}
// 8 DatasetReferenceDate mandatory
if (record.Gemini.DatasetReferenceDate.IsBlank())
{
result.Errors.Add("Dataset Reference Date must be provided" + GeminiSuffix,
r => r.Gemini.DatasetReferenceDate);
}
// 10 Lineage is mandatory
if (record.Gemini.Lineage.IsBlank())
{
result.Errors.Add("Lineage must be provided" + GeminiSuffix, r => r.Gemini.Lineage);
}
// 15 extent is optional and not used
// 16 Vertical extent information is optional and not used
// 17 Spatial reference system is optional
// 18 Spatial resolution, where it can be specified it should - so its optional
// 19 resource location, conditional
// when online access is availble, should be a valid url
// when do not yet perform a get request and get a 200 response, the only true way to validate a url
if (record.Gemini.ResourceLocator.IsNotBlank())
{
Uri url;
if (Uri.TryCreate(record.Gemini.ResourceLocator, UriKind.Absolute, out url))
{
if (url.Scheme != Uri.UriSchemeHttp)
{
result.Errors.Add("Resource locator must be an http url",
r => r.Gemini.ResourceLocator);
}
}
else
{
result.Errors.Add("Resource locator must be a valid url",
r => r.Gemini.ResourceLocator);
}
}
// 21 DataFormat optional
// 23 reponsible Organisation
if (record.Gemini.ResponsibleOrganisation.Email.IsBlank())
{
result.Errors.Add("Email address for responsible organisation must be provided" + GeminiSuffix,
r => r.Gemini.ResponsibleOrganisation.Email);
}
if (record.Gemini.ResponsibleOrganisation.Name.IsBlank())
{
result.Errors.Add("Name of responsible organisation must be provided" + GeminiSuffix,
r => r.Gemini.ResponsibleOrganisation.Name);
}
if (record.Gemini.ResponsibleOrganisation.Role.IsBlank())
{
result.Errors.Add("Role of responsible organisation must be provided" + GeminiSuffix,
r => r.Gemini.ResponsibleOrganisation.Role);
}
// 24 frequency of update is optional
// 25 limitations on publci access is mandatory
if (record.Gemini.LimitationsOnPublicAccess.IsBlank())
{
result.Errors.Add("Limitations On Public Access must be provided" + GeminiSuffix,
r => r.Gemini.LimitationsOnPublicAccess);
}
// 26 use constraints are mandatory
if (record.Gemini.UseConstraints.IsBlank())
{
result.Errors.Add("Use Constraints must be provided (if there are none, leave as 'no conditions apply')",
r => r.Gemini.UseConstraints);
}
// 27 Additional information source is optional
// 30 metadatadate is mandatory
if (record.Gemini.MetadataDate.Equals(DateTime.MinValue))
{
result.Errors.Add("A metadata reference date must be provided" + GeminiSuffix,
r => r.Gemini.MetadataDate);
}
// 33 Metadatalanguage
// 35 Point of contacts
// org name and email contact mandatory
if (record.Gemini.MetadataPointOfContact.Email.IsBlank())
{
result.Errors.Add("A metadata point of contact email address must be provided" + GeminiSuffix,
r => r.Gemini.MetadataPointOfContact.Email);
}
if (record.Gemini.MetadataPointOfContact.Name.IsBlank())
{
result.Errors.Add("A metadata point of contact organisation name must be provided" + GeminiSuffix,
r => r.Gemini.MetadataPointOfContact.Name);
}
if (record.Gemini.MetadataPointOfContact.Role != "pointOfContact")
{
result.Errors.Add("The metadata point of contact role must be 'pointOfContact'" + GeminiSuffix,
r => r.Gemini.MetadataPointOfContact.Name);
}
// 36 Unique resource identifier
// 39 resource type is mandatory
if (record.Gemini.ResourceType.IsBlank())
{
result.Errors.Add("A resource type must be provided" + GeminiSuffix,
r => r.Gemini.ResourceType);
}
// 40 Keywords from controlled vocabularys must be defined, they cannot be added.
//ValidateControlledKeywords(record, recordValidationResult<Record>);
// Conformity, required if claiming conformity to INSPIRE
// not yet implemented
// Equivalent scale, optional
// we're going to try to squash gemini and non-geographic iso metadata together in the same validation
if (record.Gemini.ResourceType != "nonGeographicDataset")
{
// BoundingBox
// mandatory
// valid todo
if (BoundingBoxUtility.IsBlank(record.Gemini.BoundingBox))
{
result.Errors.Add("A bounding box must be supplied", r => r.Gemini.BoundingBox);
}
}
}
void ValidateDatasetReferenceDate(Record record, ValidationResult<Record> result)
{
// dataset_reference_date_must_be_valid_date
if (!String.IsNullOrWhiteSpace(record.Gemini.DatasetReferenceDate) && !IsValidDate(record.Gemini.DatasetReferenceDate))
{
result.Errors.Add("Dataset reference date is not a valid date", r => r.Gemini.DatasetReferenceDate);
}
}
