private bool CreateOrUpdateMetadata(Dataset dataset, OpenMetadataEndpoint endpoint)
{
string identifier = null;
try
{
identifier = GetIdentifierFromUri(dataset.identifier);
var metadataModel = _metadataService.GetMetadataModel(identifier);
if (metadataModel == null)
{
InsertOpenMetadata(identifier, dataset, endpoint);
Log.Info(
$"Created metadata entry for open dataset: [identifier={identifier}, title={dataset.title}, organization={endpoint.OrganizationName}");
metadataModel = _metadataService.GetMetadataModel(identifier);
}
MapDatasetToMetadataViewModel(dataset, metadataModel, endpoint);
_metadataService.SaveMetadataModel(metadataModel, SecurityClaim.GetUsername());
Log.Info(
$"Updated metadata entry for open dataset: [identifier={identifier}, title={metadataModel.Title}, organization={endpoint.OrganizationName}");
}
catch (Exception e)
{
Log.Error(
$"Error while creating or updating open metadata with identifier={identifier} for endpoint={endpoint}. {e.Message}",
e);
return(false);
}
return(true);
}
private void MapDatasetToMetadataViewModel(Dataset dataset, MetadataViewModel model,
OpenMetadataEndpoint openMetadataEndpoint)
{
model.Title = dataset.title;
DateTime modified;
if (DateTime.TryParse(dataset.modified.ToString(), out modified))
{
model.DateUpdated = modified;
}
DateTime issued;
if (DateTime.TryParse(dataset.issued, out issued))
{
model.DatePublished = issued;
}
model.ContactMetadata.Organization =
!string.IsNullOrEmpty(openMetadataEndpoint.OrganizationName)
? openMetadataEndpoint.OrganizationName
: dataset.publisher.name;
model.ContactMetadata.Email = dataset.contactPoint.hasEmail.Replace("mailto:", "");
model.ContactMetadata.Name = dataset.contactPoint.fn;
model.ContactPublisher.Organization = model.ContactMetadata.Organization;
model.ContactPublisher.Email = model.ContactMetadata.Email;
model.ContactPublisher.Name = model.ContactMetadata.Name;
model.ContactOwner.Organization = model.ContactMetadata.Organization;
model.ContactOwner.Email = model.ContactMetadata.Email;
model.ContactOwner.Name = model.ContactMetadata.Name;
model.Abstract = dataset.description;
if (dataset.keyword != null && dataset.keyword.Length > 0)
{
model.KeywordsOther = dataset.keyword.ToList();
}
model.DistributionsFormats =
GetDistributionsFormats(dataset.distribution, model.ContactMetadata.Organization);
if (string.IsNullOrEmpty(model.MaintenanceFrequency))
{
model.MaintenanceFrequency = "unknown";
}
var spatial = dataset.spatial.Split(',');
if (spatial.Length == 4)
{
model.BoundingBoxWest = spatial[0];
model.BoundingBoxSouth = spatial[1];
model.BoundingBoxEast = spatial[2];
model.BoundingBoxNorth = spatial[3];
}
model.AccessConstraints = "no restrictions";
}
public async Task <Metadata> FetchMetadataAsync(OpenMetadataEndpoint endpoint)
{
Log.Info("Fetching open metadata from: " + endpoint);
var request = new HttpRequestMessage(HttpMethod.Get, endpoint.Url);
request.Headers.Add("Accept", "application/json");
HttpResponseMessage response = await _httpClientFactory.GetHttpClient().SendAsync(request).ConfigureAwait(false);
Log.Debug($"Status from [endpoint={endpoint.Url}] was {response.StatusCode}");
response.EnsureSuccessStatusCode();
return(await response.Content.ReadAsAsync <Metadata>().ConfigureAwait(false));
}
public async Task <int> SynchronizeMetadata(OpenMetadataEndpoint endpoint)
{
var openMetadata = await _metadataFetcher.FetchMetadataAsync(endpoint).ConfigureAwait(false);
var numberOfMetadataCreatedUpdated = 0;
foreach (var dataset in openMetadata.dataset)
{
if (CreateOrUpdateMetadata(dataset, endpoint))
{
numberOfMetadataCreatedUpdated++;
}
}
return(numberOfMetadataCreatedUpdated);
}
private void InsertOpenMetadata(string identifier, Dataset dataset, OpenMetadataEndpoint openMetadataEndpoint)
{
var newMetadata = new MetadataCreateViewModel();
newMetadata.Uuid = identifier;
newMetadata.Type = "dataset";
newMetadata.Title = dataset.title;
newMetadata.MetadataContactOrganization =
!string.IsNullOrEmpty(openMetadataEndpoint.OrganizationName)
? openMetadataEndpoint.OrganizationName
: dataset.publisher.name;
newMetadata.MetadataContactName = dataset.contactPoint.fn;
newMetadata.MetadataContactEmail = dataset.contactPoint.hasEmail.Replace("mailto:", "");
var uuid = _metadataService.CreateMetadata(newMetadata, SecurityClaim.GetUsername());
Log.Info("Created open metadata uuid: " + uuid);
}
