private bool CreateOrUpdateMetadata(Dataset dataset, OpenMetadataEndpoint endpoint) { string identifier = null; try { identifier = GetIdentifierFromUri(dataset.identifier); var metadataModel = _metadataService.GetMetadataModel(identifier); if (metadataModel == null) { InsertOpenMetadata(identifier, dataset, endpoint); Log.Info( $"Created metadata entry for open dataset: [identifier={identifier}, title={dataset.title}, organization={endpoint.OrganizationName}"); metadataModel = _metadataService.GetMetadataModel(identifier); } MapDatasetToMetadataViewModel(dataset, metadataModel, endpoint); _metadataService.SaveMetadataModel(metadataModel, SecurityClaim.GetUsername()); Log.Info( $"Updated metadata entry for open dataset: [identifier={identifier}, title={metadataModel.Title}, organization={endpoint.OrganizationName}"); } catch (Exception e) { Log.Error( $"Error while creating or updating open metadata with identifier={identifier} for endpoint={endpoint}. {e.Message}", e); return(false); } return(true); }
private void MapDatasetToMetadataViewModel(Dataset dataset, MetadataViewModel model, OpenMetadataEndpoint openMetadataEndpoint) { model.Title = dataset.title; DateTime modified; if (DateTime.TryParse(dataset.modified.ToString(), out modified)) { model.DateUpdated = modified; } DateTime issued; if (DateTime.TryParse(dataset.issued, out issued)) { model.DatePublished = issued; } model.ContactMetadata.Organization = !string.IsNullOrEmpty(openMetadataEndpoint.OrganizationName) ? openMetadataEndpoint.OrganizationName : dataset.publisher.name; model.ContactMetadata.Email = dataset.contactPoint.hasEmail.Replace("mailto:", ""); model.ContactMetadata.Name = dataset.contactPoint.fn; model.ContactPublisher.Organization = model.ContactMetadata.Organization; model.ContactPublisher.Email = model.ContactMetadata.Email; model.ContactPublisher.Name = model.ContactMetadata.Name; model.ContactOwner.Organization = model.ContactMetadata.Organization; model.ContactOwner.Email = model.ContactMetadata.Email; model.ContactOwner.Name = model.ContactMetadata.Name; model.Abstract = dataset.description; if (dataset.keyword != null && dataset.keyword.Length > 0) { model.KeywordsOther = dataset.keyword.ToList(); } model.DistributionsFormats = GetDistributionsFormats(dataset.distribution, model.ContactMetadata.Organization); if (string.IsNullOrEmpty(model.MaintenanceFrequency)) { model.MaintenanceFrequency = "unknown"; } var spatial = dataset.spatial.Split(','); if (spatial.Length == 4) { model.BoundingBoxWest = spatial[0]; model.BoundingBoxSouth = spatial[1]; model.BoundingBoxEast = spatial[2]; model.BoundingBoxNorth = spatial[3]; } model.AccessConstraints = "no restrictions"; }
public async Task <Metadata> FetchMetadataAsync(OpenMetadataEndpoint endpoint) { Log.Info("Fetching open metadata from: " + endpoint); var request = new HttpRequestMessage(HttpMethod.Get, endpoint.Url); request.Headers.Add("Accept", "application/json"); HttpResponseMessage response = await _httpClientFactory.GetHttpClient().SendAsync(request).ConfigureAwait(false); Log.Debug($"Status from [endpoint={endpoint.Url}] was {response.StatusCode}"); response.EnsureSuccessStatusCode(); return(await response.Content.ReadAsAsync <Metadata>().ConfigureAwait(false)); }
public async Task <int> SynchronizeMetadata(OpenMetadataEndpoint endpoint) { var openMetadata = await _metadataFetcher.FetchMetadataAsync(endpoint).ConfigureAwait(false); var numberOfMetadataCreatedUpdated = 0; foreach (var dataset in openMetadata.dataset) { if (CreateOrUpdateMetadata(dataset, endpoint)) { numberOfMetadataCreatedUpdated++; } } return(numberOfMetadataCreatedUpdated); }
private void InsertOpenMetadata(string identifier, Dataset dataset, OpenMetadataEndpoint openMetadataEndpoint) { var newMetadata = new MetadataCreateViewModel(); newMetadata.Uuid = identifier; newMetadata.Type = "dataset"; newMetadata.Title = dataset.title; newMetadata.MetadataContactOrganization = !string.IsNullOrEmpty(openMetadataEndpoint.OrganizationName) ? openMetadataEndpoint.OrganizationName : dataset.publisher.name; newMetadata.MetadataContactName = dataset.contactPoint.fn; newMetadata.MetadataContactEmail = dataset.contactPoint.hasEmail.Replace("mailto:", ""); var uuid = _metadataService.CreateMetadata(newMetadata, SecurityClaim.GetUsername()); Log.Info("Created open metadata uuid: " + uuid); }