public async Task <ManagementGroup> GetManagementGroupAsync(string id, string fields, DateTime?pointInTime, CancellationToken cancellationToken)
{
var typeAndId = SplitCode(id);
var isLocalAuthority = await IsLocalAuthorityAsync(typeAndId.Key, cancellationToken);
var managementGroup = isLocalAuthority
? await GetLocalAuthorityAsManagementGroupAsync(typeAndId.Value, pointInTime, cancellationToken)
: await GetGroupAsManagementGroupAsync(typeAndId.Value, pointInTime, cancellationToken);
if (managementGroup == null)
{
return(null);
}
if (!string.IsNullOrEmpty(fields))
{
managementGroup = managementGroup.Pick(fields);
_logger.Debug(
$"Pruned management group: {managementGroup}.");
}
else
{
_logger.Debug("No fields specified - model not pruned.");
}
return(managementGroup);
}
public async Task <LearningProvider> GetLearningProviderAsync(string id, string fields, bool readFromLive, DateTime?pointInTime, CancellationToken cancellationToken)
{
long ukprn;
if (!long.TryParse(id, out ukprn))
{
throw new ArgumentException($"id must be a number (ukprn) but received {id}", nameof(id));
}
if (id.Length != 8)
{
throw new ArgumentException($"UKPRN must be 8 digits but received {id.Length} ({id})");
}
var provider = readFromLive
? await _ukrlpApiClient.GetProviderAsync(ukprn, cancellationToken)
: await _providerRepository.GetProviderAsync(ukprn, pointInTime, cancellationToken);
if (provider == null)
{
return(null);
}
_logger.Debug($"read provider {ukprn}: {JsonConvert.SerializeObject(provider)}");
var requestedFields = string.IsNullOrEmpty(fields) ? null : fields.Split(',').Select(x => x.Trim()).ToArray();
return(await GetLearningProviderFromUkrlpProviderAsync(provider, requestedFields, cancellationToken));
}
private async Task <T> ExecuteContainerActionAsync <T>(Func <Task <T> > asyncAction, ILoggerWrapper logger, CancellationToken cancellationToken)
{
var attempt = 0;
while (true)
{
var waitFor = _retryAfter - DateTime.Now;
if (waitFor.TotalMilliseconds > 0)
{
logger.Debug($"Joining queue; waiting for {waitFor.TotalSeconds:0}s");
await Task.Delay(waitFor, cancellationToken);
}
try
{
return(await asyncAction());
}
catch (CosmosException ex)
{
if (attempt >= MaxActionAttempts - 1 || (int)ex.StatusCode != 429)
{
throw;
}
lock (this)
{
_retryAfter = DateTime.Now.Add(ex.RetryAfter ?? TimeSpan.FromSeconds(1));
logger.Debug($"Received 429 on attempt {attempt}. Will retry after {_retryAfter}");
}
}
attempt++;
}
}
private async Task <CloudBlockBlob> GetBlobReferenceAsync(string fileName, bool checkExists, CancellationToken cancellationToken)
{
CloudBlockBlob blob;
await _container.CreateIfNotExistsAsync(cancellationToken);
if (!string.IsNullOrEmpty(_configuration.StorageFolderName))
{
var folder = _container.GetDirectoryReference(_configuration.StorageFolderName);
blob = folder.GetBlockBlobReference(fileName);
if (checkExists && !await blob.ExistsAsync(cancellationToken))
{
_logger.Debug($"Cannot find blob {fileName} in folder {_configuration.StorageFolderName}");
return(null);
}
}
else
{
blob = _container.GetBlockBlobReference(fileName);
if (checkExists && !await blob.ExistsAsync(cancellationToken))
{
_logger.Debug($"Cannot find blob {fileName}");
return(null);
}
}
return(blob);
}
public async Task SyncAsync(ManagementGroup managementGroup, string source, CancellationToken cancellationToken)
{
var searchDocument = MapManagementGroupToSearchDocument(managementGroup, source);
_logger.Info($"Mapped management group to search document: {JsonConvert.SerializeObject(searchDocument)}");
await _searchIndex.UploadBatchAsync(new[] { searchDocument }, cancellationToken);
_logger.Debug($"Successfully uploaded document to search index");
}
public async Task SyncAsync(LearningProvider learningProvider, string source,
CancellationToken cancellationToken)
{
var searchDocument = MapLearningProviderToSearchDocument(learningProvider, source);
_logger.Info($"Mapped learning provider to search document: {JsonConvert.SerializeObject(searchDocument)}");
await _searchIndex.UploadBatchAsync(new[] { searchDocument }, cancellationToken);
_logger.Debug($"Successfully uploaded document to search index");
}
public async Task PublishLearningProviderCreatedAsync(LearningProvider learningProvider, DateTime pointInTime, CancellationToken cancellationToken)
{
var @event = new PointInTimeMiddlewareEvent <LearningProvider>
{
Details = learningProvider,
PointInTime = pointInTime,
};
await SendEventToMiddleware("learning-provider-created", @event, cancellationToken);
_logger.Debug($"Published learning provider created: {JsonConvert.SerializeObject(learningProvider)}");
}
public async Task DownloadProvidersToCacheAsync(CancellationToken cancellationToken)
{
_logger.Info("Acquiring providers file from UKRLP...");
var pointInTime = DateTime.UtcNow.Date;
// Last read
var lastRead = await _stateRepository.GetLastProviderReadTimeAsync(cancellationToken);
// Download
var providers = await _ukrlpApiClient.GetProvidersUpdatedSinceAsync(lastRead, cancellationToken);
_logger.Info($"Read {providers.Length} providers from UKRLP that have been updated since {lastRead}");
// Timestamp
var pointInTimeProviders = providers.Select(establishment => establishment.Clone <PointInTimeProvider>()).ToArray();
foreach (var pointInTimeEstablishment in pointInTimeProviders)
{
pointInTimeEstablishment.PointInTime = pointInTime;
}
// Store
await _providerRepository.StoreInStagingAsync(pointInTimeProviders, cancellationToken);
_logger.Debug($"Stored {providers.Length} providers in staging");
// Queue diff check
var position = 0;
const int batchSize = 100;
while (position < providers.Length)
{
var batch = providers
.Skip(position)
.Take(batchSize)
.Select(e => e.UnitedKingdomProviderReferenceNumber)
.ToArray();
_logger.Debug($"Queuing {position} to {position + batch.Length} for processing");
await _providerProcessingQueue.EnqueueBatchOfStagingAsync(batch, pointInTime, cancellationToken);
position += batchSize;
}
// Update last read
lastRead = DateTime.Now;
await _stateRepository.SetLastProviderReadTimeAsync(lastRead, cancellationToken);
_logger.Info($"Set last read time to {lastRead}");
_logger.Info("Finished downloading providers to cache");
}
public async Task <LearningProvider> GetLearningProviderAsync(string id, string fields, bool readFromLive, DateTime?pointInTime, CancellationToken cancellationToken)
{
var establishment = readFromLive
? await GetEstablishmentFromApiAsync(id, pointInTime, cancellationToken)
: await GetEstablishmentFromCacheAsync(id, pointInTime, cancellationToken);
if (establishment == null)
{
return(null);
}
_logger.Debug($"read establishment {id} from {(readFromLive ? "live" : "cache")}: {JsonConvert.SerializeObject(establishment)}");
return(await GetLearningProviderFromEstablishment(establishment, fields, cancellationToken));
}
public async Task <IActionResult> RunAsync(
[HttpTrigger(AuthorizationLevel.Function, "post", Route = "management-groups/sync/{source}")]
HttpRequest req,
string source,
CancellationToken cancellationToken)
{
_spiExecutionContextManager.SetContext(req.Headers);
_logger.Info($"Start processing sync of management group from {source}...");
ManagementGroup managementGroup;
using (var reader = new StreamReader(req.Body))
{
var json = await reader.ReadToEndAsync();
_logger.Debug($"Received body {json}");
managementGroup = JsonConvert.DeserializeObject <ManagementGroup>(json);
_logger.Info($"Received management group for sync: {JsonConvert.SerializeObject(managementGroup)}");
}
await _searchManager.SyncAsync(managementGroup, source, cancellationToken);
_logger.Info("Successfully sync'd management group");
return(new AcceptedResult());
}
public async Task <string[]> GetEnumerationValuesAsync(string enumName, CancellationToken cancellationToken)
{
var resource = $"enumerations/{enumName}";
var cached = (string[])(await _cacheProvider.GetCacheItemAsync(resource, cancellationToken));
if (cached != null)
{
return(cached);
}
_logger.Info($"Calling {resource} on translator api");
var request = new RestRequest(resource, Method.GET);
request.AppendContext(_executionContextManager.SpiExecutionContext);
var response = await _restClient.ExecuteTaskAsync(request, cancellationToken);
if (!response.IsSuccessful)
{
throw new TranslatorApiException(resource, response.StatusCode, response.Content);
}
_logger.Debug($"Received {response.Content}");
var result = JsonConvert.DeserializeObject <GetEnumerationValuesResult>(response.Content);
await _cacheProvider.AddCacheItemAsync(resource, result.EnumerationValuesResult.EnumerationValues,
new TimeSpan(0, 1, 0), cancellationToken);
return(result.EnumerationValuesResult.EnumerationValues);
}
public async Task <IActionResult> RunAsync(
[HttpTrigger(AuthorizationLevel.Function, "post", Route = "learning-providers/sync/{source}")]
HttpRequest req,
string source,
CancellationToken cancellationToken)
{
_spiExecutionContextManager.SetContext(req.Headers);
_logger.Info($"Start processing sync of learning provider from {source}...");
LearningProvider learningProvider;
using (var reader = new StreamReader(req.Body))
{
var json = await reader.ReadToEndAsync();
_logger.Debug($"Received body {json}");
learningProvider = JsonConvert.DeserializeObject <LearningProvider>(json);
_logger.Info($"Received learning provider for sync: {JsonConvert.SerializeObject(learningProvider)}");
}
await _searchManager.SyncAsync(learningProvider, source, cancellationToken);
_logger.Info("Successfully sync'd learning provider");
return(new AcceptedResult());
}
public async Task StoreAsync(PointInTimeProvider[] providers, CancellationToken cancellationToken)
{
const int batchSize = 100;
await _table.CreateIfNotExistsAsync(cancellationToken);
var entities = new List <ProviderEntity>();
foreach (var provider in providers)
{
if (provider.IsCurrent)
{
entities.Add(ModelToEntity("current", provider));
}
entities.Add(ModelToEntity(provider));
}
var partitionedEntities = entities
.GroupBy(entity => entity.PartitionKey)
.ToDictionary(g => g.Key, g => g.ToArray());
foreach (var partition in partitionedEntities.Values)
{
var position = 0;
while (position < partition.Length)
{
var batchOfEntities = partition.Skip(position).Take(batchSize).ToArray();
var batch = new TableBatchOperation();
foreach (var entity in batchOfEntities)
{
batch.InsertOrReplace(entity);
}
_logger.Debug(
$"Inserting {position} to {partition.Length} for partition {batchOfEntities.First().PartitionKey}");
await _table.ExecuteBatchAsync(batch, cancellationToken);
position += batchSize;
}
}
}
public async Task <IDictionary <LearningProviderPointer, ManagementGroup> > LoadAsync(
IEnumerable <LearningProviderPointer> keys,
CancellationToken cancellationToken)
{
var learningProviderPointers = keys.ToArray();
_logger.Debug($"Looking up management groups for {learningProviderPointers.Length} providers");
var managementGroupLinks = await GetManagementGroupLinksAsync(learningProviderPointers, cancellationToken);
var managementGroups = await LoadManagementGroupsAsync(
managementGroupLinks,
learningProviderPointers.FirstOrDefault()?.Fields,
learningProviderPointers.FirstOrDefault()?.PointInTime,
cancellationToken);
var results = TransformToDictionary(learningProviderPointers, managementGroupLinks, managementGroups);
return(results);
}
public async Task <string> TranslateEnumValue(string enumName, string sourceValue,
CancellationToken cancellationToken)
{
var mappings = await GetMappings(enumName, cancellationToken);
if (mappings == null)
{
return(null);
}
var mapping = mappings.FirstOrDefault(kvp =>
kvp.Value.Any(v => v.Equals(sourceValue, StringComparison.InvariantCultureIgnoreCase))).Key;
if (string.IsNullOrEmpty(mapping))
{
_logger.Warning($"No enum mapping found for GIAS for {enumName} with value {sourceValue}");
return(null);
}
_logger.Debug($"Found mapping of {mapping} for {enumName} with value {sourceValue}");
return(mapping);
}
public async Task SendAsync(Distribution distribution, CancellationToken cancellationToken)
{
_logger.Debug($"Reading latest distribution data for {distribution.Id} in subscription {distribution.SubscriptionId}");
distribution = await _distributionRepository.GetAsync(distribution.Id, distribution.SubscriptionId, cancellationToken);
if (distribution.Status == DistributionStatus.Sent)
{
_logger.Info($"Stopping processing distribution {distribution.Id} in subscription {distribution.SubscriptionId} as it is already sent");
return;
}
_logger.Debug($"Reading event {distribution.EventId}");
var @event = await _eventRepository.GetAsync(distribution.EventId, cancellationToken);
_logger.Debug($"Reading subscription {distribution.SubscriptionId} for {@event.Publisher}.{@event.EventType}");
var subscription = await _subscriptionRepository.GetSubscriptionToEventAsync(@event.Publisher, @event.EventType,
distribution.SubscriptionId, cancellationToken);
distribution.Attempts++;
try
{
await SendToSubscriberAsync(@event, subscription, cancellationToken);
_logger.Info($"Sent event {@event.Id} to subscriber {subscription.Id} at {subscription.EndpointUrl}");
distribution.Status = DistributionStatus.Sent;
await _distributionRepository.UpdateAsync(distribution, cancellationToken);
}
catch (Exception)
{
distribution.Status = distribution.Attempts >= 5
? DistributionStatus.Failed
: DistributionStatus.PendingRetry;
await _distributionRepository.UpdateAsync(distribution, cancellationToken);
throw;
}
}
public async Task ReceiveAsync(string source, string eventType, string payload, CancellationToken cancellationToken)
{
await ValidateRequestAsync(source, eventType, payload, cancellationToken);
var eventId = Guid.NewGuid().ToString().ToLower();
await _eventRepository.StoreAsync(new Event
{
Id = eventId,
Publisher = source,
EventType = eventType,
Payload = payload,
}, cancellationToken);
var subscriptions =
await _subscriptionRepository.GetSubscriptionsToEventAsync(source, eventType, cancellationToken);
_logger.Debug($"Found {subscriptions.Length} subscribers to event {source}.{eventType}");
foreach (var subscription in subscriptions)
{
var distribution = new Distribution
{
Id = Guid.NewGuid().ToString(),
SubscriptionId = subscription.Id,
EventId = eventId,
Status = DistributionStatus.Pending,
Attempts = 0,
};
await _distributionRepository.CreateAsync(distribution, cancellationToken);
_logger.Debug($"Created distribution with id {distribution.Id} for subscription {subscription.Id} to send event {eventId} ({source}.{eventType})");
await _distributionQueue.EnqueueAsync(distribution, cancellationToken);
_logger.Info($"Queued distribution with id {distribution.Id} for subscription {subscription.Id} to send event {eventId} ({source}.{eventType})");
}
_logger.Info($"Finished receiving and distributing event {eventId} ({source}.{eventType})");
}
public async Task ProcessSyncQueueItemAsync(SyncQueueItem queueItem, CancellationToken cancellationToken)
{
_logger.Debug($"Trying to find existing entity for {queueItem.Entity} at {queueItem.PointInTime}");
var existingEntity = await _repository.RetrieveAsync(
queueItem.Entity.EntityType,
queueItem.Entity.SourceSystemName,
queueItem.Entity.SourceSystemId,
queueItem.PointInTime,
cancellationToken);
var matchResult = await _matcher.MatchAsync(queueItem.Entity, queueItem.PointInTime, cancellationToken);
_logger.Info($"Matching found {matchResult.Synonyms?.Length} synonyms and {matchResult.Links?.Length} links");
var registeredEntity = GetRegisteredEntityForPointInTime(queueItem.Entity, queueItem.PointInTime, matchResult);
await ProcessEntityChangesAsync(existingEntity, registeredEntity, matchResult, cancellationToken);
_logger.Info($"Finished processing entity {queueItem.Entity} at {queueItem.PointInTime}");
}
public async Task Run(
[QueueTrigger(CacheQueueNames.LocalAuthorityProcessingQueue)]
string queueContent,
CancellationToken cancellationToken)
{
_httpSpiExecutionContextManager.SetInternalRequestId(Guid.NewGuid());
_logger.Info($"{FunctionName} trigger with: {queueContent}");
var queueItem = JsonConvert.DeserializeObject <StagingBatchQueueItem <int> >(queueContent);
_logger.Debug($"Deserialized to {queueItem.Urns.Length} urns for local authority {queueItem.ParentIdentifier} on {queueItem.PointInTime}");
await _cacheManager.ProcessLocalAuthorityAsync(queueItem.ParentIdentifier, queueItem.Urns, queueItem.PointInTime, cancellationToken);
}
public async Task Run(
[QueueTrigger(CacheQueueNames.ProviderProcessingQueue)]
string queueContent,
CancellationToken cancellationToken)
{
_httpSpiExecutionContextManager.SetInternalRequestId(Guid.NewGuid());
_logger.Info($"{FunctionName} trigger with: {queueContent}");
var queueItem = JsonConvert.DeserializeObject <StagingBatchQueueItem>(queueContent);
_logger.Debug($"Deserialized to {queueItem.Identifiers.Length} ukprns on {queueItem.PointInTime}");
await _cacheManager.ProcessBatchOfProviders(queueItem.Identifiers, queueItem.PointInTime, cancellationToken);
}
public async Task <T[]> RunQueryAsync <T>(QueryDefinition query, ILoggerWrapper logger, CancellationToken cancellationToken)
{
return(await ExecuteContainerActionAsync(async() =>
{
logger.Debug($"Running CosmosDB query: {query.QueryText}");
var iterator = Container.GetItemQueryIterator <T>(query);
var results = new List <T>();
while (iterator.HasMoreResults)
{
var batch = await iterator.ReadNextAsync(cancellationToken);
results.AddRange(batch);
}
return results.ToArray();
}, logger, cancellationToken));
}
public async Task <Subscription> UpdateSubscriptionAsync(Subscription subscription, CancellationToken cancellationToken)
{
// TODO: Should validate publisher and event type exist; but as it will only be internal for now should be ok.
// Worst that can happen is a number of subscriptions are added that will never receive events
if (string.IsNullOrEmpty(subscription.Id))
{
subscription.Id = Guid.NewGuid().ToString();
_logger.Debug($"No subscription id provided. Set to {subscription.Id}");
}
_logger.Info($"Storing subscription for {subscription.Publisher}.{subscription.EventType} with id {subscription.Id}");
await _subscriptionRepository.UpdateSubscriptionAsync(subscription, cancellationToken);
_logger.Info($"Stored subscription {subscription.Id}");
return(subscription);
}
public async Task RunAsync(
[QueueTrigger(QueueNames.SyncQueue)]
CloudQueueMessage queueItem,
CancellationToken cancellationToken)
{
var tempInternalRequestId = Guid.NewGuid();
_executionContextManager.SetInternalRequestId(tempInternalRequestId);
_logger.Info($"Started processing item {queueItem.Id} from {QueueNames.SyncQueue} for attempt {queueItem.DequeueCount} (Put in queue at {queueItem.InsertionTime})");
_logger.Info($"Queue item content: {queueItem.AsString}");
var syncQueueItem = JsonConvert.DeserializeObject <SyncQueueItem>(queueItem.AsString);
_logger.Debug($"Deserialized content to {JsonConvert.SerializeObject(syncQueueItem)}");
if (syncQueueItem.InternalRequestId.HasValue)
{
_executionContextManager.SetInternalRequestId(syncQueueItem.InternalRequestId.Value);
_logger.Info($"Changed internal request id from {tempInternalRequestId} to {syncQueueItem.InternalRequestId.Value} to correlate processing with receipt");
}
await _syncManager.ProcessSyncQueueItemAsync(syncQueueItem, cancellationToken);
}
private SquashedEntityResult[] Squash(EntityReferenceSourceData <T>[] candidates, AggregatesRequest aggregatesRequest, string[] fields,
EntityProfile entityProfile)
{
var results = new SquashedEntityResult[candidates.Length];
var requiredProperties = fields.Length == 0
? _typeProperties
: _typeProperties
.Where(p => fields.Any(f => f.Equals(p.Name, StringComparison.InvariantCultureIgnoreCase)))
.ToArray();
_logger.Info($"Squashing {candidates.Length} entities with {requiredProperties.Length} properties");
_logger.Debug("Required properties: " + requiredProperties.Select(x => x.Name).Aggregate((x, y) => $"{x}, {y}"));
for (var i = 0; i < candidates.Length; i++)
{
var candidate = candidates[i];
var nonErroredSources = candidate.SourceEntities.Where(e => e.AdapterError == null).ToArray();
T entity = null;
if (nonErroredSources.Length > 0)
{
entity = new T();
var entityBase = entity as EntityBase;
var isLineageRequired = entity != null &&
(fields.Length == 0 || fields.Any(x => x.Equals("_lineage", StringComparison.InvariantCultureIgnoreCase)));
if (isLineageRequired)
{
entityBase._Lineage = new Dictionary <string, LineageEntry>();
}
_logger.Debug($"Squashing {candidate.EntityReference} {(isLineageRequired ? "with" : "without")} lineage");
foreach (var property in requiredProperties)
{
var fieldProfile = entityProfile.Fields.SingleOrDefault(f => f.Name.Equals(property.Name, StringComparison.InvariantCultureIgnoreCase));
var sources = fieldProfile?.Sources != null && fieldProfile.Sources.Length > 0
? fieldProfile.Sources
: entityProfile.Sources;
var treatWhitespaceAsNull = fieldProfile?.TreatWhitespaceAsNull ?? false;
if (property.Name.Equals("_aggregations", StringComparison.InvariantCultureIgnoreCase))
{
SquashAggregationsForEntity(property, entity, nonErroredSources, sources, aggregatesRequest);
}
else
{
SquashPropertyForEntity(property, entity, entityBase, nonErroredSources, sources, treatWhitespaceAsNull, isLineageRequired);
}
}
}
results[i] = new SquashedEntityResult
{
EntityReference = candidate.EntityReference,
SquashedEntity = entity,
EntityAdapterErrorDetails = candidate.SourceEntities
.Where(e => e.AdapterError != null)
.Select(e =>
new EntityAdapterErrorDetail
{
AdapterName = e.SourceName,
RequestedId = e.SourceId,
RequestedFields = fields,
RequestedEntityName = e.AdapterError.RequestedEntityName,
HttpStatusCode = e.AdapterError.HttpStatusCode,
HttpErrorBody = e.AdapterError.HttpErrorBody
})
.ToArray(),
};
}
return(results);
}
protected async Task <DataAdapterResult <T>[]> GetEntitiesFromApi <T>(
string[] identifiers,
Dictionary <string, AggregateQuery> aggregateQueries,
string[] fields,
bool live,
DateTime?pointInTime,
CancellationToken cancellationToken)
{
var bearerToken = _executionContextManager.SpiExecutionContext.IdentityToken;
var entityType = GetPluralUrlEntityName <T>();
if (string.IsNullOrEmpty(entityType))
{
throw new Exception($"Unsupported entity type {typeof(T)}");
}
using (var client = new HttpClient())
{
client.BaseAddress = new Uri(_baseUrl, UriKind.Absolute);
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", bearerToken);
client.DefaultRequestHeaders.Add("Ocp-Apim-Subscription-Key", _subscriptionKey);
client.DefaultRequestHeaders.Add(SpiHeaderNames.InternalRequestIdHeaderName,
_executionContextManager.SpiExecutionContext.InternalRequestId.ToString());
if (!string.IsNullOrEmpty(_executionContextManager.SpiExecutionContext.ExternalRequestId))
{
client.DefaultRequestHeaders.Add(SpiHeaderNames.ExternalRequestIdHeaderName,
_executionContextManager.SpiExecutionContext.ExternalRequestId);
}
var body = JsonConvert.SerializeObject(
new SpiAdapterBatchRequest
{
Identifiers = identifiers,
AggregateQueries = aggregateQueries,
Fields = fields,
Live = live,
PointInTime = pointInTime,
}, SerializerSettings);
_logger.Debug($"Calling {SourceName} adapter at {_baseUrl} with {body}");
var response = await client.PostAsync(
entityType,
new StringContent(body, Encoding.UTF8, "application/json"),
cancellationToken);
if (!response.IsSuccessStatusCode)
{
HttpErrorBody errorBody = null;
try
{
var errorJson = await response.Content.ReadAsStringAsync();
errorBody = JsonConvert.DeserializeObject <HttpErrorBody>(errorJson);
}
catch (Exception ex)
{
var fullUrl = new Uri(new Uri(_baseUrl, UriKind.Absolute), new Uri(entityType, UriKind.Relative));
_logger.Warning($"Error reading standard error response from {(int)response.StatusCode} response from {fullUrl}: {ex.Message}");
}
throw new DataAdapterException($"Error calling {SourceName} adapter")
{
AdapterName = SourceName,
RequestedEntityName = entityType,
RequestedIds = identifiers,
RequestedFields = fields,
HttpStatusCode = response.StatusCode,
HttpErrorBody = errorBody,
};
}
var json = await response.Content.ReadAsStringAsync();
var entities = JsonConvert.DeserializeObject <T[]>(json);
var results = new DataAdapterResult <T> [identifiers.Length];
for (var i = 0; i < identifiers.Length; i++)
{
results[i] = new DataAdapterResult <T>
{
Identifier = identifiers[i],
Entity = entities[i],
};
}
return(results);
}
}
private async Task <PointInTimeGroup[]> DownloadGroupsToCacheAsync(DateTime pointInTime, CancellationToken cancellationToken)
{
_logger.Info("Acquiring groups file from GIAS...");
// Download
var groups = await _giasApiClient.DownloadGroupsAsync(cancellationToken);
_logger.Debug($"Downloaded {groups.Length} groups from GIAS");
// Timestamp
var pointInTimeGroups = groups.Select(group => group.Clone <PointInTimeGroup>()).ToArray();
foreach (var pointInTimeGroup in pointInTimeGroups)
{
pointInTimeGroup.PointInTime = pointInTime;
}
// Store
await _groupRepository.StoreInStagingAsync(pointInTimeGroups, cancellationToken);
_logger.Debug($"Stored {pointInTimeGroups.Length} groups in staging");
_logger.Info("Finished downloading groups to cache");
return(pointInTimeGroups);
}
public async Task <EntityReference[]> GetSynonymsAsync(string entityType, string sourceSystem, string sourceSystemId,
CancellationToken cancellationToken)
{
var resource = $"{entityType}/{sourceSystem}/{sourceSystemId}/synonyms";
_logger.Debug($"Looking up synonyms at {resource}");
var httpRequest = new RestRequest(resource, Method.GET);
httpRequest.AppendContext(_executionContextManager.SpiExecutionContext);
var response = await _restClient.ExecuteTaskAsync(httpRequest, cancellationToken);
if (!response.IsSuccessful)
{
if (response.StatusCode == HttpStatusCode.NotFound)
{
return(new EntityReference[0]);
}
throw new RegistryApiException(resource, response.StatusCode, response.Content);
}
_logger.Debug($"Synonyms response json from {resource} is ${response.Content}");
var results = JsonConvert.DeserializeObject <GetSynonymsResult>(response.Content);
_logger.Debug($"Deserialized response from {resource} to {JsonConvert.SerializeObject(results)}");
return(results.Synonyms);
}
private async Task <EntityCollection <T> > LoadAsync <T>(LoadEntitiesRequest request, CancellationToken cancellationToken) where T : ModelsBase
{
const string resource = "get-squashed-entity";
var squasherRequest = new GetSquashedEntitiesRequest
{
EntityName = request.EntityName,
EntityReferences = request.EntityReferences?.Select(er =>
new SquasherEntityReference
{
AdapterRecordReferences = er.AdapterRecordReferences.Select(arr =>
new SquasherAdapterReference
{
SourceSystemName = arr.SourceSystemName,
SourceSystemId = arr.SourceSystemId,
}).ToArray(),
})?.ToArray(),
Fields = request.Fields,
AggregatesRequest = request.AggregatesRequest,
Live = request.Live,
PointInTime = request.PointInTime,
};
var json = JsonConvert.SerializeObject(squasherRequest);
_logger.Debug($"Search request going to {resource} is {json}");
var httpRequest = new RestRequest(resource, Method.POST, DataFormat.Json);
httpRequest.AddParameter("", json, ParameterType.RequestBody);
httpRequest.AppendContext(_executionContextManager.SpiExecutionContext);
var response = await _restClient.ExecuteTaskAsync(httpRequest, cancellationToken);
if (response.StatusCode == HttpStatusCode.NotFound)
{
try
{
var errorBody = JsonConvert.DeserializeObject <HttpErrorBody>(response.Content);
if (errorBody.ErrorIdentifier == "SPI-ESQ-6")
{
return(new EntityCollection <T>
{
SquashedEntityResults = request.EntityReferences?.Select(x =>
new SquashedEntityResult <T>()).ToArray(),
});
}
_logger.Warning($"Received 404, but ErrorIdentifier was not expected. Expected SPI-ESQ-6, Received {errorBody.ErrorIdentifier} (message: {errorBody.Message}");
}
catch (Exception ex)
{
_logger.Warning($"Received 404, but was unable to process error body - {ex.Message}", ex);
}
}
if (!response.IsSuccessful)
{
throw new SquasherApiException(resource, response.StatusCode, response.Content);
}
_logger.Debug($"Search response json from {resource} is ${response.Content}");
var results = JsonConvert.DeserializeObject <EntityCollection <T> >(response.Content);
_logger.Debug($"Deserialized response from {resource} to {JsonConvert.SerializeObject(results)}");
return(results);
}