private static TableBuilderService BuildTableBuilderService(
StatisticsDbContext statisticsDbContext,
IFilterItemRepository?filterItemRepository = null,
IObservationService?observationService = null,
IPersistenceHelper <StatisticsDbContext>?statisticsPersistenceHelper = null,
IResultSubjectMetaService?resultSubjectMetaService = null,
ISubjectRepository?subjectRepository = null,
IUserService?userService = null,
IResultBuilder <Observation, ObservationViewModel>?resultBuilder = null,
IReleaseRepository?releaseRepository = null,
IOptions <TableBuilderOptions>?options = null)
{
return(new(
statisticsDbContext,
filterItemRepository ?? Mock.Of <IFilterItemRepository>(Strict),
observationService ?? Mock.Of <IObservationService>(Strict),
statisticsPersistenceHelper ?? new PersistenceHelper <StatisticsDbContext>(statisticsDbContext),
resultSubjectMetaService ?? Mock.Of <IResultSubjectMetaService>(Strict),
subjectRepository ?? Mock.Of <ISubjectRepository>(Strict),
userService ?? AlwaysTrueUserService().Object,
resultBuilder ?? new ResultBuilder(),
releaseRepository ?? Mock.Of <IReleaseRepository>(Strict),
options ?? DefaultOptions()
));
}
public CarAdCreatedConsumer(
StatisticsDbContext data,
IMessageService messages)
{
this.data = data;
this.messages = messages;
}
public ReleaseService(
ContentDbContext context,
IMapper mapper,
IPersistenceHelper <ContentDbContext> persistenceHelper,
IUserService userService,
IReleaseRepository repository,
IReleaseFileRepository releaseFileRepository,
ISubjectRepository subjectRepository,
IReleaseDataFileService releaseDataFileService,
IReleaseFileService releaseFileService,
IDataImportService dataImportService,
IFootnoteService footnoteService,
StatisticsDbContext statisticsDbContext,
IDataBlockService dataBlockService,
IReleaseSubjectRepository releaseSubjectRepository,
IGuidGenerator guidGenerator,
IBlobCacheService cacheService)
{
_context = context;
_mapper = mapper;
_persistenceHelper = persistenceHelper;
_userService = userService;
_repository = repository;
_releaseFileRepository = releaseFileRepository;
_subjectRepository = subjectRepository;
_releaseDataFileService = releaseDataFileService;
_releaseFileService = releaseFileService;
_dataImportService = dataImportService;
_footnoteService = footnoteService;
_statisticsDbContext = statisticsDbContext;
_dataBlockService = dataBlockService;
_releaseSubjectRepository = releaseSubjectRepository;
_guidGenerator = guidGenerator;
_cacheService = cacheService;
}
private TopicService SetupTopicService(
ContentDbContext contentContext = null,
StatisticsDbContext statisticsContext = null,
PersistenceHelper <ContentDbContext> persistenceHelper = null,
IMapper mapper = null,
IUserService userService = null,
IReleaseSubjectRepository releaseSubjectRepository = null,
IReleaseDataFileService releaseDataFileService = null,
IReleaseFileService releaseFileService = null,
IPublishingService publishingService = null,
IMethodologyService methodologyService = null)
{
return(new TopicService(
Mock.Of <IConfiguration>(),
contentContext ?? Mock.Of <ContentDbContext>(),
statisticsContext ?? Mock.Of <StatisticsDbContext>(),
persistenceHelper ?? MockUtils.MockPersistenceHelper <ContentDbContext, Topic>(_topic.Id, _topic).Object,
mapper ?? AdminMapper(),
userService ?? MockUtils.AlwaysTrueUserService().Object,
releaseSubjectRepository ?? Mock.Of <IReleaseSubjectRepository>(),
releaseDataFileService ?? Mock.Of <IReleaseDataFileService>(),
releaseFileService ?? Mock.Of <IReleaseFileService>(),
publishingService ?? Mock.Of <IPublishingService>(),
methodologyService ?? Mock.Of <IMethodologyService>(),
Mock.Of <IBlobCacheService>()
));
}
public TopicService(
IConfiguration configuration,
ContentDbContext contentContext,
StatisticsDbContext statisticsContext,
IPersistenceHelper <ContentDbContext> persistenceHelper,
IMapper mapper,
IUserService userService,
IReleaseSubjectRepository releaseSubjectRepository,
IReleaseDataFileService releaseDataFileService,
IReleaseFileService releaseFileService,
IPublishingService publishingService,
IMethodologyService methodologyService,
IBlobCacheService cacheService)
{
_contentContext = contentContext;
_statisticsContext = statisticsContext;
_persistenceHelper = persistenceHelper;
_mapper = mapper;
_userService = userService;
_releaseSubjectRepository = releaseSubjectRepository;
_releaseDataFileService = releaseDataFileService;
_releaseFileService = releaseFileService;
_publishingService = publishingService;
_methodologyService = methodologyService;
_cacheService = cacheService;
_topicDeletionAllowed = configuration.GetValue <bool>("enableThemeDeletion");
}
public SubjectMetaService(
StatisticsDbContext context,
IFilterRepository filterRepository,
IFilterItemRepository filterItemRepository,
IIndicatorGroupRepository indicatorGroupRepository,
ILocationRepository locationRepository,
ILogger <SubjectMetaService> logger,
IObservationService observationService,
IPersistenceHelper <StatisticsDbContext> persistenceHelper,
ITimePeriodService timePeriodService,
IUserService userService,
IOptions <LocationsOptions> locationOptions) :
base(filterItemRepository)
{
_context = context;
_filterRepository = filterRepository;
_indicatorGroupRepository = indicatorGroupRepository;
_locationRepository = locationRepository;
_logger = logger;
_observationService = observationService;
_persistenceHelper = persistenceHelper;
_timePeriodService = timePeriodService;
_userService = userService;
_locationOptions = locationOptions.Value;
}
public void GetSubjectMeta_SubjectIdNotFound()
{
var builder = new DbContextOptionsBuilder <StatisticsDbContext>();
builder.UseInMemoryDatabase(Guid.NewGuid().ToString());
var options = builder.Options;
using (var context = new StatisticsDbContext(options, null))
{
var(boundaryLevelService, filterService, filterItemService, geoJsonService, indicatorGroupService, locationService, observationService, timePeriodService, userService) = Mocks();
var service = new SubjectMetaService(boundaryLevelService.Object,
filterService.Object,
filterItemService.Object,
geoJsonService.Object,
indicatorGroupService.Object,
locationService.Object,
new Mock <ILogger <SubjectMetaService> >().Object,
MapperUtils.MapperForProfile <DataServiceMappingProfiles>(),
observationService.Object,
new PersistenceHelper <StatisticsDbContext>(context),
timePeriodService.Object,
userService.Object);
var result = service.GetSubjectMeta(Guid.NewGuid());
Assert.True(result.Result.IsLeft);
Assert.IsAssignableFrom <NotFoundResult>(result.Result.Left);
}
}
public async Task ImportFiltersAndLocations(DataColumnCollection cols, DataRowCollection rows,
SubjectMeta subjectMeta, StatisticsDbContext context, Guid releaseId, string dataFileName)
{
// Clearing the caches is required here as the seeder shares the cache with all subjects
_importerFilterService.ClearCache();
_importerLocationService.ClearCache();
var headers = CsvUtil.GetColumnValues(cols);
var rowCount = 1;
var totalRows = rows.Count;
foreach (DataRow row in rows)
{
if (rowCount % STAGE_2_ROW_CHECK == 0)
{
var currentStatus = await _importStatusService.GetImportStatus(releaseId, dataFileName);
if (currentStatus.IsFinishedOrAborting())
{
_logger.LogInformation($"Import for {dataFileName} has finished or is being aborted, " +
$"so finishing importing Filters and Locations early");
return;
}
await _importStatusService.UpdateStatus(releaseId,
dataFileName,
IStatus.STAGE_2,
(double)rowCount / totalRows * 100);
}
CreateFiltersAndLocationsFromCsv(context, CsvUtil.GetRowValues(row), headers, subjectMeta.Filters);
rowCount++;
}
}
private Location LookupOrCreate(
StatisticsDbContext context,
Country country,
Institution institution = null,
LocalAuthority localAuthority = null,
LocalAuthorityDistrict localAuthorityDistrict = null,
LocalEnterprisePartnership localEnterprisePartnership = null,
MayoralCombinedAuthority mayoralCombinedAuthority = null,
Mat multiAcademyTrust = null,
OpportunityArea opportunityArea = null,
ParliamentaryConstituency parliamentaryConstituency = null,
Region region = null,
RscRegion rscRegion = null,
Sponsor sponsor = null,
Ward ward = null,
PlanningArea planningArea = null)
{
var location = Lookup(
context,
country,
institution,
localAuthority,
localAuthorityDistrict,
localEnterprisePartnership,
mayoralCombinedAuthority,
multiAcademyTrust,
opportunityArea,
parliamentaryConstituency,
region,
rscRegion,
sponsor,
ward,
planningArea);
if (location == null)
{
var entityEntry = context.Location.Add(new Location
{
Id = _guidGenerator.NewGuid(),
Country = country ?? Country.Empty(),
Institution = institution ?? Institution.Empty(),
LocalAuthority = localAuthority ?? LocalAuthority.Empty(),
LocalAuthorityDistrict = localAuthorityDistrict ?? LocalAuthorityDistrict.Empty(),
LocalEnterprisePartnership = localEnterprisePartnership ?? LocalEnterprisePartnership.Empty(),
MayoralCombinedAuthority = mayoralCombinedAuthority ?? MayoralCombinedAuthority.Empty(),
MultiAcademyTrust = multiAcademyTrust ?? Mat.Empty(),
OpportunityArea = opportunityArea ?? OpportunityArea.Empty(),
ParliamentaryConstituency = parliamentaryConstituency ?? ParliamentaryConstituency.Empty(),
Region = region ?? Region.Empty(),
RscRegion = rscRegion ?? RscRegion.Empty(),
Sponsor = sponsor ?? Sponsor.Empty(),
Ward = ward ?? Ward.Empty(),
PlanningArea = planningArea ?? PlanningArea.Empty()
});
return(entityEntry.Entity);
}
return(location);
}
private IEnumerable <Observation> GetObservations(
StatisticsDbContext context,
DataRowCollection rows,
List <string> headers,
Subject subject,
SubjectMeta subjectMeta,
int batchNo,
int rowsPerBatch
)
{
var observations = new List <Observation>();
var i = 0;
foreach (DataRow row in rows)
{
var o = ObservationFromCsv(
context,
CsvUtil.GetRowValues(row).ToArray(),
headers,
subject,
subjectMeta,
((batchNo - 1) * rowsPerBatch) + i++ + 2);
if (!IgnoredGeographicLevels.Contains(o.GeographicLevel))
{
observations.Add(o);
}
}
return(observations);
}
private static Subject CreateSubject(Guid subjectId,
string filename,
string name,
Release release,
StatisticsDbContext statisticsDbContext)
{
var newSubject = statisticsDbContext.Subject.Add(new Subject
{
Id = subjectId,
Filename = filename,
Name = name
}
).Entity;
statisticsDbContext.ReleaseSubject.Add(
new ReleaseSubject
{
ReleaseId = release.Id,
SubjectId = subjectId
});
statisticsDbContext.SaveChanges();
return(newSubject);
}
public BoundaryLevelService(StatisticsDbContext context,
DataServiceMemoryCache <BoundaryLevel> cache,
ILogger <BoundaryLevelService> logger)
: base(context, logger)
{
_cache = cache;
}
public virtual void Delete(Subject subject, StatisticsDbContext context)
{
// Use raw delete as EF can't correctly figure out how to cascade the delete, whilst the database can.
// N.B. This delete will be slow if there are a large number of observations but this is only
// executed by the tests when the topic is torn down so ensure files used are < 1000 rows.
context.Subject.FromSqlInterpolated($"DELETE Subject WHERE Id = {subject.Id}");
}
public BoundaryLevelRepository(
StatisticsDbContext context,
DataServiceMemoryCache <BoundaryLevel> cache)
: base(context)
{
_cache = cache;
}
public void GetThemes_TopicHasNoPublications()
{
var builder = new DbContextOptionsBuilder <StatisticsDbContext>();
builder.UseInMemoryDatabase(Guid.NewGuid().ToString());
var options = builder.Options;
using (var context = new StatisticsDbContext(options, null))
{
var theme = new Theme
{
Id = Guid.NewGuid(),
Title = "Theme",
Slug = "theme"
};
var topic = new Topic
{
Id = Guid.NewGuid(),
Title = "Topic",
Slug = "topic",
ThemeId = theme.Id
};
context.Add(theme);
context.Add(topic);
context.SaveChanges();
var service =
new ThemeMetaService(context, MapperUtils.MapperForProfile <DataServiceMappingProfiles>());
Assert.Empty(service.GetThemes());
}
}
public Location Find(
StatisticsDbContext context,
Country country,
Institution institution = null,
LocalAuthority localAuthority = null,
LocalAuthorityDistrict localAuthorityDistrict = null,
LocalEnterprisePartnership localEnterprisePartnership = null,
MayoralCombinedAuthority mayoralCombinedAuthority = null,
Mat multiAcademyTrust = null,
OpportunityArea opportunityArea = null,
ParliamentaryConstituency parliamentaryConstituency = null,
Region region = null,
RscRegion rscRegion = null,
Sponsor sponsor = null,
Ward ward = null,
PlanningArea planningArea = null)
{
var cacheKey = GetCacheKey(country, institution, localAuthority, localAuthorityDistrict,
localEnterprisePartnership, mayoralCombinedAuthority, multiAcademyTrust, opportunityArea,
parliamentaryConstituency, region, rscRegion, sponsor, ward, planningArea);
if (GetCache().TryGetValue(cacheKey, out Location location))
{
return(location);
}
location = LookupOrCreate(context, country, institution, localAuthority, localAuthorityDistrict,
localEnterprisePartnership, mayoralCombinedAuthority, multiAcademyTrust, opportunityArea,
parliamentaryConstituency, region, rscRegion, sponsor, ward, planningArea);
GetCache().Set(cacheKey, location);
return(location);
}
private ReleaseDataFileService SetupReleaseDataFileService(
ContentDbContext contentDbContext = null,
StatisticsDbContext statisticsDbContext = null,
IPersistenceHelper <ContentDbContext> contentPersistenceHelper = null,
IBlobStorageService blobStorageService = null,
IDataArchiveValidationService dataArchiveValidationService = null,
IFileUploadsValidatorService fileUploadsValidatorService = null,
IFileRepository fileRepository = null,
IReleaseFileRepository releaseFileRepository = null,
IImportService importService = null,
IImportStatusService importStatusService = null,
IUserService userService = null)
{
return(new ReleaseDataFileService(
contentDbContext ?? new Mock <ContentDbContext>().Object,
statisticsDbContext ?? new Mock <StatisticsDbContext>().Object,
contentPersistenceHelper ?? DefaultPersistenceHelperMock().Object,
blobStorageService ?? new Mock <IBlobStorageService>().Object,
dataArchiveValidationService ?? new Mock <IDataArchiveValidationService>().Object,
fileUploadsValidatorService ?? new Mock <IFileUploadsValidatorService>().Object,
fileRepository ?? new FileRepository(contentDbContext),
releaseFileRepository ?? new ReleaseFileRepository(contentDbContext),
importService ?? new Mock <IImportService>().Object,
importStatusService ?? new Mock <IImportStatusService>().Object,
userService ?? new Mock <IUserService>().Object
));
}
public async Task ImportObservations(ImportObservationsMessage message, StatisticsDbContext context)
{
var import = await _dataImportService.GetImport(message.Id);
_logger.LogInformation($"Importing Observations for {import.File.Filename} batchNo {message.BatchNo}");
if (import.Status.IsFinished())
{
_logger.LogInformation($"Import for {import.File.Filename} already finished with state " +
$"{import.Status} - ignoring Observations in file {message.ObservationsFilePath}");
return;
}
if (import.Status == CANCELLING)
{
_logger.LogInformation($"Import for {import.File.Filename} is " +
$"{import.Status} - ignoring Observations in file {message.ObservationsFilePath} " +
"and marking import as CANCELLED");
await _dataImportService.UpdateStatus(message.Id, CANCELLED, 100);
return;
}
var subject = await context.Subject.FindAsync(import.SubjectId);
var datafileStream = await _blobStorageService.StreamBlob(PrivateReleaseFiles, message.ObservationsFilePath);
var dataFileTable = DataTableUtils.CreateFromStream(datafileStream);
var metaFileStream = await _blobStorageService.StreamBlob(PrivateReleaseFiles, import.MetaFile.Path());
var metaFileTable = DataTableUtils.CreateFromStream(metaFileStream);
await context.Database.CreateExecutionStrategy().Execute(async() =>
{
await using var transaction = await context.Database.BeginTransactionAsync();
await _importerService.ImportObservations(
import,
dataFileTable.Columns,
dataFileTable.Rows,
subject,
_importerService.GetMeta(metaFileTable, subject, context),
message.BatchNo,
context
);
await transaction.CommitAsync();
await context.Database.CloseConnectionAsync();
});
if (import.NumBatches > 1)
{
await _blobStorageService.DeleteBlob(PrivateReleaseFiles, message.ObservationsFilePath);
}
await CheckComplete(message, context);
}
public SyncMatchesCommandHandler(ServerStatsDbContext serverStatsDbContext, StatisticsDbContext statsDbContext,
ISteamService steamService, IHubContext <StatisticsHub> hubContext)
{
_serverStatsDbContext = serverStatsDbContext;
_statsDbContext = statsDbContext;
_steamService = steamService;
_hubContext = hubContext;
}
public ReleaseSubjectService(
StatisticsDbContext statisticsDbContext,
IFootnoteRepository footnoteRepository,
SubjectDeleter subjectDeleter)
{
_statisticsDbContext = statisticsDbContext;
_footnoteRepository = footnoteRepository;
_subjectDeleter = subjectDeleter;
}
public ReleaseSubjectRepository(
StatisticsDbContext statisticsDbContext,
IFootnoteRepository footnoteRepository,
SubjectDeleter?subjectDeleter = null)
{
_statisticsDbContext = statisticsDbContext;
_footnoteRepository = footnoteRepository;
_subjectDeleter = subjectDeleter ?? new SubjectDeleter();
}
BuildServiceAndMocks(
ContentDbContext contentDbContext,
StatisticsDbContext statisticsDbContext)
{
var fastTrackService = new Mock <IFastTrackService>(Strict);
var controller = new CacheKeyService(contentDbContext, statisticsDbContext, fastTrackService.Object);
return(controller, (fastTrackService));
}
private IEnumerable <(Filter Filter, string Column, string FilterGroupingColumn)> ImportFilters(
IEnumerable <MetaRow> metaRows, Subject subject, StatisticsDbContext context)
{
var filters = GetFilters(metaRows, subject, context).ToList();
context.Filter.AddRange(filters.Select(triple => triple.Filter));
return(filters);
}
public ReleaseRepository(
ContentDbContext contentDbContext,
StatisticsDbContext statisticsDbContext,
IMapper mapper)
{
_contentDbContext = contentDbContext;
_statisticsDbContext = statisticsDbContext;
_mapper = mapper;
}
public TaxonomyService(
ContentDbContext contentDbContext,
StatisticsDbContext statisticsDbContext,
IMapper mapper)
{
_contentDbContext = contentDbContext;
_statisticsDbContext = statisticsDbContext;
_mapper = mapper;
}
public SubjectBelongsToViewableReleaseAuthorizationHandler(
ContentDbContext contentDbContext,
StatisticsDbContext statisticsDbContext,
IPreReleaseService preReleaseService)
{
_contentDbContext = contentDbContext;
_statisticsDbContext = statisticsDbContext;
_preReleaseService = preReleaseService;
}
public CacheKeyService(
ContentDbContext contentDbContext,
StatisticsDbContext statisticsDbContext,
IFastTrackService fastTrackService)
{
_contentDbContext = contentDbContext;
_statisticsDbContext = statisticsDbContext;
_fastTrackService = fastTrackService;
}
public ViewSubjectDataAuthorizationHandler(
ContentDbContext contentDbContext,
StatisticsDbContext statisticsDbContext,
IPreReleaseService preReleaseService) : base(
new ViewSubjectDataForPublishedReleasesAuthorizationHandler(statisticsDbContext),
new SubjectBelongsToViewableReleaseAuthorizationHandler(contentDbContext, statisticsDbContext, preReleaseService)
)
{
}
private static SubjectRepository BuildSubjectService(
StatisticsDbContext statisticsDbContext,
IReleaseRepository?releaseRepository = null)
{
return(new SubjectRepository(
statisticsDbContext,
releaseRepository ?? new Mock <IReleaseRepository>().Object
));
}
public void GetSubjectMeta_EmptyModelReturnedForSubject()
{
var builder = new DbContextOptionsBuilder <StatisticsDbContext>();
builder.UseInMemoryDatabase(Guid.NewGuid().ToString());
var options = builder.Options;
using (var context = new StatisticsDbContext(options, null))
{
var release = new Release
{
Id = Guid.NewGuid(),
};
var subject = new Subject
{
Id = Guid.NewGuid()
};
var releaseSubjectLink = new ReleaseSubject
{
ReleaseId = release.Id,
SubjectId = subject.Id
};
context.Add(release);
context.Add(subject);
context.Add(releaseSubjectLink);
context.SaveChanges();
var(boundaryLevelService, filterService, filterItemService, geoJsonService, indicatorGroupService, locationService, observationService, timePeriodService, userService) = Mocks();
filterService.Setup(s => s.GetFiltersIncludingItems(subject.Id)).Returns(Enumerable.Empty <Filter>());
indicatorGroupService.Setup(s => s.GetIndicatorGroups(subject.Id)).Returns(Enumerable.Empty <IndicatorGroup>());
locationService.Setup(s => s.GetObservationalUnits(subject.Id)).Returns(new Dictionary <GeographicLevel, IEnumerable <IObservationalUnit> >());
timePeriodService.Setup(s => s.GetTimePeriods(subject.Id)).Returns(Enumerable.Empty <(int Year, TimeIdentifier TimeIdentifier)>());
var service = new SubjectMetaService(boundaryLevelService.Object,
filterService.Object,
filterItemService.Object,
geoJsonService.Object,
indicatorGroupService.Object,
locationService.Object,
new Mock <ILogger <SubjectMetaService> >().Object,
MapperUtils.MapperForProfile <DataServiceMappingProfiles>(),
observationService.Object,
new PersistenceHelper <StatisticsDbContext>(context),
timePeriodService.Object,
userService.Object);
var result = service.GetSubjectMeta(subject.Id);
Assert.True(result.Result.IsRight);
Assert.IsAssignableFrom <SubjectMetaViewModel>(result.Result.Right);
}
}
