/// <inheritdoc /> public async Task CreateDocumentMetadataAsync( int establishmentId, string establishmentName, FileTypeOption fileType, string fileName, string fileUrl, CancellationToken cancellationToken) { byte fileTypeId = (byte)fileType; DateTime siteVisitDate = DateTime.Parse(DefaultSiteVisitDate); var parameters = new { EstablishmentID = establishmentId, SupplierKeyID = DefaultSupplierKeyIDValue, EstablishmentName = establishmentName, FileTypeID = fileTypeId, SiteVisitDate = siteVisitDate, FileName = fileName, FileURL = fileUrl, }; using (SqlConnection sqlConnection = await this.GetOpenConnectionAsync(cancellationToken).ConfigureAwait(false)) { await sqlConnection.ExecuteAsync( "sp_INSERT_FileData", parameters, commandType : CommandType.StoredProcedure) .ConfigureAwait(false); } }
private async Task ProcessArchivedReport( Establishment establishment, DocumentFile documentFile, List <string> usedFileNames, bool appendToDirectoryIfNotExists, CancellationToken cancellationToken) { string name = establishment.Name; bool append = ShouldAppend( appendToDirectoryIfNotExists, usedFileNames, name); if (append) { string filename = name + DestinationReportFileExtension; filename = this.GenerateUniqueName( filename, " ", DestinationReportFileExtension, usedFileNames); byte[] reportBytes = await this.DownloadAndUnzip( documentFile, cancellationToken) .ConfigureAwait(false); Uri uri = await this.SendFileToDestinationStorage( establishment, DestinationReportSubDirectory, filename, DestinationReportMimeType, reportBytes, cancellationToken) .ConfigureAwait(false); FileTypeOption fileType = FileTypeOption.Report; await this.InsertMetaData( establishment, fileType, uri, name, filename, cancellationToken) .ConfigureAwait(false); } else { this.loggerWrapper.Warning( $"A file already exists with a name similar to " + $"\"{name}\", and " + $"{nameof(appendToDirectoryIfNotExists)} = false. " + $"Therefore, this file will be ignored."); } }
public async Task GetFileAsync_OneMetaDataRecordFoundForUrnFileExists_ReturnsOriginalFile() { // Arrange int urn = 1234; int[] fallbackUrns = null; FileTypeOption fileType = FileTypeOption.Report; CancellationToken cancellationToken = CancellationToken.None; Uri location = new Uri( "https://some-storage-container.azure.example/exists.png", UriKind.Absolute); FileMetaData fileMetaData = new FileMetaData() { Location = location, }; IEnumerable <FileMetaData> fileMetaDatas = new FileMetaData[] { fileMetaData, }; this.mockFileMetaDataAdapter .Setup(x => x.GetFileMetaDatasAsync(It.IsAny <int>(), It.IsAny <FileTypeOption>(), It.IsAny <CancellationToken>())) .Returns(Task.FromResult(fileMetaDatas)); Random random = new Random(); byte[] contentBytes = new byte[2048]; random.NextBytes(contentBytes); File expectedFile = new File() { FileName = "exists.png", ContentType = "image/png", ContentBytes = contentBytes, }; this.mockFileStorageAdapter .Setup(x => x.GetFileAsync(It.IsAny <FileMetaData>(), It.IsAny <CancellationToken>())) .Returns(Task.FromResult(expectedFile)); File actualFile = null; // Act actualFile = await this.fileManager.GetFileAsync( urn, fileType, fallbackUrns, cancellationToken); // Assert Assert.AreEqual(actualFile, expectedFile); }
private async Task <IDictionary <int, IEnumerable <File> > > GetUrnFilesAsync( int[] urns, FileTypeOption fileType, CancellationToken cancellationToken) { Dictionary <int, IEnumerable <File> > toReturn = new Dictionary <int, IEnumerable <File> >(); foreach (int urn in urns) { // First, get the meta-data (i.e. location to files). this.loggerProvider.Debug( $"Pulling back file list for {nameof(urn)} = {urn} and " + $"{nameof(fileType)} = {fileType}..."); IEnumerable <FileMetaData> fileMetaDatas = await this.fileMetaDataAdapter.GetFileMetaDatasAsync( urn, fileType, cancellationToken) .ConfigureAwait(false); this.loggerProvider.Info( $"Returned {fileMetaDatas.Count()} result(s)."); // Then, actually pull the files. List <File> files = new List <File>(); File file = null; foreach (FileMetaData fileMetaData in fileMetaDatas) { this.loggerProvider.Debug($"Pulling {fileMetaData}..."); file = await this.fileStorageAdapter.GetFileAsync( fileMetaData, cancellationToken) .ConfigureAwait(false); if (file != null) { this.loggerProvider.Info($"Pulled {file}."); files.Add(file); } else { this.loggerProvider.Warning( $"No file could be found for {fileMetaData}!"); } } toReturn.Add(urn, files); } return(toReturn); }
private async Task <IActionResult> GetFileAsync( int urn, FileTypeOption fileType, int[] fallbackUrns, CancellationToken cancellationToken) { IActionResult toReturn = null; Domain.Models.File file = await this.fileManager.GetFileAsync( urn, fileType, fallbackUrns, cancellationToken) .ConfigureAwait(false); if (file != null) { this.loggerProvider.Info( $"The method " + $"{nameof(IFileManager)}.{nameof(IFileManager.GetFileAsync)} " + $"method returned {file} for {nameof(urn)} = \"{urn}\" " + $"and {nameof(fileType)} = \"{fileType}\" - returning " + $"with a {nameof(FileContentResult)}."); byte[] contentBytes = file.ContentBytes.ToArray(); string contentType = file.ContentType; FileContentResult fileContentResult = new FileContentResult( contentBytes, contentType); fileContentResult.FileDownloadName = file.FileName; toReturn = fileContentResult; } else { this.loggerProvider.Warning( $"The method " + $"{nameof(IFileManager)}.{nameof(IFileManager.GetFileAsync)} " + $"method returned null for {nameof(urn)} = \"{urn}\" " + $"and {nameof(fileType)} = \"{fileType}\" - returning " + $"{nameof(NotFoundResult)}."); toReturn = new NotFoundResult(); } return(toReturn); }
private async Task <IDictionary <int, IEnumerable <File> > > GetUrnFilesAsync( int urn, FileTypeOption fileType, CancellationToken cancellationToken) { IDictionary <int, IEnumerable <File> > toReturn = null; int[] urns = new int[] { urn }; toReturn = await this.GetUrnFilesAsync( urns, fileType, cancellationToken) .ConfigureAwait(false); return(toReturn); }
public async Task GetFileAsync_NoMetaDataFoundForUrnOrFallbackUrns_ReturnsNull() { // Arrange int urn = 1234; FileTypeOption fileType = FileTypeOption.Report; int[] fallbackUrns = new int[] { 4567, 890 }; CancellationToken cancellationToken = CancellationToken.None; File file = null; // Act file = await this.fileManager.GetFileAsync( urn, fileType, fallbackUrns, cancellationToken); // Assert Assert.IsNull(file); }
private async Task ProcessSitePlanZip( Establishment establishment, DocumentFile documentFile, List <string> usedFileNames, CancellationToken cancellationToken) { byte[] sitePlanBytes = await this.DownloadAndUnzip( documentFile, cancellationToken) .ConfigureAwait(false); string name = establishment.Name; string filename = name + DestinationSitePlanFileExtension; filename = this.GenerateUniqueName( filename, " ", DestinationSitePlanFileExtension, usedFileNames); Uri uri = await this.SendFileToDestinationStorage( establishment, DestinationSitePlanSubDirectory, filename, DestinationSitePlanMimeType, sitePlanBytes, cancellationToken) .ConfigureAwait(false); FileTypeOption fileType = FileTypeOption.SitePlan; await this.InsertMetaData( establishment, fileType, uri, name, filename, cancellationToken) .ConfigureAwait(false); }
public async Task GetFileAsync_OneMetaDataRecordFoundForUrnFileMissing_ReturnsNull() { // Arrange int urn = 1234; FileTypeOption fileType = FileTypeOption.Report; int[] fallbackUrns = null; CancellationToken cancellationToken = CancellationToken.None; Uri location = new Uri( "https://some-storage-container.azure.example/doesntexist.pdf", UriKind.Absolute); FileMetaData fileMetaData = new FileMetaData() { Location = location, }; IEnumerable <FileMetaData> fileMetaDatas = new FileMetaData[] { fileMetaData, }; this.mockFileMetaDataAdapter .Setup(x => x.GetFileMetaDatasAsync(It.IsAny <int>(), It.IsAny <FileTypeOption>(), It.IsAny <CancellationToken>())) .Returns(Task.FromResult(fileMetaDatas)); File file = null; // Act file = await this.fileManager.GetFileAsync( urn, fileType, fallbackUrns, cancellationToken); // Assert Assert.IsNull(file); }
private async Task InsertMetaData( Establishment establishment, FileTypeOption fileType, Uri uri, string name, string filename, CancellationToken cancellationToken) { // Then insert the metadata. string[] segments = uri.Segments; string fileName = segments.Last(); string fileNameDecoded = HttpUtility.UrlDecode(filename); string fileUrl = uri.AbsoluteUri.Replace(fileName, string.Empty); await this.documentMetadataAdapter.CreateDocumentMetadataAsync( establishment.Urn.Value, name, fileType, fileNameDecoded, fileUrl, cancellationToken) .ConfigureAwait(false); }
public async Task GetFileAsync_NoResultsForPrimaryUrnButResultsForFallbackUrns_ReturnsZip() { // Arrange int urn = 1234; int[] fallbackUrns = new int[] { 3456, 7890 }; FileTypeOption fileType = FileTypeOption.Report; CancellationToken cancellationToken = CancellationToken.None; Func <int, FileTypeOption, CancellationToken, Task <IEnumerable <FileMetaData> > > getFileMetaDataCallback = (urn, fileType, cancellationToken) => { IEnumerable <FileMetaData> results = null; // Only return results for a fallback. if (fallbackUrns.Contains(urn)) { results = new string[] { $"{urn} docs/report-{urn}.pdf", $"{urn} docs/{Guid.NewGuid()}.docx", } .Select(x => $"https://some-storage-container.azure.example/{x}") .Select(x => new Uri(x, UriKind.Absolute)) .Select(x => new FileMetaData() { Location = x }) .ToArray(); } else { results = Array.Empty <FileMetaData>(); } return(Task.FromResult(results)); }; this.mockFileMetaDataAdapter .Setup(x => x.GetFileMetaDatasAsync(It.IsAny <int>(), It.IsAny <FileTypeOption>(), It.IsAny <CancellationToken>())) .Returns(getFileMetaDataCallback); Random random = new Random(); byte[] contentBytes = null; File file = null; Func <FileMetaData, CancellationToken, Task <File> > getFileAsyncCallback = (fmd, ct) => { contentBytes = new byte[random.Next(1024, 2048)]; random.NextBytes(contentBytes); file = new File() { FileName = fmd.Location.Segments.Last(), ContentType = "application/pdf", ContentBytes = contentBytes, }; return(Task.FromResult(file)); }; this.mockFileStorageAdapter .Setup(x => x.GetFileAsync(It.IsAny <FileMetaData>(), It.IsAny <CancellationToken>())) .Returns(getFileAsyncCallback); File actualFile = null; string expectedContentType = "application/zip"; string actualContentType = null; string expectedFilename = $"{urn} files.zip"; string actualFilename = null; IEnumerable <ZipArchiveEntry> zipArchiveEntries = null; int expectedNumberOfArchiveEntries = 4; int actualNumberOfArchiveEntries; // Act actualFile = await this.fileManager.GetFileAsync( urn, fileType, fallbackUrns, cancellationToken); // Assert actualContentType = actualFile.ContentType; Assert.AreEqual(expectedContentType, actualContentType); actualFilename = actualFile.FileName; Assert.AreEqual(expectedFilename, actualFilename); // -> Actually open the zip to check that the bytes are all good. zipArchiveEntries = this.ExtractZipEntries(actualFile); actualNumberOfArchiveEntries = zipArchiveEntries.Count(); Assert.AreEqual( expectedNumberOfArchiveEntries, actualNumberOfArchiveEntries); }
public async Task GetFileAsync_MultipleMetaDataRecordsFoundForUrnAllFilesExist_ReturnsZip() { // Arrange int urn = 1234; int[] fallbackUrns = null; FileTypeOption fileType = FileTypeOption.Report; CancellationToken cancellationToken = CancellationToken.None; string[] exampleDocs = new string[] { "doc1.pdf", "doc2.pdf", "doc3.pdf" }; IEnumerable <FileMetaData> fileMetaDatas = exampleDocs .Select(x => $"https://some-storage-container.azure.example/{x}") .Select(x => new Uri(x, UriKind.Absolute)) .Select(x => new FileMetaData() { Location = x }) .ToArray(); this.mockFileMetaDataAdapter .Setup(x => x.GetFileMetaDatasAsync(It.IsAny <int>(), It.IsAny <FileTypeOption>(), It.IsAny <CancellationToken>())) .Returns(Task.FromResult(fileMetaDatas)); Random random = new Random(); byte[] contentBytes = null; File file = null; Func <FileMetaData, CancellationToken, Task <File> > getFileAsyncCallback = (fmd, ct) => { contentBytes = new byte[random.Next(1024, 2048)]; random.NextBytes(contentBytes); file = new File() { FileName = fmd.Location.Segments.Last(), ContentType = "application/pdf", ContentBytes = contentBytes, }; return(Task.FromResult(file)); }; this.mockFileStorageAdapter .Setup(x => x.GetFileAsync(It.IsAny <FileMetaData>(), It.IsAny <CancellationToken>())) .Returns(getFileAsyncCallback); File actualFile = null; string expectedContentType = "application/zip"; string actualContentType = null; string expectedFilename = $"{urn} files.zip"; string actualFilename = null; IEnumerable <ZipArchiveEntry> zipArchiveEntries = null; int expectedNumberOfArchiveEntries = 3; int actualNumberOfArchiveEntries; // Act actualFile = await this.fileManager.GetFileAsync( urn, fileType, fallbackUrns, cancellationToken); // Assert actualContentType = actualFile.ContentType; Assert.AreEqual(expectedContentType, actualContentType); actualFilename = actualFile.FileName; Assert.AreEqual(expectedFilename, actualFilename); // -> Actually open the zip to check that the bytes are all good. zipArchiveEntries = this.ExtractZipEntries(actualFile); actualNumberOfArchiveEntries = zipArchiveEntries.Count(); Assert.AreEqual( expectedNumberOfArchiveEntries, actualNumberOfArchiveEntries); }
/// <inheritdoc /> public async Task <IEnumerable <FileMetaData> > GetFileMetaDatasAsync( int establishmentId, FileTypeOption fileType, CancellationToken cancellationToken) { IEnumerable <FileMetaData> toReturn = null; byte typeId = (byte)fileType; var parameters = new { EstablishmentID = establishmentId, FileTypeID = typeId, }; IEnumerable <GetFileListResult> getFileListResults = null; using (SqlConnection sqlConnection = await this.GetOpenConnectionAsync(cancellationToken).ConfigureAwait(false)) { this.loggerProvider.Debug( $"Executing \"{SprocNameGetFileList}\" with " + $"{nameof(establishmentId)} = {establishmentId}, " + $"{nameof(typeId)} = {typeId}..."); getFileListResults = await sqlConnection.QueryAsync <GetFileListResult>( "sp_SELECT_FileData", parameters, commandType : CommandType.StoredProcedure) .ConfigureAwait(false); this.loggerProvider.Info( $"{getFileListResults.Count()} result(s) returned."); } if (getFileListResults.Any()) { // Quiz the SiteVisitDate column. Get the most recent date. DateTime mostRecentDate = getFileListResults .Select(x => x.SiteVisitDate) .Max(); this.loggerProvider.Debug( $"Mapping all results with " + $"{nameof(GetFileListResult.SiteVisitDate)} = " + $"\"{mostRecentDate}\"..."); toReturn = getFileListResults .Where(x => x.SiteVisitDate == mostRecentDate) .Select(x => { FileMetaData fileMetaData = null; string fileUrl = x.FileURL; string fileName = x.FileName; Uri baseUri = new Uri(fileUrl, UriKind.Absolute); Uri relativeUri = new Uri(fileName, UriKind.Relative); Uri location = new Uri(baseUri, relativeUri); fileMetaData = new FileMetaData() { Location = location, }; return(fileMetaData); }); } else { toReturn = Array.Empty <FileMetaData>(); this.loggerProvider.Info( $"No records found for {nameof(establishmentId)} = " + $"{establishmentId} and {nameof(fileType)} = {fileType}."); } return(toReturn); }
public async Task <File> GetFileAsync( int urn, FileTypeOption fileType, int[] fallbackUrns, CancellationToken cancellationToken) { File toReturn = null; this.loggerProvider.Debug( $"Firstly, searching for files under {nameof(urn)} {urn}..."); IDictionary <int, IEnumerable <File> > fileSearch = await this.GetUrnFilesAsync(urn, fileType, cancellationToken) .ConfigureAwait(false); IEnumerable <File> files = fileSearch[urn]; if (files.Any()) { if (files.Count() == 1) { // Then just return this one file. toReturn = files.Single(); this.loggerProvider.Info( $"Only one {nameof(File)} available ({toReturn}). " + $"Returning this {nameof(File)}, as-is."); } else { this.loggerProvider.Debug( $"More than one file available ({files.Count()} in " + $"total). Zipping up..."); // Else, there's more than one, and we'll need to zip them // up, then return them. toReturn = this.ZipMultipleFiles(urn, files); this.loggerProvider.Info( $"Returning zip {nameof(File)}: {toReturn}."); } } else { this.loggerProvider.Warning( $"Could not find any files for {nameof(urn)} = {urn} " + $"and {nameof(fileType)} = {fileType}."); if (fallbackUrns != null) { this.loggerProvider.Info( "Fallback URNs were provided, however. Pulling back " + "files for fallback URNs..."); fileSearch = await this.GetUrnFilesAsync( fallbackUrns, fileType, cancellationToken) .ConfigureAwait(false); // We only want to return a file (i.e. zip) if there are // files in our search results to return. bool fallbackFilesAvailable = fileSearch .SelectMany(x => x.Value) .Any(); if (fallbackFilesAvailable) { toReturn = this.ZipMultipleFiles( urn, fileSearch); } else { this.loggerProvider.Info( $"No files available for the primary " + $"{nameof(urn)} = {urn}, or for any " + $"{nameof(fallbackUrns)}. Null will be returned."); } } } return(toReturn); }
public async Task <IActionResult> RunAsync( [HttpTrigger(AuthorizationLevel.Function, "GET", Route = "cdc-file/{urn}")] HttpRequest httpRequest, int urn, CancellationToken cancellationToken) { IActionResult toReturn = null; this.loggerProvider.Debug( $"File requested. {nameof(urn)} = \"{urn}\"."); // urn never can be null; if someone tries to call ./cdc-file or // ./cdc-file/, a 404 is returned by default. if (httpRequest == null) { throw new ArgumentNullException(nameof(httpRequest)); } // The following header is optional. // If the requested file is not available, then fetch the files // contained within this CSL. IHeaderDictionary headers = httpRequest.Headers; int[] fallbackUrns = null; if (headers.ContainsKey(FallbackUrnsHeaderName)) { string fallbackUrnsCsl = headers[FallbackUrnsHeaderName]; this.loggerProvider.Debug( $"Header \"{FallbackUrnsHeaderName}\" was specified: " + $"\"{fallbackUrnsCsl}\". This should be a CSL of " + $"integers. Parsing..."); string[] fallbackUrnsStr = fallbackUrnsCsl.Split(','); // Try and parse each one. try { fallbackUrns = fallbackUrnsStr .Select(x => x.Trim()) .Select(x => int.Parse(x, CultureInfo.InvariantCulture)) .ToArray(); this.loggerProvider.Info( $"Parsed {fallbackUrns.Length} fallback URNs."); } catch (FormatException) { this.loggerProvider.Warning( $"Could not parse header value for " + $"\"{FallbackUrnsHeaderName}\" into an array of " + $"{nameof(Int32)} values. Original string: " + $"\"{fallbackUrnsCsl}\"."); toReturn = new BadRequestResult(); } } if (toReturn == null) { string typeStr = httpRequest.Query["type"]; // We need a null-check on type. if (!string.IsNullOrEmpty(typeStr)) { this.loggerProvider.Debug( $"{nameof(typeStr)} = \"{typeStr}\""); if (this.fileTypeMap.ContainsKey(typeStr)) { FileTypeOption fileType = this.fileTypeMap[typeStr]; toReturn = await this.GetFileAsync( urn, fileType, fallbackUrns, cancellationToken) .ConfigureAwait(false); } else { string[] validTypes = this.fileTypeMap.Keys .Select(x => $"\"{x}\"") .ToArray(); string validTypesList = string.Join(", ", validTypes); this.loggerProvider.Warning( $"The type was supplied was not valid. This " + $"needs to be one of the following values: " + $"{validTypesList}. Returning " + $"{nameof(BadRequestResult)}."); // Return "bad request" to indicate we need it. toReturn = new BadRequestResult(); } } else { this.loggerProvider.Warning( $"The type was not supplied. This is required. " + $"Returning {nameof(BadRequestResult)}."); // Return "bad request" to indicate we need it. toReturn = new BadRequestResult(); } } return(toReturn); }
private async Task ProcessSitePlanZip( Establishment establishment, DocumentFile documentFile, List <string> usedFileNames, CancellationToken cancellationToken) { byte[] sitePlanBytes = await this.DownloadAndUnzip( documentFile, cancellationToken) .ConfigureAwait(false); if (sitePlanBytes.Length < 10000) { // probably a text file, skip it this.loggerWrapper.Info($"{documentFile.Name} with mime type octet-stream is less than 10kb, skipping"); return; } string name = establishment.Name; string filename = name + DestinationSitePlanFileExtension; // Attempt to get filename from evidence data string idSegment = this.GetIdFromName(documentFile.Name); string evidenceName = string.Empty; if (idSegment != string.Empty) { var keyExists = this._cdc1Evidence.TryGetValue(idSegment.ToLower(), out evidenceName); if (keyExists) { this.loggerWrapper.Info($"Evidence name: {evidenceName}"); } else { this.loggerWrapper.Info($"Entry not found for id {idSegment}, skipping file"); return; } evidenceName += this.GetFileExtension(documentFile.Name); evidenceName = this.StripIllegalCharacters(evidenceName); } Uri uri = await this.SendFileToDestinationStorage( establishment, DestinationSitePlanSubDirectory, evidenceName, DestinationSitePlanMimeType, sitePlanBytes, cancellationToken) .ConfigureAwait(false); FileTypeOption fileType = FileTypeOption.SitePlan; await this.InsertMetaData( establishment, fileType, uri, name, evidenceName, cancellationToken) .ConfigureAwait(false); }