/// <inheritdoc />
public async Task CreateDocumentMetadataAsync(
int establishmentId,
string establishmentName,
FileTypeOption fileType,
string fileName,
string fileUrl,
CancellationToken cancellationToken)
{
byte fileTypeId = (byte)fileType;
DateTime siteVisitDate = DateTime.Parse(DefaultSiteVisitDate);
var parameters = new
{
EstablishmentID = establishmentId,
SupplierKeyID = DefaultSupplierKeyIDValue,
EstablishmentName = establishmentName,
FileTypeID = fileTypeId,
SiteVisitDate = siteVisitDate,
FileName = fileName,
FileURL = fileUrl,
};
using (SqlConnection sqlConnection = await this.GetOpenConnectionAsync(cancellationToken).ConfigureAwait(false))
{
await sqlConnection.ExecuteAsync(
"sp_INSERT_FileData",
parameters,
commandType : CommandType.StoredProcedure)
.ConfigureAwait(false);
}
}
private async Task ProcessArchivedReport(
Establishment establishment,
DocumentFile documentFile,
List <string> usedFileNames,
bool appendToDirectoryIfNotExists,
CancellationToken cancellationToken)
{
string name = establishment.Name;
bool append = ShouldAppend(
appendToDirectoryIfNotExists,
usedFileNames,
name);
if (append)
{
string filename = name + DestinationReportFileExtension;
filename = this.GenerateUniqueName(
filename,
" ",
DestinationReportFileExtension,
usedFileNames);
byte[] reportBytes = await this.DownloadAndUnzip(
documentFile,
cancellationToken)
.ConfigureAwait(false);
Uri uri = await this.SendFileToDestinationStorage(
establishment,
DestinationReportSubDirectory,
filename,
DestinationReportMimeType,
reportBytes,
cancellationToken)
.ConfigureAwait(false);
FileTypeOption fileType = FileTypeOption.Report;
await this.InsertMetaData(
establishment,
fileType,
uri,
name,
filename,
cancellationToken)
.ConfigureAwait(false);
}
else
{
this.loggerWrapper.Warning(
$"A file already exists with a name similar to " +
$"\"{name}\", and " +
$"{nameof(appendToDirectoryIfNotExists)} = false. " +
$"Therefore, this file will be ignored.");
}
}
public async Task GetFileAsync_OneMetaDataRecordFoundForUrnFileExists_ReturnsOriginalFile()
{
// Arrange
int urn = 1234;
int[] fallbackUrns = null;
FileTypeOption fileType = FileTypeOption.Report;
CancellationToken cancellationToken = CancellationToken.None;
Uri location = new Uri(
"https://some-storage-container.azure.example/exists.png",
UriKind.Absolute);
FileMetaData fileMetaData = new FileMetaData()
{
Location = location,
};
IEnumerable <FileMetaData> fileMetaDatas = new FileMetaData[]
{
fileMetaData,
};
this.mockFileMetaDataAdapter
.Setup(x => x.GetFileMetaDatasAsync(It.IsAny <int>(), It.IsAny <FileTypeOption>(), It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(fileMetaDatas));
Random random = new Random();
byte[] contentBytes = new byte[2048];
random.NextBytes(contentBytes);
File expectedFile = new File()
{
FileName = "exists.png",
ContentType = "image/png",
ContentBytes = contentBytes,
};
this.mockFileStorageAdapter
.Setup(x => x.GetFileAsync(It.IsAny <FileMetaData>(), It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(expectedFile));
File actualFile = null;
// Act
actualFile = await this.fileManager.GetFileAsync(
urn,
fileType,
fallbackUrns,
cancellationToken);
// Assert
Assert.AreEqual(actualFile, expectedFile);
}
private async Task <IDictionary <int, IEnumerable <File> > > GetUrnFilesAsync(
int[] urns,
FileTypeOption fileType,
CancellationToken cancellationToken)
{
Dictionary <int, IEnumerable <File> > toReturn =
new Dictionary <int, IEnumerable <File> >();
foreach (int urn in urns)
{
// First, get the meta-data (i.e. location to files).
this.loggerProvider.Debug(
$"Pulling back file list for {nameof(urn)} = {urn} and " +
$"{nameof(fileType)} = {fileType}...");
IEnumerable <FileMetaData> fileMetaDatas =
await this.fileMetaDataAdapter.GetFileMetaDatasAsync(
urn,
fileType,
cancellationToken)
.ConfigureAwait(false);
this.loggerProvider.Info(
$"Returned {fileMetaDatas.Count()} result(s).");
// Then, actually pull the files.
List <File> files = new List <File>();
File file = null;
foreach (FileMetaData fileMetaData in fileMetaDatas)
{
this.loggerProvider.Debug($"Pulling {fileMetaData}...");
file = await this.fileStorageAdapter.GetFileAsync(
fileMetaData,
cancellationToken)
.ConfigureAwait(false);
if (file != null)
{
this.loggerProvider.Info($"Pulled {file}.");
files.Add(file);
}
else
{
this.loggerProvider.Warning(
$"No file could be found for {fileMetaData}!");
}
}
toReturn.Add(urn, files);
}
return(toReturn);
}
private async Task <IActionResult> GetFileAsync(
int urn,
FileTypeOption fileType,
int[] fallbackUrns,
CancellationToken cancellationToken)
{
IActionResult toReturn = null;
Domain.Models.File file =
await this.fileManager.GetFileAsync(
urn,
fileType,
fallbackUrns,
cancellationToken)
.ConfigureAwait(false);
if (file != null)
{
this.loggerProvider.Info(
$"The method " +
$"{nameof(IFileManager)}.{nameof(IFileManager.GetFileAsync)} " +
$"method returned {file} for {nameof(urn)} = \"{urn}\" " +
$"and {nameof(fileType)} = \"{fileType}\" - returning " +
$"with a {nameof(FileContentResult)}.");
byte[] contentBytes = file.ContentBytes.ToArray();
string contentType = file.ContentType;
FileContentResult fileContentResult = new FileContentResult(
contentBytes,
contentType);
fileContentResult.FileDownloadName = file.FileName;
toReturn = fileContentResult;
}
else
{
this.loggerProvider.Warning(
$"The method " +
$"{nameof(IFileManager)}.{nameof(IFileManager.GetFileAsync)} " +
$"method returned null for {nameof(urn)} = \"{urn}\" " +
$"and {nameof(fileType)} = \"{fileType}\" - returning " +
$"{nameof(NotFoundResult)}.");
toReturn = new NotFoundResult();
}
return(toReturn);
}
private async Task <IDictionary <int, IEnumerable <File> > > GetUrnFilesAsync(
int urn,
FileTypeOption fileType,
CancellationToken cancellationToken)
{
IDictionary <int, IEnumerable <File> > toReturn = null;
int[] urns = new int[] { urn };
toReturn = await this.GetUrnFilesAsync(
urns,
fileType,
cancellationToken)
.ConfigureAwait(false);
return(toReturn);
}
public async Task GetFileAsync_NoMetaDataFoundForUrnOrFallbackUrns_ReturnsNull()
{
// Arrange
int urn = 1234;
FileTypeOption fileType = FileTypeOption.Report;
int[] fallbackUrns = new int[] { 4567, 890 };
CancellationToken cancellationToken = CancellationToken.None;
File file = null;
// Act
file = await this.fileManager.GetFileAsync(
urn,
fileType,
fallbackUrns,
cancellationToken);
// Assert
Assert.IsNull(file);
}
private async Task ProcessSitePlanZip(
Establishment establishment,
DocumentFile documentFile,
List <string> usedFileNames,
CancellationToken cancellationToken)
{
byte[] sitePlanBytes = await this.DownloadAndUnzip(
documentFile,
cancellationToken)
.ConfigureAwait(false);
string name = establishment.Name;
string filename = name + DestinationSitePlanFileExtension;
filename = this.GenerateUniqueName(
filename,
" ",
DestinationSitePlanFileExtension,
usedFileNames);
Uri uri = await this.SendFileToDestinationStorage(
establishment,
DestinationSitePlanSubDirectory,
filename,
DestinationSitePlanMimeType,
sitePlanBytes,
cancellationToken)
.ConfigureAwait(false);
FileTypeOption fileType = FileTypeOption.SitePlan;
await this.InsertMetaData(
establishment,
fileType,
uri,
name,
filename,
cancellationToken)
.ConfigureAwait(false);
}
public async Task GetFileAsync_OneMetaDataRecordFoundForUrnFileMissing_ReturnsNull()
{
// Arrange
int urn = 1234;
FileTypeOption fileType = FileTypeOption.Report;
int[] fallbackUrns = null;
CancellationToken cancellationToken = CancellationToken.None;
Uri location = new Uri(
"https://some-storage-container.azure.example/doesntexist.pdf",
UriKind.Absolute);
FileMetaData fileMetaData = new FileMetaData()
{
Location = location,
};
IEnumerable <FileMetaData> fileMetaDatas = new FileMetaData[]
{
fileMetaData,
};
this.mockFileMetaDataAdapter
.Setup(x => x.GetFileMetaDatasAsync(It.IsAny <int>(), It.IsAny <FileTypeOption>(), It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(fileMetaDatas));
File file = null;
// Act
file = await this.fileManager.GetFileAsync(
urn,
fileType,
fallbackUrns,
cancellationToken);
// Assert
Assert.IsNull(file);
}
private async Task InsertMetaData(
Establishment establishment,
FileTypeOption fileType,
Uri uri,
string name,
string filename,
CancellationToken cancellationToken)
{
// Then insert the metadata.
string[] segments = uri.Segments;
string fileName = segments.Last();
string fileNameDecoded = HttpUtility.UrlDecode(filename);
string fileUrl = uri.AbsoluteUri.Replace(fileName, string.Empty);
await this.documentMetadataAdapter.CreateDocumentMetadataAsync(
establishment.Urn.Value,
name,
fileType,
fileNameDecoded,
fileUrl,
cancellationToken)
.ConfigureAwait(false);
}
public async Task GetFileAsync_NoResultsForPrimaryUrnButResultsForFallbackUrns_ReturnsZip()
{
// Arrange
int urn = 1234;
int[] fallbackUrns = new int[] { 3456, 7890 };
FileTypeOption fileType = FileTypeOption.Report;
CancellationToken cancellationToken = CancellationToken.None;
Func <int, FileTypeOption, CancellationToken, Task <IEnumerable <FileMetaData> > > getFileMetaDataCallback =
(urn, fileType, cancellationToken) =>
{
IEnumerable <FileMetaData> results = null;
// Only return results for a fallback.
if (fallbackUrns.Contains(urn))
{
results = new string[]
{
$"{urn} docs/report-{urn}.pdf",
$"{urn} docs/{Guid.NewGuid()}.docx",
}
.Select(x => $"https://some-storage-container.azure.example/{x}")
.Select(x => new Uri(x, UriKind.Absolute))
.Select(x => new FileMetaData()
{
Location = x
})
.ToArray();
}
else
{
results = Array.Empty <FileMetaData>();
}
return(Task.FromResult(results));
};
this.mockFileMetaDataAdapter
.Setup(x => x.GetFileMetaDatasAsync(It.IsAny <int>(), It.IsAny <FileTypeOption>(), It.IsAny <CancellationToken>()))
.Returns(getFileMetaDataCallback);
Random random = new Random();
byte[] contentBytes = null;
File file = null;
Func <FileMetaData, CancellationToken, Task <File> > getFileAsyncCallback =
(fmd, ct) =>
{
contentBytes = new byte[random.Next(1024, 2048)];
random.NextBytes(contentBytes);
file = new File()
{
FileName = fmd.Location.Segments.Last(),
ContentType = "application/pdf",
ContentBytes = contentBytes,
};
return(Task.FromResult(file));
};
this.mockFileStorageAdapter
.Setup(x => x.GetFileAsync(It.IsAny <FileMetaData>(), It.IsAny <CancellationToken>()))
.Returns(getFileAsyncCallback);
File actualFile = null;
string expectedContentType = "application/zip";
string actualContentType = null;
string expectedFilename = $"{urn} files.zip";
string actualFilename = null;
IEnumerable <ZipArchiveEntry> zipArchiveEntries = null;
int expectedNumberOfArchiveEntries = 4;
int actualNumberOfArchiveEntries;
// Act
actualFile = await this.fileManager.GetFileAsync(
urn,
fileType,
fallbackUrns,
cancellationToken);
// Assert
actualContentType = actualFile.ContentType;
Assert.AreEqual(expectedContentType, actualContentType);
actualFilename = actualFile.FileName;
Assert.AreEqual(expectedFilename, actualFilename);
// -> Actually open the zip to check that the bytes are all good.
zipArchiveEntries = this.ExtractZipEntries(actualFile);
actualNumberOfArchiveEntries = zipArchiveEntries.Count();
Assert.AreEqual(
expectedNumberOfArchiveEntries,
actualNumberOfArchiveEntries);
}
public async Task GetFileAsync_MultipleMetaDataRecordsFoundForUrnAllFilesExist_ReturnsZip()
{
// Arrange
int urn = 1234;
int[] fallbackUrns = null;
FileTypeOption fileType = FileTypeOption.Report;
CancellationToken cancellationToken = CancellationToken.None;
string[] exampleDocs = new string[]
{
"doc1.pdf",
"doc2.pdf",
"doc3.pdf"
};
IEnumerable <FileMetaData> fileMetaDatas = exampleDocs
.Select(x => $"https://some-storage-container.azure.example/{x}")
.Select(x => new Uri(x, UriKind.Absolute))
.Select(x => new FileMetaData()
{
Location = x
})
.ToArray();
this.mockFileMetaDataAdapter
.Setup(x => x.GetFileMetaDatasAsync(It.IsAny <int>(), It.IsAny <FileTypeOption>(), It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(fileMetaDatas));
Random random = new Random();
byte[] contentBytes = null;
File file = null;
Func <FileMetaData, CancellationToken, Task <File> > getFileAsyncCallback =
(fmd, ct) =>
{
contentBytes = new byte[random.Next(1024, 2048)];
random.NextBytes(contentBytes);
file = new File()
{
FileName = fmd.Location.Segments.Last(),
ContentType = "application/pdf",
ContentBytes = contentBytes,
};
return(Task.FromResult(file));
};
this.mockFileStorageAdapter
.Setup(x => x.GetFileAsync(It.IsAny <FileMetaData>(), It.IsAny <CancellationToken>()))
.Returns(getFileAsyncCallback);
File actualFile = null;
string expectedContentType = "application/zip";
string actualContentType = null;
string expectedFilename = $"{urn} files.zip";
string actualFilename = null;
IEnumerable <ZipArchiveEntry> zipArchiveEntries = null;
int expectedNumberOfArchiveEntries = 3;
int actualNumberOfArchiveEntries;
// Act
actualFile = await this.fileManager.GetFileAsync(
urn,
fileType,
fallbackUrns,
cancellationToken);
// Assert
actualContentType = actualFile.ContentType;
Assert.AreEqual(expectedContentType, actualContentType);
actualFilename = actualFile.FileName;
Assert.AreEqual(expectedFilename, actualFilename);
// -> Actually open the zip to check that the bytes are all good.
zipArchiveEntries = this.ExtractZipEntries(actualFile);
actualNumberOfArchiveEntries = zipArchiveEntries.Count();
Assert.AreEqual(
expectedNumberOfArchiveEntries,
actualNumberOfArchiveEntries);
}
/// <inheritdoc />
public async Task <IEnumerable <FileMetaData> > GetFileMetaDatasAsync(
int establishmentId,
FileTypeOption fileType,
CancellationToken cancellationToken)
{
IEnumerable <FileMetaData> toReturn = null;
byte typeId = (byte)fileType;
var parameters = new
{
EstablishmentID = establishmentId,
FileTypeID = typeId,
};
IEnumerable <GetFileListResult> getFileListResults = null;
using (SqlConnection sqlConnection = await this.GetOpenConnectionAsync(cancellationToken).ConfigureAwait(false))
{
this.loggerProvider.Debug(
$"Executing \"{SprocNameGetFileList}\" with " +
$"{nameof(establishmentId)} = {establishmentId}, " +
$"{nameof(typeId)} = {typeId}...");
getFileListResults =
await sqlConnection.QueryAsync <GetFileListResult>(
"sp_SELECT_FileData",
parameters,
commandType : CommandType.StoredProcedure)
.ConfigureAwait(false);
this.loggerProvider.Info(
$"{getFileListResults.Count()} result(s) returned.");
}
if (getFileListResults.Any())
{
// Quiz the SiteVisitDate column. Get the most recent date.
DateTime mostRecentDate = getFileListResults
.Select(x => x.SiteVisitDate)
.Max();
this.loggerProvider.Debug(
$"Mapping all results with " +
$"{nameof(GetFileListResult.SiteVisitDate)} = " +
$"\"{mostRecentDate}\"...");
toReturn = getFileListResults
.Where(x => x.SiteVisitDate == mostRecentDate)
.Select(x =>
{
FileMetaData fileMetaData = null;
string fileUrl = x.FileURL;
string fileName = x.FileName;
Uri baseUri = new Uri(fileUrl, UriKind.Absolute);
Uri relativeUri = new Uri(fileName, UriKind.Relative);
Uri location = new Uri(baseUri, relativeUri);
fileMetaData = new FileMetaData()
{
Location = location,
};
return(fileMetaData);
});
}
else
{
toReturn = Array.Empty <FileMetaData>();
this.loggerProvider.Info(
$"No records found for {nameof(establishmentId)} = " +
$"{establishmentId} and {nameof(fileType)} = {fileType}.");
}
return(toReturn);
}
public async Task <File> GetFileAsync(
int urn,
FileTypeOption fileType,
int[] fallbackUrns,
CancellationToken cancellationToken)
{
File toReturn = null;
this.loggerProvider.Debug(
$"Firstly, searching for files under {nameof(urn)} {urn}...");
IDictionary <int, IEnumerable <File> > fileSearch =
await this.GetUrnFilesAsync(urn, fileType, cancellationToken)
.ConfigureAwait(false);
IEnumerable <File> files = fileSearch[urn];
if (files.Any())
{
if (files.Count() == 1)
{
// Then just return this one file.
toReturn = files.Single();
this.loggerProvider.Info(
$"Only one {nameof(File)} available ({toReturn}). " +
$"Returning this {nameof(File)}, as-is.");
}
else
{
this.loggerProvider.Debug(
$"More than one file available ({files.Count()} in " +
$"total). Zipping up...");
// Else, there's more than one, and we'll need to zip them
// up, then return them.
toReturn = this.ZipMultipleFiles(urn, files);
this.loggerProvider.Info(
$"Returning zip {nameof(File)}: {toReturn}.");
}
}
else
{
this.loggerProvider.Warning(
$"Could not find any files for {nameof(urn)} = {urn} " +
$"and {nameof(fileType)} = {fileType}.");
if (fallbackUrns != null)
{
this.loggerProvider.Info(
"Fallback URNs were provided, however. Pulling back " +
"files for fallback URNs...");
fileSearch = await this.GetUrnFilesAsync(
fallbackUrns,
fileType,
cancellationToken)
.ConfigureAwait(false);
// We only want to return a file (i.e. zip) if there are
// files in our search results to return.
bool fallbackFilesAvailable = fileSearch
.SelectMany(x => x.Value)
.Any();
if (fallbackFilesAvailable)
{
toReturn = this.ZipMultipleFiles(
urn,
fileSearch);
}
else
{
this.loggerProvider.Info(
$"No files available for the primary " +
$"{nameof(urn)} = {urn}, or for any " +
$"{nameof(fallbackUrns)}. Null will be returned.");
}
}
}
return(toReturn);
}
public async Task <IActionResult> RunAsync(
[HttpTrigger(AuthorizationLevel.Function, "GET", Route = "cdc-file/{urn}")]
HttpRequest httpRequest,
int urn,
CancellationToken cancellationToken)
{
IActionResult toReturn = null;
this.loggerProvider.Debug(
$"File requested. {nameof(urn)} = \"{urn}\".");
// urn never can be null; if someone tries to call ./cdc-file or
// ./cdc-file/, a 404 is returned by default.
if (httpRequest == null)
{
throw new ArgumentNullException(nameof(httpRequest));
}
// The following header is optional.
// If the requested file is not available, then fetch the files
// contained within this CSL.
IHeaderDictionary headers = httpRequest.Headers;
int[] fallbackUrns = null;
if (headers.ContainsKey(FallbackUrnsHeaderName))
{
string fallbackUrnsCsl = headers[FallbackUrnsHeaderName];
this.loggerProvider.Debug(
$"Header \"{FallbackUrnsHeaderName}\" was specified: " +
$"\"{fallbackUrnsCsl}\". This should be a CSL of " +
$"integers. Parsing...");
string[] fallbackUrnsStr = fallbackUrnsCsl.Split(',');
// Try and parse each one.
try
{
fallbackUrns = fallbackUrnsStr
.Select(x => x.Trim())
.Select(x => int.Parse(x, CultureInfo.InvariantCulture))
.ToArray();
this.loggerProvider.Info(
$"Parsed {fallbackUrns.Length} fallback URNs.");
}
catch (FormatException)
{
this.loggerProvider.Warning(
$"Could not parse header value for " +
$"\"{FallbackUrnsHeaderName}\" into an array of " +
$"{nameof(Int32)} values. Original string: " +
$"\"{fallbackUrnsCsl}\".");
toReturn = new BadRequestResult();
}
}
if (toReturn == null)
{
string typeStr = httpRequest.Query["type"];
// We need a null-check on type.
if (!string.IsNullOrEmpty(typeStr))
{
this.loggerProvider.Debug(
$"{nameof(typeStr)} = \"{typeStr}\"");
if (this.fileTypeMap.ContainsKey(typeStr))
{
FileTypeOption fileType = this.fileTypeMap[typeStr];
toReturn = await this.GetFileAsync(
urn,
fileType,
fallbackUrns,
cancellationToken)
.ConfigureAwait(false);
}
else
{
string[] validTypes = this.fileTypeMap.Keys
.Select(x => $"\"{x}\"")
.ToArray();
string validTypesList = string.Join(", ", validTypes);
this.loggerProvider.Warning(
$"The type was supplied was not valid. This " +
$"needs to be one of the following values: " +
$"{validTypesList}. Returning " +
$"{nameof(BadRequestResult)}.");
// Return "bad request" to indicate we need it.
toReturn = new BadRequestResult();
}
}
else
{
this.loggerProvider.Warning(
$"The type was not supplied. This is required. " +
$"Returning {nameof(BadRequestResult)}.");
// Return "bad request" to indicate we need it.
toReturn = new BadRequestResult();
}
}
return(toReturn);
}
private async Task ProcessSitePlanZip(
Establishment establishment,
DocumentFile documentFile,
List <string> usedFileNames,
CancellationToken cancellationToken)
{
byte[] sitePlanBytes = await this.DownloadAndUnzip(
documentFile,
cancellationToken)
.ConfigureAwait(false);
if (sitePlanBytes.Length < 10000)
{
// probably a text file, skip it
this.loggerWrapper.Info($"{documentFile.Name} with mime type octet-stream is less than 10kb, skipping");
return;
}
string name = establishment.Name;
string filename = name + DestinationSitePlanFileExtension;
// Attempt to get filename from evidence data
string idSegment = this.GetIdFromName(documentFile.Name);
string evidenceName = string.Empty;
if (idSegment != string.Empty)
{
var keyExists = this._cdc1Evidence.TryGetValue(idSegment.ToLower(), out evidenceName);
if (keyExists)
{
this.loggerWrapper.Info($"Evidence name: {evidenceName}");
}
else
{
this.loggerWrapper.Info($"Entry not found for id {idSegment}, skipping file");
return;
}
evidenceName += this.GetFileExtension(documentFile.Name);
evidenceName = this.StripIllegalCharacters(evidenceName);
}
Uri uri = await this.SendFileToDestinationStorage(
establishment,
DestinationSitePlanSubDirectory,
evidenceName,
DestinationSitePlanMimeType,
sitePlanBytes,
cancellationToken)
.ConfigureAwait(false);
FileTypeOption fileType = FileTypeOption.SitePlan;
await this.InsertMetaData(
establishment,
fileType,
uri,
name,
evidenceName,
cancellationToken)
.ConfigureAwait(false);
}
