public async Task CanGetExistingFolderId()
{
var config = serviceProvider.GetRequiredService <IConfigurationStore>();
var dataOceanBaseUrl = config.GetValueString("DATA_OCEAN_URL");
var dataOceanRootFolderId = config.GetValueString("DATA_OCEAN_ROOT_FOLDER_ID");
const string folderName = "unittest";
var expectedFolderResult = new DataOceanDirectory {
Id = Guid.NewGuid(), Name = folderName, ParentId = Guid.Parse(dataOceanRootFolderId)
};
var expectedBrowseResult =
new BrowseDirectoriesResult {
Directories = new List <DataOceanDirectory> {
expectedFolderResult
}
};
var browseUrl = $"{dataOceanBaseUrl}/api/browse/keyset_directories?name={folderName}&owner=true&parent_id={dataOceanRootFolderId}";
var gracefulMock = new Mock <IWebRequest>();
gracefulMock.Setup(g => g.ExecuteRequest <BrowseDirectoriesResult>(browseUrl, null, null, HttpMethod.Get, null, 0, false))
.Returns(Task.FromResult(expectedBrowseResult));
serviceCollection.AddTransient(g => gracefulMock.Object);
var serviceProvider2 = serviceCollection.BuildServiceProvider();
var client = serviceProvider2.GetRequiredService <IDataOceanClient>();
var Id = await client.GetFolderId($"{DataOceanUtil.PathSeparator}{dataOceanRootFolderId}{DataOceanUtil.PathSeparator}{folderName}", null);
Assert.Equal(expectedFolderResult.Id, Id);
}
public async Task CanGenerateGeoTiffTilesFailToCreateExecution()
{
//Set up DataOcean stuff
var expectedTopFolderResult = new DataOceanDirectory {
Id = Guid.NewGuid(), Name = topLevelFolderName
};
var expectedFileResult = new DataOceanFile {
Id = Guid.NewGuid(), Name = geoTiffFileName, ParentId = expectedTopFolderResult.Id
};
var subFolderPath = new DataOceanFileUtil(geoTiffFullName).GeneratedTilesFolder;
var dataOceanMock = new Mock <IDataOceanClient>();
dataOceanMock.Setup(d => d.GetFileId(geoTiffFullName, null)).ReturnsAsync(expectedFileResult.Id);
dataOceanMock.Setup(d => d.MakeFolder(subFolderPath, null)).ReturnsAsync(true);
dataOceanMock.Setup(d => d.GetFolderId($"{DataOceanUtil.PathSeparator}{topLevelFolderName}", null)).ReturnsAsync(expectedTopFolderResult.Id);
//Set up Pegasus stuff
var config = serviceProvider.GetRequiredService <Common.Abstractions.Configuration.IConfigurationStore>();
var pegasusBaseUrl = config.GetValueString("PEGASUS_URL");
var baseRoute = "/api/executions";
var createExecutionUrl = $"{pegasusBaseUrl}{baseRoute}";
var gracefulMock = new Mock <IWebRequest>();
gracefulMock
.Setup(g => g.ExecuteRequest <PegasusExecutionResult>(createExecutionUrl, It.IsAny <MemoryStream>(), null, HttpMethod.Post, null, 0,
false)).ReturnsAsync((PegasusExecutionResult)null);
await ProcessWithFailure(gracefulMock, dataOceanMock,
$"Failed to create execution for {geoTiffFullName}", false);
}
private Task <TileMetadata> CanGenerateDxfTiles(string status)
{
//Set up DataOcean stuff
var expectedTopFolderResult = new DataOceanDirectory {
Id = Guid.NewGuid(), Name = topLevelFolderName
};
var expectedDcFileResult = new DataOceanFile {
Id = Guid.NewGuid(), Name = dcFileName, ParentId = expectedTopFolderResult.Id
};
var expectedDxfFileResult = new DataOceanFile {
Id = Guid.NewGuid(), Name = dxfFileName, ParentId = expectedTopFolderResult.Id
};
var subFolderPath = new DataOceanFileUtil(dxfFullName).GeneratedTilesFolder;
var parts = subFolderPath.Split(DataOceanUtil.PathSeparator);
var subFolderName = parts[parts.Length - 1];
var dataOceanMock = new Mock <IDataOceanClient>();
dataOceanMock.Setup(d => d.GetFileId(dcFullName, null)).ReturnsAsync(expectedDcFileResult.Id);
dataOceanMock.Setup(d => d.GetFileId(dxfFullName, null)).ReturnsAsync(expectedDxfFileResult.Id);
dataOceanMock.Setup(d => d.MakeFolder(subFolderPath, null)).ReturnsAsync(true);
dataOceanMock.Setup(d => d.GetFolderId($"{DataOceanUtil.PathSeparator}{topLevelFolderName}", null)).ReturnsAsync(expectedTopFolderResult.Id);
//Set up Pegasus stuff
var units = DxfUnitsType.UsSurveyFeet.ToString();
var expectedExecution =
NewDxfPegasusExecution(expectedDcFileResult, expectedDxfFileResult, subFolderName, units, status);
var expectedExecutionResult = new PegasusExecutionResult {
Execution = expectedExecution
};
var expectedExecutionAttemptResult = new PegasusExecutionAttemptResult
{
ExecutionAttempt = new PegasusExecutionAttempt {
Id = Guid.NewGuid(), Status = ExecutionStatus.EXECUTING
}
};
var config = serviceProvider.GetRequiredService <Common.Abstractions.Configuration.IConfigurationStore>();
var pegasusBaseUrl = config.GetValueString("PEGASUS_URL");
var baseRoute = "/api/executions";
var createExecutionUrl = $"{pegasusBaseUrl}{baseRoute}";
var startExecutionUrl = $"{pegasusBaseUrl}{baseRoute}/{expectedExecution.Id}/start";
var executionStatusUrl = $"{pegasusBaseUrl}{baseRoute}/{expectedExecution.Id}";
var gracefulMock = new Mock <IWebRequest>();
gracefulMock
.Setup(g => g.ExecuteRequest <PegasusExecutionResult>(createExecutionUrl, It.IsAny <MemoryStream>(), null, HttpMethod.Post, null, 0,
false)).ReturnsAsync(expectedExecutionResult);
gracefulMock
.Setup(g => g.ExecuteRequest <PegasusExecutionAttemptResult>(startExecutionUrl, null, null, HttpMethod.Post, null, 0,
false)).ReturnsAsync(expectedExecutionAttemptResult);
gracefulMock
.Setup(g => g.ExecuteRequest <PegasusExecutionResult>(executionStatusUrl, null, null, HttpMethod.Get, null, 0,
false)).ReturnsAsync(expectedExecutionResult);
return(ProcessWithSuccess(gracefulMock, dataOceanMock, subFolderPath, true));
}
public async Task BadTileResponseNullHasNoCache()
{
var fileName = $"{DataOceanUtil.PathSeparator}tiles{DataOceanUtil.PathSeparator}tiles.json";
var downloadUrl = TestConstants.DownloadUrl;
var substitutedDownloadUrl = downloadUrl.Replace("{path}", fileName.Substring(1));
var config = ServiceProvider.GetRequiredService <IConfigurationStore>();
var dataOceanBaseUrl = config.GetValueString("DATA_OCEAN_URL");
var dataOceanRootFolderId = config.GetValueString("DATA_OCEAN_ROOT_FOLDER_ID");
const string folderName = "unittest";
var expectedFolderResult = new DataOceanDirectory {
Id = Guid.NewGuid(), Name = folderName, ParentId = Guid.Parse(dataOceanRootFolderId)
};
var expectedFolderBrowseResult =
new BrowseDirectoriesResult {
Directories = new List <DataOceanDirectory> {
expectedFolderResult
}
};
const string multiFileName = "dummy.dxf_Tiles$";
var expectedFileBrowseResult = SetupExpectedMutipleFileVersionsResult(Guid.NewGuid(), multiFileName, expectedFolderResult.Id, downloadUrl);
var browseFolderUrl = $"{dataOceanBaseUrl}/api/browse/keyset_directories?name={folderName}&owner=true&parent_id={dataOceanRootFolderId}";
var browseFileUrl = $"{dataOceanBaseUrl}/api/browse/keyset_files?name={multiFileName}&owner=true&parent_id={expectedFolderResult.Id}";
var gracefulMock = new Mock <IWebRequest>();
gracefulMock
.Setup(g => g.ExecuteRequest <BrowseDirectoriesResult>(browseFolderUrl, null, null, HttpMethod.Get, null, 0, false))
.Returns(Task.FromResult(expectedFolderBrowseResult));
gracefulMock
.Setup(g => g.ExecuteRequest <BrowseFilesResult>(browseFileUrl, null, null, HttpMethod.Get, null, 0, false))
.Returns(Task.FromResult(expectedFileBrowseResult));
gracefulMock
.Setup(g => g.ExecuteRequestAsStreamContent(substitutedDownloadUrl, HttpMethod.Get, null, null, null, 0, false))
.ReturnsAsync((HttpContent)null);
ServiceCollection.AddTransient(g => gracefulMock.Object);
ServiceProvider = ServiceCollection.BuildServiceProvider();
var client = ServiceProvider.GetRequiredService <IDataOceanClient>();
client.GetTileCache().GetCache().CacheKeys.Count.Should().Be(0);
var fullFileName = $"{DataOceanUtil.PathSeparator}{dataOceanRootFolderId}{DataOceanUtil.PathSeparator}{folderName}{DataOceanUtil.PathSeparator}{multiFileName}{fileName}";
var resultStream = await client.GetFile(fullFileName, null);
resultStream.Should().BeNull();
}
public async Task CanGenerateGeoTiffTilesMissingGeoTiffFile()
{
var expectedTopFolderResult = new DataOceanDirectory {
Id = Guid.NewGuid(), Name = topLevelFolderName
};
_ = new DataOceanFile {
Id = Guid.NewGuid(), Name = geoTiffFileName, ParentId = expectedTopFolderResult.Id
};
var dataOceanMock = new Mock <IDataOceanClient>();
dataOceanMock.Setup(d => d.GetFileId(geoTiffFullName, null)).ReturnsAsync((Guid?)null);
var gracefulMock = new Mock <IWebRequest>();
await ProcessWithFailure(gracefulMock, dataOceanMock,
$"Failed to find GeoTIFF file {geoTiffFullName}. Has it been uploaded successfully?", false);
}
public async Task CanGenerateDxfTilesMissingDcFile()
{
var expectedTopFolderResult = new DataOceanDirectory {
Id = Guid.NewGuid(), Name = topLevelFolderName
};
var expectedDxfFileResult = new DataOceanFile {
Id = Guid.NewGuid(), Name = dxfFileName, ParentId = expectedTopFolderResult.Id
};
var dataOceanMock = new Mock <IDataOceanClient>();
dataOceanMock.Setup(d => d.GetFileId(dxfFullName, null)).ReturnsAsync(expectedDxfFileResult.Id);
dataOceanMock.Setup(d => d.GetFileId(dcFullName, null)).ReturnsAsync((Guid?)null);
var gracefulMock = new Mock <IWebRequest>();
await ProcessWithFailure(gracefulMock, dataOceanMock,
$"Failed to find coordinate system file {dcFullName}. Has it been uploaded successfully?", true);
}
public async Task CanCreateTopLevelFolder()
{
var config = serviceProvider.GetRequiredService <IConfigurationStore>();
var dataOceanBaseUrl = config.GetValueString("DATA_OCEAN_URL");
var dataOceanRootFolderId = config.GetValueString("DATA_OCEAN_ROOT_FOLDER_ID");
const string folderName = "unittest";
var expectedFolderResult = new DataOceanDirectory {
Id = Guid.NewGuid(), Name = folderName, ParentId = Guid.Parse(dataOceanRootFolderId)
};
var expectedBrowseResult = new BrowseDirectoriesResult {
Directories = new List <DataOceanDirectory>()
};
var browseUrl = $"{dataOceanBaseUrl}/api/browse/keyset_directories?name={folderName}&owner=true&parent_id={dataOceanRootFolderId}";
var createUrl = $"{dataOceanBaseUrl}/api/directories";
var gracefulMock = new Mock <IWebRequest>();
gracefulMock.Setup(g => g.ExecuteRequest <BrowseDirectoriesResult>(browseUrl, null, null, HttpMethod.Get, null, 0, false))
.Returns(Task.FromResult(expectedBrowseResult));
gracefulMock
.Setup(g => g.ExecuteRequest <DataOceanDirectoryResult>(createUrl, It.IsAny <MemoryStream>(), null, HttpMethod.Post, null, 0,
false)).ReturnsAsync(new DataOceanDirectoryResult {
Directory = expectedFolderResult
});
serviceCollection.AddTransient(g => gracefulMock.Object);
var serviceProvider2 = serviceCollection.BuildServiceProvider();
var client = serviceProvider2.GetRequiredService <IDataOceanClient>();
var success = await client.MakeFolder($"{DataOceanUtil.PathSeparator}{dataOceanRootFolderId}{DataOceanUtil.PathSeparator}{folderName}", null);
Assert.True(success);
//Check it also succeeds when the folder already exists
expectedBrowseResult.Directories = new List <DataOceanDirectory> {
expectedFolderResult
};
success = await client.MakeFolder($"{DataOceanUtil.PathSeparator}{dataOceanRootFolderId}{DataOceanUtil.PathSeparator}{folderName}", null);
Assert.True(success);
}
public async Task CanGetNonExistingSingleFile()
{
const string folderName = "unittest";
var expectedFolderResult = new DataOceanDirectory {
Id = Guid.NewGuid(), Name = folderName
};
var expectedFolderBrowseResult =
new BrowseDirectoriesResult {
Directories = new List <DataOceanDirectory> {
expectedFolderResult
}
};
const string fileName = "dummy.dxf";
var expectedFileBrowseResult = new BrowseFilesResult {
Files = new List <DataOceanFile>()
};
var config = serviceProvider.GetRequiredService <IConfigurationStore>();
var dataOceanBaseUrl = config.GetValueString("DATA_OCEAN_URL");
var browseFolderUrl = $"{dataOceanBaseUrl}/api/browse/keyset_directories?name={folderName}&owner=true";
var browseFileUrl = $"{dataOceanBaseUrl}/api/browse/keyset_files?name={fileName}&owner=true&parent_id={expectedFolderResult.Id}";
var gracefulMock = new Mock <IWebRequest>();
gracefulMock
.Setup(g => g.ExecuteRequest <BrowseDirectoriesResult>(browseFolderUrl, null, null, HttpMethod.Get, null, 0, false))
.Returns(Task.FromResult(expectedFolderBrowseResult));
gracefulMock
.Setup(g => g.ExecuteRequest <BrowseFilesResult>(browseFileUrl, null, null, HttpMethod.Get, null, 0, false))
.Returns(Task.FromResult(expectedFileBrowseResult));
serviceCollection.AddTransient(g => gracefulMock.Object);
var serviceProvider2 = serviceCollection.BuildServiceProvider();
var client = serviceProvider2.GetRequiredService <IDataOceanClient>();
var resultStream = await client.GetFile($"{DataOceanUtil.PathSeparator}{folderName}{DataOceanUtil.PathSeparator}{fileName}", null);
Assert.Null(resultStream);
}
public async Task CanGetNonExistingMultiFile()
{
var fileName = $"{DataOceanUtil.PathSeparator}tiles{DataOceanUtil.PathSeparator}15{DataOceanUtil.PathSeparator}18756{DataOceanUtil.PathSeparator}2834.png";
var downloadUrl = TestConstants.DownloadUrl;
var substitutedDownloadUrl = downloadUrl.Replace("{path}", fileName.Substring(1));
const string folderName = "unittest";
var expectedFolderResult = new DataOceanDirectory {
Id = Guid.NewGuid(), Name = folderName
};
var expectedFolderBrowseResult =
new BrowseDirectoriesResult {
Directories = new List <DataOceanDirectory> {
expectedFolderResult
}
};
const string multiFileName = "dummy.dxf_Tiles$";
var expectedFileResult = new DataOceanFile
{
Id = Guid.NewGuid(),
Name = multiFileName,
ParentId = expectedFolderResult.Id,
Multifile = true,
RegionPreferences = new List <string> {
"us1"
},
Status = "AVAILABLE",
DataOceanDownload = new DataOceanTransfer {
Url = downloadUrl
}
};
var expectedFileBrowseResult = new BrowseFilesResult {
Files = new List <DataOceanFile> {
expectedFileResult
}
};
var config = serviceProvider.GetRequiredService <IConfigurationStore>();
var dataOceanBaseUrl = config.GetValueString("DATA_OCEAN_URL");
var browseFolderUrl = $"{dataOceanBaseUrl}/api/browse/keyset_directories?name={folderName}&owner=true";
var browseFileUrl = $"{dataOceanBaseUrl}/api/browse/keyset_files?name={multiFileName}&owner=true&parent_id={expectedFolderResult.Id}";
var gracefulMock = new Mock <IWebRequest>();
gracefulMock
.Setup(g => g.ExecuteRequest <BrowseDirectoriesResult>(browseFolderUrl, null, null, HttpMethod.Get, null, 0, false))
.Returns(Task.FromResult(expectedFolderBrowseResult));
gracefulMock
.Setup(g => g.ExecuteRequest <BrowseFilesResult>(browseFileUrl, null, null, HttpMethod.Get, null, 0, false))
.Returns(Task.FromResult(expectedFileBrowseResult));
gracefulMock
.Setup(g => g.ExecuteRequestAsStreamContent(substitutedDownloadUrl, HttpMethod.Get, null, null, null, 0, false))
.ReturnsAsync((HttpContent)null);
serviceCollection.AddTransient(g => gracefulMock.Object);
var serviceProvider2 = serviceCollection.BuildServiceProvider();
var client = serviceProvider2.GetRequiredService <IDataOceanClient>();
var fullFileName = $"{DataOceanUtil.PathSeparator}{folderName}{DataOceanUtil.PathSeparator}{multiFileName}{fileName}";
var resultStream = await client.GetFile(fullFileName, null);
Assert.Null(resultStream);
}
public async Task CanGetExistingMultiFile()
{
var fileName = $"{DataOceanUtil.PathSeparator}tiles{DataOceanUtil.PathSeparator}tiles.json";
var downloadUrl = TestConstants.DownloadUrl;
var substitutedDownloadUrl = downloadUrl.Replace("{path}", fileName.Substring(1));
var expectedResult = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3 };
var stream = new MemoryStream(expectedResult);
var expectedDownloadResult = new StreamContent(stream);
var config = serviceProvider.GetRequiredService <IConfigurationStore>();
var dataOceanBaseUrl = config.GetValueString("DATA_OCEAN_URL");
var dataOceanRootFolderId = config.GetValueString("DATA_OCEAN_ROOT_FOLDER_ID");
const string folderName = "unittest";
var expectedFolderResult = new DataOceanDirectory {
Id = Guid.NewGuid(), Name = folderName, ParentId = Guid.Parse(dataOceanRootFolderId)
};
var expectedFolderBrowseResult =
new BrowseDirectoriesResult {
Directories = new List <DataOceanDirectory> {
expectedFolderResult
}
};
const string multiFileName = "dummy.dxf_Tiles$";
var fileUid = Guid.NewGuid();
var updatedAt = DateTime.UtcNow.AddHours(-2);
var expectedFileResult = new DataOceanFile
{
Id = fileUid,
Name = multiFileName,
ParentId = expectedFolderResult.Id,
Multifile = true,
RegionPreferences = new List <string> {
"us1"
},
Status = "AVAILABLE",
DataOceanDownload = new DataOceanTransfer {
Url = downloadUrl
},
UpdatedAt = updatedAt
};
var otherFileResult = new DataOceanFile
{
Id = fileUid,
Name = multiFileName,
ParentId = expectedFolderResult.Id,
Multifile = true,
RegionPreferences = new List <string> {
"us1"
},
Status = "AVAILABLE",
DataOceanDownload = new DataOceanTransfer {
Url = downloadUrl
},
UpdatedAt = updatedAt.AddHours(-5)
};
var expectedFileBrowseResult = new BrowseFilesResult {
Files = new List <DataOceanFile> {
expectedFileResult, otherFileResult
}
};
var browseFolderUrl = $"{dataOceanBaseUrl}/api/browse/keyset_directories?name={folderName}&owner=true&parent_id={dataOceanRootFolderId}";
var browseFileUrl = $"{dataOceanBaseUrl}/api/browse/keyset_files?name={multiFileName}&owner=true&parent_id={expectedFolderResult.Id}";
var gracefulMock = new Mock <IWebRequest>();
gracefulMock
.Setup(g => g.ExecuteRequest <BrowseDirectoriesResult>(browseFolderUrl, null, null, HttpMethod.Get, null, 0, false))
.Returns(Task.FromResult(expectedFolderBrowseResult));
gracefulMock
.Setup(g => g.ExecuteRequest <BrowseFilesResult>(browseFileUrl, null, null, HttpMethod.Get, null, 0, false))
.Returns(Task.FromResult(expectedFileBrowseResult));
gracefulMock
.Setup(g => g.ExecuteRequestAsStreamContent(substitutedDownloadUrl, HttpMethod.Get, null, null, null, 0, false))
.ReturnsAsync(expectedDownloadResult);
serviceCollection.AddTransient(g => gracefulMock.Object);
var serviceProvider2 = serviceCollection.BuildServiceProvider();
var client = serviceProvider2.GetRequiredService <IDataOceanClient>();
var fullFileName = $"{DataOceanUtil.PathSeparator}{dataOceanRootFolderId}{DataOceanUtil.PathSeparator}{folderName}{DataOceanUtil.PathSeparator}{multiFileName}{fileName}";
var resultStream = await client.GetFile(fullFileName, null);
using (var ms = new MemoryStream())
{
resultStream.CopyTo(ms);
var result = ms.ToArray();
Assert.Equal(expectedResult, result);
}
}
/// <summary>
/// Gets the lowest level folder metadata in the path. Creates it unless it this is purely a query and therefore must exist.
/// //NOTE: DataOcean requires / regardless of OS. However we construct the path and split using DataOceanUtil.PathSeparator.
/// This is merely a convenience as DataOcean doesn't use paths but a hierarchy of folders above the file, linked using parent_id.
/// We traverse this hierarchy. The only place it matters is the multi-file structure for tiles. This is contained in the multi-file
/// download url so we don't have to worry about it.
/// </summary>
private async Task <DataOceanDirectory> GetFolderMetadata(string path, bool mustExist, IHeaderDictionary customHeaders)
{
_log.LogDebug($"{nameof(GetFolderMetadata)}: path: {path}, mustExist: {mustExist}");
// part1 is dataOcean root folder, 1 more level is required
var parts = path.Split(DataOceanUtil.PathSeparator);
parts = parts.Where(p => !string.IsNullOrEmpty(p)).ToArray();
if (parts.Length < 2)
{
_log.LogError($"{nameof(GetFolderMetadata)} Not enough parts in folder path. path {path}");
return(null);
}
DataOceanDirectory folder = null;
var parentId = parts[0];
var creatingPath = false;
var currentDataOceanFolderPath = _dataOceanFolderCache.GetRootFolder(parentId);
if (currentDataOceanFolderPath == null)
{
_log.LogError($"{nameof(GetFolderMetadata)} Unable to retrieve root cache. dataOceanRootFolderId {parts[0]}");
return(null);
}
for (var i = 1; i < parts.Length; i++)
{
if (currentDataOceanFolderPath.Nodes.TryGetValue(parts[i], out var retrievedCurrentDataOceanFolderPath))
{
currentDataOceanFolderPath = retrievedCurrentDataOceanFolderPath;
folder = new DataOceanDirectory()
{
Id = Guid.Parse(currentDataOceanFolderPath.DataOceanFolderId), Name = parts[i], ParentId = Guid.Parse(parentId)
};
_log.LogDebug($"{nameof(GetFolderMetadata)}: found cached folder. parts[i]: {parts[i]}, Id: {currentDataOceanFolderPath.DataOceanFolderId}, ParentId: {parentId}");
parentId = currentDataOceanFolderPath.DataOceanFolderId;
}
else
{
int?directoriesCount = 0;
if (!creatingPath)
{
var result = await BrowseFolder(parts[i], Guid.Parse(parentId), customHeaders);
directoriesCount = result?.Directories?.Count;
if (directoriesCount == 1)
{
folder = result.Directories[0];
parentId = folder.Id.ToString();
currentDataOceanFolderPath = currentDataOceanFolderPath.CreateNode(parentId, parts[i]);
_log.LogDebug($"{nameof(GetFolderMetadata)}: create cache for existing folder. parts[i]: {parts[i]}, Id: {currentDataOceanFolderPath.DataOceanFolderId}");
}
}
if (directoriesCount == 0)
{
if (mustExist)
{
return(null);
}
folder = (await CreateDirectory(parts[i], Guid.Parse(parentId), customHeaders)).Directory;
parentId = folder.Id.ToString();
currentDataOceanFolderPath = currentDataOceanFolderPath.CreateNode(parentId, parts[i]);
_log.LogDebug($"{nameof(GetFolderMetadata)}: create cache for created folder. parts[i]: {parts[i]}, Id: {currentDataOceanFolderPath.DataOceanFolderId}, ParentId: {parentId}");
creatingPath = true;
}
else if (directoriesCount > 1)
{
_log.LogWarning($"{nameof(GetFolderMetadata)} Duplicate folders {parts[i]} in path {path}");
return(null);
}
}
}
//Folders in path already exist or have been created successfully
return(folder);
}
public async Task GoodTileNoCache()
{
var fileName = $"{DataOceanUtil.PathSeparator}tiles{DataOceanUtil.PathSeparator}tiles.json";
var downloadUrl = TestConstants.DownloadUrl;
var substitutedDownloadUrl = downloadUrl.Replace("{path}", fileName.Substring(1));
var expectedResult = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3 };
var stream = new MemoryStream(expectedResult);
var expectedDownloadResult = new StreamContent(stream);
var config = ServiceProvider.GetRequiredService <IConfigurationStore>();
var dataOceanBaseUrl = config.GetValueString("DATA_OCEAN_URL");
var dataOceanRootFolderId = config.GetValueString("DATA_OCEAN_ROOT_FOLDER_ID");
const string folderName = "unittest";
var expectedFolderResult = new DataOceanDirectory {
Id = Guid.NewGuid(), Name = folderName, ParentId = Guid.Parse(dataOceanRootFolderId)
};
var expectedFolderBrowseResult =
new BrowseDirectoriesResult {
Directories = new List <DataOceanDirectory> {
expectedFolderResult
}
};
const string multiFileName = "dummy.dxf_Tiles$";
var expectedFileBrowseResult = SetupExpectedMutipleFileVersionsResult(Guid.NewGuid(), multiFileName, expectedFolderResult.Id, downloadUrl);
var browseFolderUrl = $"{dataOceanBaseUrl}/api/browse/keyset_directories?name={folderName}&owner=true&parent_id={dataOceanRootFolderId}";
var browseFileUrl = $"{dataOceanBaseUrl}/api/browse/keyset_files?name={multiFileName}&owner=true&parent_id={expectedFolderResult.Id}";
var gracefulMock = new Mock <IWebRequest>();
gracefulMock
.Setup(g => g.ExecuteRequest <BrowseDirectoriesResult>(browseFolderUrl, null, null, HttpMethod.Get, null, 0, false))
.Returns(Task.FromResult(expectedFolderBrowseResult));
gracefulMock
.Setup(g => g.ExecuteRequest <BrowseFilesResult>(browseFileUrl, null, null, HttpMethod.Get, null, 0, false))
.Returns(Task.FromResult(expectedFileBrowseResult));
gracefulMock
.Setup(g => g.ExecuteRequestAsStreamContent(substitutedDownloadUrl, HttpMethod.Get, null, null, null, 0, false))
.ReturnsAsync(expectedDownloadResult);
ServiceCollection.AddTransient(g => gracefulMock.Object);
ServiceProvider = ServiceCollection.BuildServiceProvider();
var client = ServiceProvider.GetRequiredService <IDataOceanClient>();
client.GetTileCache().GetCache().CacheKeys.Count.Should().Be(0);
var fullFileName = $"{DataOceanUtil.PathSeparator}{dataOceanRootFolderId}{DataOceanUtil.PathSeparator}{folderName}{DataOceanUtil.PathSeparator}{multiFileName}{fileName}";
var resultStream = await client.GetFile(fullFileName, null);
resultStream.Should().NotBeNull();
client.GetTileCache().GetCache().CacheKeys.Count.Should().Be(0);
// Generate an event, that will trigger a call to project repo for the file
var rasterTileNotificationParameters = new RasterTileNotificationParameters {
FileUid = Guid.Parse("ED279023-6A51-45B7-B4D0-2A5BF1ECA60C")
};
var notificationHub = ServiceProvider.GetService <INotificationHubClient>() as NotificationHubClient;
Assert.NotNull(notificationHub);
var tasks = notificationHub.ProcessNotificationAsTasks(new ProjectFileRasterTilesGeneratedNotification(rasterTileNotificationParameters));
Task.WaitAll(tasks.ToArray());
client.GetTileCache().GetCache().CacheKeys.Count.Should().Be(0);
}
public async Task FileUpdateNotificationClearTileCache()
{
var fileUid = Guid.NewGuid();
var multiFileName = $"{fileUid}_Tiles$";
var tileIdentifier = $"tiles{DataOceanUtil.PathSeparator}xyz{DataOceanUtil.PathSeparator}21{DataOceanUtil.PathSeparator}822028{DataOceanUtil.PathSeparator}378508.png";
var tileDetail = $"{multiFileName}{DataOceanUtil.PathSeparator}{tileIdentifier}";
var downloadUrl = TestConstants.DownloadUrl;
var substitutedDownloadUrl = downloadUrl.Replace("{path}", tileIdentifier);
var config = ServiceProvider.GetRequiredService <IConfigurationStore>();
var dataOceanBaseUrl = config.GetValueString("DATA_OCEAN_URL");
var dataOceanRootFolderId = config.GetValueString("DATA_OCEAN_ROOT_FOLDER_ID");
var customerFolderName = "customerFolderName";
var projectFolderName = "projectFolderName";
var expectedCustomerFolderResult = new DataOceanDirectory {
Id = Guid.NewGuid(), Name = customerFolderName, ParentId = Guid.Parse(dataOceanRootFolderId)
};
var expectedCustomerBrowseResult =
new BrowseDirectoriesResult {
Directories = new List <DataOceanDirectory> {
expectedCustomerFolderResult
}
};
var expectedProjectFolderResult = new DataOceanDirectory
{
Id = Guid.NewGuid(),
Name = projectFolderName,
ParentId = expectedCustomerFolderResult.Id
};
var expectedProjectBrowseResult =
new BrowseDirectoriesResult {
Directories = new List <DataOceanDirectory> {
expectedProjectFolderResult
}
};
var browseCustomerUrl = $"{dataOceanBaseUrl}/api/browse/keyset_directories?name={customerFolderName}&owner=true&parent_id={dataOceanRootFolderId}";
var browseProjectUrl = $"{dataOceanBaseUrl}/api/browse/keyset_directories?name={projectFolderName}&owner=true&parent_id={expectedCustomerFolderResult.Id}";
var expectedFileBrowseResult = SetupExpectedSingleFileVersionResult(fileUid, multiFileName, expectedProjectFolderResult.Id, downloadUrl);
// got this "http://nowhere.in.particular/api/browse/keyset_files?name=f68e8360-eefb-4269-b456-37aa0a3f86b5_Tiles$&owner=true&parent_id=b39967e6-5733-4f8b-9d0c-31bfbad5c2c9"
var browseFileUrl = $"{dataOceanBaseUrl}/api/browse/keyset_files?name={multiFileName}&owner=true&parent_id={expectedProjectFolderResult.Id}";
var gracefulMock = new Mock <IWebRequest>();
gracefulMock
.Setup(g => g.ExecuteRequest <BrowseDirectoriesResult>(browseCustomerUrl, null, null, HttpMethod.Get, null, 0, false))
.Returns(Task.FromResult(expectedCustomerBrowseResult));
gracefulMock
.Setup(g => g.ExecuteRequest <BrowseDirectoriesResult>(browseProjectUrl, null, null, HttpMethod.Get, null, 0, false))
.Returns(Task.FromResult(expectedProjectBrowseResult));
gracefulMock
.Setup(g => g.ExecuteRequest <BrowseFilesResult>(browseFileUrl, null, null, HttpMethod.Get, null, 0, false))
.Returns(Task.FromResult(expectedFileBrowseResult));
gracefulMock
.Setup(g => g.ExecuteRequestAsStreamContent(substitutedDownloadUrl, HttpMethod.Get, null, null, null, 0, false))
.ThrowsAsync(new HttpRequestException(HttpStatusCode.Forbidden.ToString()));
ServiceCollection.AddTransient(g => gracefulMock.Object);
ServiceProvider = ServiceCollection.BuildServiceProvider();
var client = ServiceProvider.GetRequiredService <IDataOceanClient>();
(await client.GetTileCache().IsTileKnownToBeMissing(tileDetail)).Should().BeFalse();
// "/a3f51fdf-69e4-4d80-b734-e72e4f9b36d9/customerFolderName/projectFolderName/cb44d207-0bec-46ad-8bb3-4388137eae53_Tiles$/tiles/xyz/21/822028/378508.png"
var fullFileName = $"{DataOceanUtil.PathSeparator}{dataOceanRootFolderId}{DataOceanUtil.PathSeparator}{customerFolderName}{DataOceanUtil.PathSeparator}{projectFolderName}{DataOceanUtil.PathSeparator}{tileDetail}";
var resultStream = await client.GetFile(fullFileName, null);
resultStream.Should().BeNull();
(await client.GetTileCache().IsTileKnownToBeMissing(tileDetail)).Should().BeTrue();
// Generate an event, that will trigger a call to DataOceanClient for the file
var rasterTileNotificationParameters = new RasterTileNotificationParameters {
FileUid = fileUid
};
var notificationHub = ServiceProvider.GetService <INotificationHubClient>() as NotificationHubClient;
var tasks = notificationHub.ProcessNotificationAsTasks(new ProjectFileRasterTilesGeneratedNotification(rasterTileNotificationParameters));
Task.WaitAll(tasks.ToArray());
(await client.GetTileCache().IsTileKnownToBeMissing(tileDetail)).Should().BeFalse();
}
