public async Task CanGenerateGeoTiffTilesFailToCreateExecution()
{
//Set up DataOcean stuff
var expectedTopFolderResult = new DataOceanDirectory {
Id = Guid.NewGuid(), Name = topLevelFolderName
};
var expectedFileResult = new DataOceanFile {
Id = Guid.NewGuid(), Name = geoTiffFileName, ParentId = expectedTopFolderResult.Id
};
var subFolderPath = new DataOceanFileUtil(geoTiffFullName).GeneratedTilesFolder;
var dataOceanMock = new Mock <IDataOceanClient>();
dataOceanMock.Setup(d => d.GetFileId(geoTiffFullName, null)).ReturnsAsync(expectedFileResult.Id);
dataOceanMock.Setup(d => d.MakeFolder(subFolderPath, null)).ReturnsAsync(true);
dataOceanMock.Setup(d => d.GetFolderId($"{DataOceanUtil.PathSeparator}{topLevelFolderName}", null)).ReturnsAsync(expectedTopFolderResult.Id);
//Set up Pegasus stuff
var config = serviceProvider.GetRequiredService <Common.Abstractions.Configuration.IConfigurationStore>();
var pegasusBaseUrl = config.GetValueString("PEGASUS_URL");
var baseRoute = "/api/executions";
var createExecutionUrl = $"{pegasusBaseUrl}{baseRoute}";
var gracefulMock = new Mock <IWebRequest>();
gracefulMock
.Setup(g => g.ExecuteRequest <PegasusExecutionResult>(createExecutionUrl, It.IsAny <MemoryStream>(), null, HttpMethod.Post, null, 0,
false)).ReturnsAsync((PegasusExecutionResult)null);
await ProcessWithFailure(gracefulMock, dataOceanMock,
$"Failed to create execution for {geoTiffFullName}", false);
}
public BrowseFilesResult SetupExpectedSingleFileVersionResult(Guid fileUid, string multiFileName, Guid?expectedFolderResultId, string downloadUrl)
{
var updatedAt = DateTime.UtcNow.AddHours(-2);
var expectedFileResult = new DataOceanFile
{
Id = fileUid,
Name = multiFileName,
ParentId = expectedFolderResultId,
Multifile = true,
RegionPreferences = new List <string> {
"us1"
},
Status = "AVAILABLE",
DataOceanDownload = new DataOceanTransfer {
Url = downloadUrl
},
UpdatedAt = updatedAt
};
return(new BrowseFilesResult {
Files = new List <DataOceanFile> {
expectedFileResult
}
});
}
private Task <TileMetadata> CanGenerateDxfTiles(string status)
{
//Set up DataOcean stuff
var expectedTopFolderResult = new DataOceanDirectory {
Id = Guid.NewGuid(), Name = topLevelFolderName
};
var expectedDcFileResult = new DataOceanFile {
Id = Guid.NewGuid(), Name = dcFileName, ParentId = expectedTopFolderResult.Id
};
var expectedDxfFileResult = new DataOceanFile {
Id = Guid.NewGuid(), Name = dxfFileName, ParentId = expectedTopFolderResult.Id
};
var subFolderPath = new DataOceanFileUtil(dxfFullName).GeneratedTilesFolder;
var parts = subFolderPath.Split(DataOceanUtil.PathSeparator);
var subFolderName = parts[parts.Length - 1];
var dataOceanMock = new Mock <IDataOceanClient>();
dataOceanMock.Setup(d => d.GetFileId(dcFullName, null)).ReturnsAsync(expectedDcFileResult.Id);
dataOceanMock.Setup(d => d.GetFileId(dxfFullName, null)).ReturnsAsync(expectedDxfFileResult.Id);
dataOceanMock.Setup(d => d.MakeFolder(subFolderPath, null)).ReturnsAsync(true);
dataOceanMock.Setup(d => d.GetFolderId($"{DataOceanUtil.PathSeparator}{topLevelFolderName}", null)).ReturnsAsync(expectedTopFolderResult.Id);
//Set up Pegasus stuff
var units = DxfUnitsType.UsSurveyFeet.ToString();
var expectedExecution =
NewDxfPegasusExecution(expectedDcFileResult, expectedDxfFileResult, subFolderName, units, status);
var expectedExecutionResult = new PegasusExecutionResult {
Execution = expectedExecution
};
var expectedExecutionAttemptResult = new PegasusExecutionAttemptResult
{
ExecutionAttempt = new PegasusExecutionAttempt {
Id = Guid.NewGuid(), Status = ExecutionStatus.EXECUTING
}
};
var config = serviceProvider.GetRequiredService <Common.Abstractions.Configuration.IConfigurationStore>();
var pegasusBaseUrl = config.GetValueString("PEGASUS_URL");
var baseRoute = "/api/executions";
var createExecutionUrl = $"{pegasusBaseUrl}{baseRoute}";
var startExecutionUrl = $"{pegasusBaseUrl}{baseRoute}/{expectedExecution.Id}/start";
var executionStatusUrl = $"{pegasusBaseUrl}{baseRoute}/{expectedExecution.Id}";
var gracefulMock = new Mock <IWebRequest>();
gracefulMock
.Setup(g => g.ExecuteRequest <PegasusExecutionResult>(createExecutionUrl, It.IsAny <MemoryStream>(), null, HttpMethod.Post, null, 0,
false)).ReturnsAsync(expectedExecutionResult);
gracefulMock
.Setup(g => g.ExecuteRequest <PegasusExecutionAttemptResult>(startExecutionUrl, null, null, HttpMethod.Post, null, 0,
false)).ReturnsAsync(expectedExecutionAttemptResult);
gracefulMock
.Setup(g => g.ExecuteRequest <PegasusExecutionResult>(executionStatusUrl, null, null, HttpMethod.Get, null, 0,
false)).ReturnsAsync(expectedExecutionResult);
return(ProcessWithSuccess(gracefulMock, dataOceanMock, subFolderPath, true));
}
private Task <bool> CanPutFile(string status)
{
const string fileName = "dummy.dxf";
var expectedFile = new DataOceanFile
{
Id = Guid.NewGuid(),
Name = fileName,
Multifile = false,
RegionPreferences = new List <string> {
"us1"
},
Status = status,
DataOceanUpload = new DataOceanTransfer {
Url = TestConstants.UploadUrl
}
};
var expectedFileResult = new DataOceanFileResult {
File = expectedFile
};
var folderName = $"{DataOceanUtil.PathSeparator}";
var expectedBrowseResult = new BrowseDirectoriesResult {
Directories = new List <DataOceanDirectory>()
};
var expectedUploadResult = new StringContent("some ok result");
var config = serviceProvider.GetRequiredService <IConfigurationStore>();
var dataOceanBaseUrl = config.GetValueString("DATA_OCEAN_URL");
var browseUrl = $"{dataOceanBaseUrl}/api/browse/keyset_directories?name={folderName}&owner=true";
var createUrl = $"{dataOceanBaseUrl}/api/files";
var getUrl = $"{createUrl}/{expectedFile.Id}";
var deleteFileUrl = $"{dataOceanBaseUrl}/api/files/{expectedFileResult.File.Id}";
var gracefulMock = new Mock <IWebRequest>();
gracefulMock.Setup(g => g.ExecuteRequest <BrowseDirectoriesResult>(browseUrl, null, null, HttpMethod.Get, null, 0, false))
.Returns(Task.FromResult(expectedBrowseResult));
gracefulMock
.Setup(g => g.ExecuteRequest <DataOceanFileResult>(createUrl, It.IsAny <MemoryStream>(), null, HttpMethod.Post, null, 0, false))
.ReturnsAsync(expectedFileResult);
gracefulMock
.Setup(g => g.ExecuteRequestAsStreamContent(TestConstants.UploadUrl, HttpMethod.Put, null, It.IsAny <Stream>(), null, 0, false))
.ReturnsAsync(expectedUploadResult);
gracefulMock.Setup(g => g.ExecuteRequest <DataOceanFileResult>(getUrl, null, null, HttpMethod.Get, null, 0, false))
.Returns(Task.FromResult(expectedFileResult));
gracefulMock
.Setup(g => g.ExecuteRequest(deleteFileUrl, null, null, HttpMethod.Delete, null, 0, false))
.Returns(Task.FromResult(HttpStatusCode.Accepted));
serviceCollection.AddTransient(g => gracefulMock.Object);
var serviceProvider2 = serviceCollection.BuildServiceProvider();
var client = serviceProvider2.GetRequiredService <IDataOceanClient>();
return(client.PutFile(folderName, fileName, null));
}
public async Task CanDeleteExistingFile()
{
var config = serviceProvider.GetRequiredService <IConfigurationStore>();
var dataOceanBaseUrl = config.GetValueString("DATA_OCEAN_URL");
var dataOceanRootFolderId = config.GetValueString("DATA_OCEAN_ROOT_FOLDER_ID");
const string folderName = "unittest";
var expectedFolderResult = new DataOceanDirectory {
Id = Guid.NewGuid(), Name = folderName, ParentId = Guid.Parse(dataOceanRootFolderId)
};
var expectedFolderBrowseResult =
new BrowseDirectoriesResult {
Directories = new List <DataOceanDirectory> {
expectedFolderResult
}
};
const string fileName = "dummy.dxf";
var expectedFileResult = new DataOceanFile {
Id = Guid.NewGuid(), Name = fileName, ParentId = expectedFolderResult.Id
};
var expectedFileBrowseResult = new BrowseFilesResult {
Files = new List <DataOceanFile> {
expectedFileResult
}
};
var browseFolderUrl = $"{dataOceanBaseUrl}/api/browse/keyset_directories?name={folderName}&owner=true&parent_id={dataOceanRootFolderId}";
var browseFileUrl = $"{dataOceanBaseUrl}/api/browse/keyset_files?name={fileName}&owner=true&parent_id={expectedFolderResult.Id}";
var deleteFileUrl = $"{dataOceanBaseUrl}/api/files/{expectedFileResult.Id}";
var gracefulMock = new Mock <IWebRequest>();
gracefulMock
.Setup(g => g.ExecuteRequest <BrowseDirectoriesResult>(browseFolderUrl, null, null, HttpMethod.Get, null, 0, false))
.Returns(Task.FromResult(expectedFolderBrowseResult));
gracefulMock
.Setup(g => g.ExecuteRequest <BrowseFilesResult>(browseFileUrl, null, null, HttpMethod.Get, null, 0, false))
.Returns(Task.FromResult(expectedFileBrowseResult));
gracefulMock
.Setup(g => g.ExecuteRequest(deleteFileUrl, null, null, HttpMethod.Delete, null, 0, false))
.Returns(Task.FromResult(HttpStatusCode.Accepted));
serviceCollection.AddTransient(g => gracefulMock.Object);
var serviceProvider2 = serviceCollection.BuildServiceProvider();
var client = serviceProvider2.GetRequiredService <IDataOceanClient>();
var success =
await client.DeleteFile($"{DataOceanUtil.PathSeparator}{dataOceanRootFolderId}{DataOceanUtil.PathSeparator}{folderName}{DataOceanUtil.PathSeparator}{fileName}", null);
Assert.True(success);
}
public async Task CanGenerateGeoTiffTilesMissingGeoTiffFile()
{
var expectedTopFolderResult = new DataOceanDirectory {
Id = Guid.NewGuid(), Name = topLevelFolderName
};
_ = new DataOceanFile {
Id = Guid.NewGuid(), Name = geoTiffFileName, ParentId = expectedTopFolderResult.Id
};
var dataOceanMock = new Mock <IDataOceanClient>();
dataOceanMock.Setup(d => d.GetFileId(geoTiffFullName, null)).ReturnsAsync((Guid?)null);
var gracefulMock = new Mock <IWebRequest>();
await ProcessWithFailure(gracefulMock, dataOceanMock,
$"Failed to find GeoTIFF file {geoTiffFullName}. Has it been uploaded successfully?", false);
}
public async Task CanGenerateDxfTilesMissingDcFile()
{
var expectedTopFolderResult = new DataOceanDirectory {
Id = Guid.NewGuid(), Name = topLevelFolderName
};
var expectedDxfFileResult = new DataOceanFile {
Id = Guid.NewGuid(), Name = dxfFileName, ParentId = expectedTopFolderResult.Id
};
var dataOceanMock = new Mock <IDataOceanClient>();
dataOceanMock.Setup(d => d.GetFileId(dxfFullName, null)).ReturnsAsync(expectedDxfFileResult.Id);
dataOceanMock.Setup(d => d.GetFileId(dcFullName, null)).ReturnsAsync((Guid?)null);
var gracefulMock = new Mock <IWebRequest>();
await ProcessWithFailure(gracefulMock, dataOceanMock,
$"Failed to find coordinate system file {dcFullName}. Has it been uploaded successfully?", true);
}
private static PegasusExecution NewGeoTiffPegasusExecution(DataOceanFile expectedFileResult, string subFolderName, string status)
{
return(new PegasusExecution
{
Id = Guid.NewGuid(),
ProcedureId = new Guid("f61c965b-0828-40b6-8980-26c7ee164566"),
Parameters = new GeoTiffPegasusExecutionParameters
{
GeoTiffFileId = expectedFileResult.Id,
ParentId = expectedFileResult.ParentId,
TileOrder = "YX",
TileExportFormat = "xyz",
TileOutputFormat = "PNGRASTER",
TileCrs = "EPSG:3857",
MultiFile = "true",
Public = "false",
Name = subFolderName,
},
ExecutionStatus = status
});
}
private static PegasusExecution NewDxfPegasusExecution(DataOceanFile expectedDcFileResult, DataOceanFile expectedDxfFileResult, string subFolderName, string units, string status)
{
return(new PegasusExecution
{
Id = Guid.NewGuid(),
ProcedureId = new Guid("b8431158-1917-4d18-9f2e-e26b255900b7"),
Parameters = new DxfPegasusExecutionParameters
{
DcFileId = expectedDcFileResult.Id,
DxfFileId = expectedDxfFileResult.Id,
ParentId = expectedDxfFileResult.ParentId,
MaxZoom = "21",
TileType = "xyz",
TileOrder = "YX",
MultiFile = "true",
Public = "false",
Name = subFolderName,
AngularUnit = units,
PlaneUnit = units,
VerticalUnit = units
},
ExecutionStatus = status
});
}
public async Task CanGetNonExistingMultiFile()
{
var fileName = $"{DataOceanUtil.PathSeparator}tiles{DataOceanUtil.PathSeparator}15{DataOceanUtil.PathSeparator}18756{DataOceanUtil.PathSeparator}2834.png";
var downloadUrl = TestConstants.DownloadUrl;
var substitutedDownloadUrl = downloadUrl.Replace("{path}", fileName.Substring(1));
const string folderName = "unittest";
var expectedFolderResult = new DataOceanDirectory {
Id = Guid.NewGuid(), Name = folderName
};
var expectedFolderBrowseResult =
new BrowseDirectoriesResult {
Directories = new List <DataOceanDirectory> {
expectedFolderResult
}
};
const string multiFileName = "dummy.dxf_Tiles$";
var expectedFileResult = new DataOceanFile
{
Id = Guid.NewGuid(),
Name = multiFileName,
ParentId = expectedFolderResult.Id,
Multifile = true,
RegionPreferences = new List <string> {
"us1"
},
Status = "AVAILABLE",
DataOceanDownload = new DataOceanTransfer {
Url = downloadUrl
}
};
var expectedFileBrowseResult = new BrowseFilesResult {
Files = new List <DataOceanFile> {
expectedFileResult
}
};
var config = serviceProvider.GetRequiredService <IConfigurationStore>();
var dataOceanBaseUrl = config.GetValueString("DATA_OCEAN_URL");
var browseFolderUrl = $"{dataOceanBaseUrl}/api/browse/keyset_directories?name={folderName}&owner=true";
var browseFileUrl = $"{dataOceanBaseUrl}/api/browse/keyset_files?name={multiFileName}&owner=true&parent_id={expectedFolderResult.Id}";
var gracefulMock = new Mock <IWebRequest>();
gracefulMock
.Setup(g => g.ExecuteRequest <BrowseDirectoriesResult>(browseFolderUrl, null, null, HttpMethod.Get, null, 0, false))
.Returns(Task.FromResult(expectedFolderBrowseResult));
gracefulMock
.Setup(g => g.ExecuteRequest <BrowseFilesResult>(browseFileUrl, null, null, HttpMethod.Get, null, 0, false))
.Returns(Task.FromResult(expectedFileBrowseResult));
gracefulMock
.Setup(g => g.ExecuteRequestAsStreamContent(substitutedDownloadUrl, HttpMethod.Get, null, null, null, 0, false))
.ReturnsAsync((HttpContent)null);
serviceCollection.AddTransient(g => gracefulMock.Object);
var serviceProvider2 = serviceCollection.BuildServiceProvider();
var client = serviceProvider2.GetRequiredService <IDataOceanClient>();
var fullFileName = $"{DataOceanUtil.PathSeparator}{folderName}{DataOceanUtil.PathSeparator}{multiFileName}{fileName}";
var resultStream = await client.GetFile(fullFileName, null);
Assert.Null(resultStream);
}
public async Task CanGetExistingMultiFile()
{
var fileName = $"{DataOceanUtil.PathSeparator}tiles{DataOceanUtil.PathSeparator}tiles.json";
var downloadUrl = TestConstants.DownloadUrl;
var substitutedDownloadUrl = downloadUrl.Replace("{path}", fileName.Substring(1));
var expectedResult = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3 };
var stream = new MemoryStream(expectedResult);
var expectedDownloadResult = new StreamContent(stream);
var config = serviceProvider.GetRequiredService <IConfigurationStore>();
var dataOceanBaseUrl = config.GetValueString("DATA_OCEAN_URL");
var dataOceanRootFolderId = config.GetValueString("DATA_OCEAN_ROOT_FOLDER_ID");
const string folderName = "unittest";
var expectedFolderResult = new DataOceanDirectory {
Id = Guid.NewGuid(), Name = folderName, ParentId = Guid.Parse(dataOceanRootFolderId)
};
var expectedFolderBrowseResult =
new BrowseDirectoriesResult {
Directories = new List <DataOceanDirectory> {
expectedFolderResult
}
};
const string multiFileName = "dummy.dxf_Tiles$";
var fileUid = Guid.NewGuid();
var updatedAt = DateTime.UtcNow.AddHours(-2);
var expectedFileResult = new DataOceanFile
{
Id = fileUid,
Name = multiFileName,
ParentId = expectedFolderResult.Id,
Multifile = true,
RegionPreferences = new List <string> {
"us1"
},
Status = "AVAILABLE",
DataOceanDownload = new DataOceanTransfer {
Url = downloadUrl
},
UpdatedAt = updatedAt
};
var otherFileResult = new DataOceanFile
{
Id = fileUid,
Name = multiFileName,
ParentId = expectedFolderResult.Id,
Multifile = true,
RegionPreferences = new List <string> {
"us1"
},
Status = "AVAILABLE",
DataOceanDownload = new DataOceanTransfer {
Url = downloadUrl
},
UpdatedAt = updatedAt.AddHours(-5)
};
var expectedFileBrowseResult = new BrowseFilesResult {
Files = new List <DataOceanFile> {
expectedFileResult, otherFileResult
}
};
var browseFolderUrl = $"{dataOceanBaseUrl}/api/browse/keyset_directories?name={folderName}&owner=true&parent_id={dataOceanRootFolderId}";
var browseFileUrl = $"{dataOceanBaseUrl}/api/browse/keyset_files?name={multiFileName}&owner=true&parent_id={expectedFolderResult.Id}";
var gracefulMock = new Mock <IWebRequest>();
gracefulMock
.Setup(g => g.ExecuteRequest <BrowseDirectoriesResult>(browseFolderUrl, null, null, HttpMethod.Get, null, 0, false))
.Returns(Task.FromResult(expectedFolderBrowseResult));
gracefulMock
.Setup(g => g.ExecuteRequest <BrowseFilesResult>(browseFileUrl, null, null, HttpMethod.Get, null, 0, false))
.Returns(Task.FromResult(expectedFileBrowseResult));
gracefulMock
.Setup(g => g.ExecuteRequestAsStreamContent(substitutedDownloadUrl, HttpMethod.Get, null, null, null, 0, false))
.ReturnsAsync(expectedDownloadResult);
serviceCollection.AddTransient(g => gracefulMock.Object);
var serviceProvider2 = serviceCollection.BuildServiceProvider();
var client = serviceProvider2.GetRequiredService <IDataOceanClient>();
var fullFileName = $"{DataOceanUtil.PathSeparator}{dataOceanRootFolderId}{DataOceanUtil.PathSeparator}{folderName}{DataOceanUtil.PathSeparator}{multiFileName}{fileName}";
var resultStream = await client.GetFile(fullFileName, null);
using (var ms = new MemoryStream())
{
resultStream.CopyTo(ms);
var result = ms.ToArray();
Assert.Equal(expectedResult, result);
}
}