public async Task CSVExportRequest_Execute_ExceedsLimit()
{
DILoggingFixture.SetMaxExportRowsConfig(1);
AddApplicationGridRouting();
AddClusterComputeGridRouting();
var tempFileName = MockS3FileTransfer_UploadToBucket();
var siteModel = DITAGFileAndSubGridRequestsWithIgniteFixture.NewEmptyModel();
var request = new CSVExportRequest();
var baseDate = DateTime.SpecifyKind(new DateTime(2000, 1, 1, 1, 0, 0, 0), DateTimeKind.Utc);
var cellPasses = new CellPass[SubGridTreeConsts.SubGridTreeDimension, SubGridTreeConsts.SubGridTreeDimension][];
SubGridUtilities.SubGridDimensionalIterator((x, y) =>
{
cellPasses[x, y] = new[] { new CellPass {
Time = baseDate, Height = 1.0f
} };
});
DITAGFileAndSubGridRequestsFixture.AddSingleSubGridWithPasses(siteModel,
SubGridTreeConsts.DefaultIndexOriginOffset, SubGridTreeConsts.DefaultIndexOriginOffset, cellPasses);
var response = await request.ExecuteAsync(SimpleCSVExportRequestArgument(siteModel.ID));
response.Should().NotBeNull();
response.ResultStatus.Should().Be(RequestErrorStatus.ExportExceededRowLimit);
CleanupMockedFile(tempFileName, siteModel.ID);
}
public async Task CSVExportRequest_Execute_SingleCellSinglePass_CellProfileAllPasses()
{
AddApplicationGridRouting();
AddClusterComputeGridRouting();
var tempFileName = MockS3FileTransfer_UploadToBucket();
var siteModel = DITAGFileAndSubGridRequestsWithIgniteFixture.NewEmptyModel();
var request = new CSVExportRequest();
var baseDate = DateTime.SpecifyKind(new DateTime(2000, 1, 1, 1, 0, 0, 0), DateTimeKind.Utc);
var cellPasses = new[] { new CellPass {
Time = baseDate, Height = 1.0f
} };
DITAGFileAndSubGridRequestsFixture.AddSingleCellWithPasses(siteModel,
SubGridTreeConsts.DefaultIndexOriginOffset, SubGridTreeConsts.DefaultIndexOriginOffset, cellPasses);
var invalidatedStreams = new List <ISubGridSpatialAffinityKey>();
siteModel.ExistenceMap.ScanAllSubGrids(leaf =>
{
if (leaf is IServerLeafSubGrid serverLeaf)
{
(serverLeaf.Owner as ServerSubGridTree).SaveLeafSubGrid(serverLeaf, siteModel.PrimaryStorageProxy, siteModel.PrimaryStorageProxy, invalidatedStreams);
}
return(true);
});
siteModel.PrimaryStorageProxy.Commit();
DITAGFileAndSubGridRequestsFixture.ConvertSiteModelToImmutable(siteModel);
var response = await request.ExecuteAsync(SimpleCSVExportRequestArgument(siteModel.ID, OutputTypes.PassCountAllPasses));
response.Should().NotBeNull();
response.ResultStatus.Should().Be(RequestErrorStatus.OK);
// Read back the zip file
using (var archive = ZipFile.Open(tempFileName, ZipArchiveMode.Read))
{
var extractedFileName = tempFileName.Remove(tempFileName.Length - 4) + ".csv";
archive.Entries[0].ExtractToFile(extractedFileName);
var lines = File.ReadAllLines(extractedFileName);
lines.Length.Should().Be(2);
lines[0].Should()
.Be(
"Time,CellN,CellE,Elevation,PassNumber,LastRadioLtncy,DesignName,Machine,Speed,LastGPSMode,GPSAccTol,TargPassCount,ValidPos,Lift,LastCMV,TargCMV,LastMDP,TargMDP,LastRMV,LastFreq,LastAmp,TargThickness,MachineGear,VibeState,LastTemp");
lines[1].Should()
.Be(
@"2000/Jan/01 01:00:00.000,0.170m,0.170m,1.000m,1,0,?,""Unknown"",0.0km/h,Old Position,?,?,0,1,?,?,0.0,?,?,?,?,?,?,?,0.0°C");
}
CleanupMockedFile(tempFileName, siteModel.ID);
}
public async Task CSVExportRequest_Execute_EmptySiteModel()
{
AddApplicationGridRouting();
AddClusterComputeGridRouting();
var siteModel = DITAGFileAndSubGridRequestsWithIgniteFixture.NewEmptyModel();
var request = new CSVExportRequest();
var response = await request.ExecuteAsync(SimpleCSVExportRequestArgument(siteModel.ID));
response.Should().NotBeNull();
response.ResultStatus.Should().Be(RequestErrorStatus.FailedToRequestDatamodelStatistics);
}
public async Task CSVExportRequest_Execute_SingleSubGridSinglePass()
{
AddApplicationGridRouting();
AddClusterComputeGridRouting();
var tempFileName = MockS3FileTransfer_UploadToBucket();
var siteModel = DITAGFileAndSubGridRequestsWithIgniteFixture.NewEmptyModel();
var request = new CSVExportRequest();
var baseDate = DateTime.SpecifyKind(new DateTime(2000, 1, 1, 1, 0, 0, 0), DateTimeKind.Utc);
var cellPasses = new CellPass[SubGridTreeConsts.SubGridTreeDimension, SubGridTreeConsts.SubGridTreeDimension][];
SubGridUtilities.SubGridDimensionalIterator((x, y) =>
{
cellPasses[x, y] = new[] { new CellPass {
Time = baseDate, Height = 1.0f
} };
});
DITAGFileAndSubGridRequestsFixture.AddSingleSubGridWithPasses(siteModel,
SubGridTreeConsts.DefaultIndexOriginOffset, SubGridTreeConsts.DefaultIndexOriginOffset, cellPasses);
var response = await request.ExecuteAsync(SimpleCSVExportRequestArgument(siteModel.ID));
response.Should().NotBeNull();
response.ResultStatus.Should().Be(RequestErrorStatus.OK);
// Read back the zip file
using (var archive = ZipFile.Open(tempFileName, ZipArchiveMode.Read))
{
var extractedFileName = tempFileName.Remove(tempFileName.Length - 4) + ".csv";
archive.Entries[0].ExtractToFile(extractedFileName);
var lines = File.ReadAllLines(extractedFileName);
lines.Length.Should().Be(SubGridTreeConsts.SubGridTreeCellsPerSubGrid + 1);
lines[0].Should().BeEquivalentTo(
"Time,CellN,CellE,Elevation,PassCount,LastRadioLtncy,DesignName,Machine,Speed,LastGPSMode,GPSAccTol,TargPassCount,TotalPasses,Lift,LastCMV,TargCMV,LastMDP,TargMDP,LastRMV,LastFreq,LastAmp,TargThickness,MachineGear,VibeState,LastTemp");
lines[1].Should()
.BeEquivalentTo(
"2000/Jan/01 01:00:00.000,0.170m,0.170m,1.000m,1,0,?,\"Unknown\",0.0km/h,Old Position,?,?,1,1,?,?,0.0,?,?,?,?,?,?,?,0.0°C");
lines[10].Length.Should().Be(118);
lines[10].Should()
.BeEquivalentTo(
"2000/Jan/01 01:00:00.000,3.230m,0.170m,1.000m,1,0,?,\"Unknown\",0.0km/h,Old Position,?,?,1,1,?,?,0.0,?,?,?,?,?,?,?,0.0°C");
}
CleanupMockedFile(tempFileName, siteModel.ID);
}
public async Task CSVExportRequest_Execute_NoProductionData()
{
AddApplicationGridRouting();
AddClusterComputeGridRouting();
var tempFileName = MockS3FileTransfer_UploadToBucket();
var siteModel = DITAGFileAndSubGridRequestsWithIgniteFixture.NewEmptyModel();
var request = new CSVExportRequest();
var response = await request.ExecuteAsync(SimpleCSVExportRequestArgument(siteModel.ID));
response.Should().NotBeNull();
response.ResultStatus.Should().Be(RequestErrorStatus.FailedToRequestDatamodelStatistics);
CleanupMockedFile(tempFileName, siteModel.ID);
}
public async Task CSVExportRequest_Execute_UnableToWriteResultToS3()
{
AddApplicationGridRouting();
AddClusterComputeGridRouting();
var tempFileName = MockS3FileTransfer_UploadToBucket("InvalidFilename*@");
var siteModel = DITAGFileAndSubGridRequestsWithIgniteFixture.NewEmptyModel();
var request = new CSVExportRequest();
var baseDate = DateTime.SpecifyKind(new DateTime(2000, 1, 1, 1, 0, 0, 0), DateTimeKind.Utc);
var cellPasses = new CellPass[SubGridTreeConsts.SubGridTreeDimension, SubGridTreeConsts.SubGridTreeDimension][];
SubGridUtilities.SubGridDimensionalIterator((x, y) =>
{
cellPasses[x, y] = new[] { new CellPass {
Time = baseDate, Height = 1.0f
} };
});
DITAGFileAndSubGridRequestsFixture.AddSingleSubGridWithPasses(siteModel,
SubGridTreeConsts.DefaultIndexOriginOffset, SubGridTreeConsts.DefaultIndexOriginOffset, cellPasses);
var mockTransferProxy = new Mock <ITransferProxy>();
mockTransferProxy.Setup(t => t.UploadToBucket(It.IsAny <Stream>(), It.IsAny <string>(), It.IsAny <string>())).Callback(() => throw new IOException("S3 not available"));
var mockTransferProxyFactory = new Mock <ITransferProxyFactory>();
mockTransferProxyFactory.Setup(x => x.NewProxy(It.IsAny <TransferProxyType>())).Returns(mockTransferProxy.Object);
DIBuilder
.Continue()
.Add(x => x.AddSingleton(mockTransferProxyFactory.Object))
.Complete();
var response = await request.ExecuteAsync(SimpleCSVExportRequestArgument(siteModel.ID));
response.Should().NotBeNull();
response.ResultStatus.Should().Be(RequestErrorStatus.ExportUnableToLoadFileToS3);
CleanupMockedFile(tempFileName, siteModel.ID);
}
protected override async Task <ContractExecutionResult> ProcessAsyncEx <T>(T item)
{
var request = CastRequestObjectTo <CompactionCSVExportRequest>(item);
var siteModel = GetSiteModel(request.ProjectUid);
if (request.CoordType == CoordType.LatLon && siteModel.CSIBLoaded == false)
{
log.LogError($"#Out# CSVExportExecutor. CoordinateType of LatLong requested, but CSIB not found : Project: {request.ProjectUid} Filename: {request.FileName}");
throw CreateServiceException <CSVExportExecutor>
(HttpStatusCode.InternalServerError, ContractExecutionStatesEnum.InternalProcessingError,
RequestErrorStatus.ExportInvalidCSIB);
}
var filter = ConvertFilter(request.Filter, siteModel);
var startEndDate = CSVExportHelper.GetDateRange(siteModel, request.Filter);
filter.AttributeFilter.StartTime = startEndDate.Item1;
filter.AttributeFilter.EndTime = startEndDate.Item2;
var tRexRequest = new CSVExportRequest();
var csvExportRequestArgument = AutoMapperUtility.Automapper.Map <CSVExportRequestArgument>(request);
csvExportRequestArgument.MappedMachines = CSVExportHelper.MapRequestedMachines(siteModel, request.MachineNames);
csvExportRequestArgument.Filters = new FilterSet(filter);
var response = await tRexRequest.ExecuteAsync(csvExportRequestArgument);
if (response == null || response.ResultStatus != RequestErrorStatus.OK)
{
log.LogError($"CSVExportExecutor unable to process request. Project: {request.ProjectUid} Filename: {request.FileName} Response: {response?.ResultStatus.ToString()}");
throw CreateServiceException <CSVExportExecutor>
(HttpStatusCode.InternalServerError, ContractExecutionStatesEnum.InternalProcessingError,
response?.ResultStatus ?? RequestErrorStatus.FailedToConfigureInternalPipeline);
}
var s3FileTransfer = new S3FileTransfer(TransferProxyType.Temporary);
return(new CompactionExportResult(s3FileTransfer.GeneratePreSignedUrl(response.fileName)));
}
public void CSVExportRequest_Creation()
{
var request = new CSVExportRequest();
request.Should().NotBeNull();
}
