public void Persist_UnSuccessful_NoAWS()
{
var projectUid = Guid.NewGuid();
var csvExportUserPreference = new CSVExportUserPreferences();
var requestArgument = new CSVExportRequestArgument
(
projectUid, new FilterSet(new CombinedFilter()), "the filename",
CoordType.Northeast, OutputTypes.PassCountLastPass,
csvExportUserPreference, new List <CSVExportMappedMachine>(), false, false
)
{
TRexNodeID = Guid.NewGuid()
};
var originalConfiguration = DIContext.Obtain <IConfigurationStore>();
var moqConfiguration = DIContext.Obtain <Mock <IConfigurationStore> >();
moqConfiguration.Setup(c => c.GetValueString("AWS_TEMPORARY_BUCKET_NAME")).Returns((string)null);
moqConfiguration.Setup(c => c.GetValueString("AWS_TEMPORARY_BUCKET_NAME", It.IsAny <string>())).Returns((string)null);
DIBuilder
.Continue()
.Add(x => x.AddSingleton <IConfigurationStore>(moqConfiguration.Object))
.Complete();
Action act = () => new CSVExportFileWriter(requestArgument);
act.Should().Throw <ServiceException>();
DIBuilder
.Continue()
.Add(x => x.AddSingleton <IConfigurationStore>(originalConfiguration))
.Complete();
}
public void Persist_UnSuccessful()
{
var projectUid = Guid.NewGuid();
var csvExportUserPreference = new CSVExportUserPreferences();
var requestArgument = new CSVExportRequestArgument
(
projectUid, new FilterSet(new CombinedFilter()), null,
CoordType.Northeast, OutputTypes.PassCountLastPass,
csvExportUserPreference, new List <CSVExportMappedMachine>(), false, false
)
{
TRexNodeID = Guid.NewGuid()
};
var dataRows = new List <string>()
{
"string one", "string two"
};
var csvExportFileWriter = new CSVExportFileWriter(requestArgument);
Action act = () => csvExportFileWriter.PersistResult(dataRows);
act.Should().Throw <ArgumentNullException>();
var projectDir = Path.Combine(Path.GetTempPath(), projectUid.ToString());
if (Directory.Exists(projectDir))
{
Directory.Delete(projectDir, true);
}
}
public void Test_CSVExportRequestArgument_WithContent()
{
var request = new CSVExportRequestArgument
(
Guid.NewGuid(),
new FilterSet(new CombinedFilter()),
"the filename",
CoordType.LatLon,
OutputTypes.PassCountAllPasses,
new CSVExportUserPreferences(),
new List <CSVExportMappedMachine>()
{
new CSVExportMappedMachine()
{
Uid = Guid.NewGuid(), InternalSiteModelMachineIndex = 0, Name = "Machine 1"
},
new CSVExportMappedMachine()
{
Uid = Guid.NewGuid(), InternalSiteModelMachineIndex = 1, Name = "Machine 2"
},
new CSVExportMappedMachine()
{
Uid = Guid.NewGuid(), InternalSiteModelMachineIndex = 2, Name = "Machine 3"
}
}, false, false
);
SimpleBinarizableInstanceTester.TestClass(request, "Empty CSVExportRequestArgument not same after round trip serialization");
}
private List <IClientLeafSubGrid> GetSubGrids(CoordType coordType, OutputTypes outputType, bool isRawDataAsDBaseRequired,
out CSVExportRequestArgument requestArgument, out ISiteModel siteModel, string tagFileDirectory = "ElevationMappingMode-KettlewellDrive")
{
siteModel = SetupSiteAndRequestArgument(coordType, outputType, isRawDataAsDBaseRequired, tagFileDirectory, out requestArgument);
var overrides = requestArgument.Overrides;
var liftParams = requestArgument.LiftParams;
var utilities = DIContext.Obtain <IRequestorUtilities>();
var gridDataType = outputType == OutputTypes.PassCountLastPass || outputType == OutputTypes.VedaFinalPass
? GridDataType.CellProfile : GridDataType.CellPasses;
var requestors = utilities.ConstructRequestors(null, siteModel, requestArgument.Overrides, requestArgument.LiftParams,
utilities.ConstructRequestorIntermediaries(siteModel, requestArgument.Filters, false, gridDataType),
AreaControlSet.CreateAreaControlSet(), siteModel.ExistenceMap);
requestors.Should().NotBeNull();
requestors.Length.Should().Be(1);
// Request sub grids from the model
var requestedSubGrids = new List <IClientLeafSubGrid>();
siteModel.ExistenceMap.ScanAllSetBitsAsSubGridAddresses(x =>
{
var requestSubGridInternalResult = requestors[0].RequestSubGridInternal(x, true, false);
if (requestSubGridInternalResult.requestResult == ServerRequestResult.NoError)
{
requestedSubGrids.Add(requestSubGridInternalResult.clientGrid);
}
});
requestedSubGrids.Count.Should().Be(tagFileDirectory == "ElevationMappingMode-KettlewellDrive" ? 18 : 9);
return(requestedSubGrids);
}
public void CreateHeaders_VedaFinalPass()
{
var projectUid = Guid.NewGuid();
var csvExportUserPreference = new CSVExportUserPreferences();
var requestArgument = new CSVExportRequestArgument
(
projectUid, new FilterSet(new CombinedFilter()), "the filename",
CoordType.Northeast, OutputTypes.VedaFinalPass,
csvExportUserPreference, new List <CSVExportMappedMachine>(), false, false
)
{
TRexNodeID = Guid.NewGuid()
};
var dataRows = new List <string>()
{
"string one", "string two"
};
var mockTransferProxy = new Mock <ITransferProxy>();
mockTransferProxy.Setup(t => t.UploadToBucket(It.IsAny <Stream>(), It.IsAny <string>(), It.IsAny <string>()));
var mockTransferProxyFactory = new Mock <ITransferProxyFactory>();
mockTransferProxyFactory.Setup(x => x.NewProxy(It.IsAny <TransferProxyType>())).Returns(mockTransferProxy.Object);
DIBuilder
.Continue()
.Add(x => x.AddSingleton(mockTransferProxyFactory.Object))
.Complete();
var csvExportFileWriter = new CSVExportFileWriter(requestArgument, true);
csvExportFileWriter.PersistResult(dataRows);
var localExportPath = FilePathHelper.GetTempFolderForExport(requestArgument.ProjectID, "");
var uniqueFileName = requestArgument.FileName + "__" + requestArgument.TRexNodeID;
var firstFile = Path.Combine(localExportPath, uniqueFileName, requestArgument.FileName + CSV_extension);
if (!FileSystem.Exists(firstFile))
{
Assert.True(true, $"firstFile: {firstFile} should exist");
}
string header;
using (var fs = new StreamReader(firstFile)) header = fs.ReadLine();
header.Should().NotBeNullOrEmpty();
header.Should().Be("Time,CellN_m,CellE_m,Elevation_m,PassCount,LastRadioLtncy,DesignName,Machine,Speed_km/h,LastGPSMode,GPSAccTol_m,TargPassCount,TotalPasses,Lift,LastCMV,TargCMV,LastMDP,TargMDP,LastRMV,LastFreq_Hz,LastAmp_mm,TargThickness_m,MachineGear,VibeState,LastTemp,_c");
var projectDir = localExportPath.Remove(localExportPath.Length - 7);
if (Directory.Exists(projectDir))
{
Directory.Delete(projectDir, true);
}
}
public void Test_CSVExportRequestArgument_Empty()
{
var request = new CSVExportRequestArgument()
{
};
SimpleBinarizableInstanceTester.TestClass(request, "Empty CSVExportRequestArgument not same after round trip serialization");
}
public CSVExportSubGridProcessor(CSVExportRequestArgument requestArgument)
{
_maxExportRows = DIContext.Obtain <IConfigurationStore>().GetValueInt("MAX_EXPORT_ROWS", Consts.DEFAULT_MAX_EXPORT_ROWS);
_requestArgument = requestArgument;
_csvExportFormatter = new CSVExportFormatter(requestArgument.UserPreferences, requestArgument.OutputType, requestArgument.RawDataAsDBase);
_siteModel = DIContext.Obtain <ISiteModels>().GetSiteModel(requestArgument.ProjectID);
_cellPassTimeString = _coordString = _heightString = _lastDesignNameString = _lastMachineNameString =
_machineSpeedString = _gpsAccuracyToleranceString = _targetPassCountString = _lastPassValidCcvString =
_lastTargetCcvString = _lastPassValidMdpString = _lastTargetMdpString = _lastValidRmvString =
_lastValidFreqString = _lastValidAmpString = _lastTargetThicknessString = _lastEventMachineGearString =
_lastEventVibrationStateString = _lastPassValidTemperatureString = _csvExportFormatter.NullString;
}
private ISiteModel SetupSiteAndRequestArgument(CoordType coordType, OutputTypes outputType, bool isRawDataAsDBaseRequired, string tagFileDirectory,
out CSVExportRequestArgument requestArgument)
{
// tagFileDirectory: "Dimensions2018-CaseMachine" - extents match the CSIB constant
// "ElevationMappingMode-KettlewellDrive"
var tagFiles = Directory.GetFiles(Path.Combine("TestData", "TAGFiles", tagFileDirectory), "*.tag").ToArray();
var siteModel = DITAGFileAndSubGridRequestsFixture.BuildModel(tagFiles, out _);
var csvExportUserPreference = new CSVExportUserPreferences();
requestArgument = new CSVExportRequestArgument
(
siteModel.ID, new FilterSet(new CombinedFilter()), "the filename",
coordType, outputType, csvExportUserPreference, new List <CSVExportMappedMachine>(), false, isRawDataAsDBaseRequired
);
return(siteModel);
}
public void Persist_Successful()
{
var projectUid = Guid.NewGuid();
var csvExportUserPreference = new CSVExportUserPreferences();
var requestArgument = new CSVExportRequestArgument
(
projectUid, new FilterSet(new CombinedFilter()), "the filename",
CoordType.Northeast, OutputTypes.PassCountLastPass,
csvExportUserPreference, new List <CSVExportMappedMachine>(), false, false
)
{
TRexNodeID = Guid.NewGuid()
};
var dataRows = new List <string> {
"string one", "string two"
};
var mockTransferProxy = new Mock <ITransferProxy>();
mockTransferProxy.Setup(t => t.UploadToBucket(It.IsAny <Stream>(), It.IsAny <string>(), It.IsAny <string>()));
var mockTransferProxyFactory = new Mock <ITransferProxyFactory>();
mockTransferProxyFactory.Setup(x => x.NewProxy(It.IsAny <TransferProxyType>())).Returns(mockTransferProxy.Object);
DIBuilder
.Continue()
.Add(x => x.AddSingleton(mockTransferProxyFactory.Object))
.Complete();
var csvExportFileWriter = new CSVExportFileWriter(requestArgument);
var s3FullPath = csvExportFileWriter.PersistResult(dataRows);
s3FullPath.Should().NotBeNull();
s3FullPath.Should().Be($"project/{requestArgument.ProjectID}/TRexExport/{requestArgument.FileName}__{requestArgument.TRexNodeID}.zip");
var projectDir = Path.Combine(Path.GetTempPath(), projectUid.ToString());
if (Directory.Exists(projectDir))
{
Directory.Delete(projectDir, true);
}
}
public void CreateHeaders_RestrictOutputSize_TwoFiles()
{
var projectUid = Guid.NewGuid();
var userPreferences = new UserPreferences {
TemperatureUnits = (int)TemperatureUnitEnum.Fahrenheit
};
var csvExportUserPreference = AutoMapperUtility.Automapper.Map <CSVExportUserPreferences>(userPreferences);
var requestArgument = new CSVExportRequestArgument
(
projectUid, new FilterSet(new CombinedFilter()), "the filename",
CoordType.Northeast, OutputTypes.VedaFinalPass,
csvExportUserPreference, new List <CSVExportMappedMachine>(), true, true
)
{
TRexNodeID = Guid.NewGuid()
};
var dataRows = new List <string>(70000);
for (int i = 0; i < 70000; i++)
{
dataRows.Add($"row number: {i}");
}
;
var mockTransferProxy = new Mock <ITransferProxy>();
mockTransferProxy.Setup(t => t.UploadToBucket(It.IsAny <Stream>(), It.IsAny <string>(), It.IsAny <string>()));
var mockTransferProxyFactory = new Mock <ITransferProxyFactory>();
mockTransferProxyFactory.Setup(x => x.NewProxy(It.IsAny <TransferProxyType>())).Returns(mockTransferProxy.Object);
DIBuilder
.Continue()
.Add(x => x.AddSingleton(mockTransferProxyFactory.Object))
.Complete();
var csvExportFileWriter = new CSVExportFileWriter(requestArgument, true);
csvExportFileWriter.PersistResult(dataRows);
var localExportPath = FilePathHelper.GetTempFolderForExport(requestArgument.ProjectID, "");
var uniqueFileName = requestArgument.FileName + "__" + requestArgument.TRexNodeID;
var zipFullPath = Path.Combine(localExportPath, uniqueFileName) + ZIP_extension;
if (!FileSystem.Exists(zipFullPath))
{
Assert.True(true, $"zipFullPath: {zipFullPath} should exist");
}
using (var zip = ZipFile.OpenRead(zipFullPath))
{
zip.Entries.Count.Should().Be(2);
var firstFileName = Path.Combine(requestArgument.FileName + "(1)" + CSV_extension);
var firstFileNamePath = Path.Combine(localExportPath, uniqueFileName, firstFileName);
new FileInfo(firstFileNamePath).Length.Should().Be(1234316);
zip.Entries[0].Name.Should().Be(firstFileName);
var fileInfo = new FileInfo(firstFileNamePath);
zip.Entries[0].Length.Should().Be(fileInfo.Length);
var secondFileName = Path.Combine(requestArgument.FileName + "(2)" + CSV_extension);
var secondFileNamePath = Path.Combine(localExportPath, uniqueFileName, secondFileName);
new FileInfo(secondFileNamePath).Length.Should().Be(85096);
zip.Entries[1].Name.Should().Be(secondFileName);
fileInfo = new FileInfo(secondFileNamePath);
zip.Entries[1].Length.Should().Be(fileInfo.Length);
}
var projectDir = localExportPath.Remove(localExportPath.Length - 7);
if (Directory.Exists(projectDir))
{
Directory.Delete(projectDir, true);
}
}
public void CreateHeaders_RestrictOutputSize_SingleFile()
{
var projectUid = Guid.NewGuid();
var userPreferences = new UserPreferences()
{
TemperatureUnits = (int)TemperatureUnitEnum.Fahrenheit
};
var csvExportUserPreference = AutoMapperUtility.Automapper.Map <CSVExportUserPreferences>(userPreferences);
var requestArgument = new CSVExportRequestArgument
(
projectUid, new FilterSet(new CombinedFilter()), "the filename",
CoordType.Northeast, OutputTypes.VedaFinalPass,
csvExportUserPreference, new List <CSVExportMappedMachine>(), true, true
)
{
TRexNodeID = Guid.NewGuid()
};
var dataRows = new List <string> {
"string one", "string two"
};
var mockTransferProxy = new Mock <ITransferProxy>();
mockTransferProxy.Setup(t => t.UploadToBucket(It.IsAny <Stream>(), It.IsAny <string>(), It.IsAny <string>()));
var mockTransferProxyFactory = new Mock <ITransferProxyFactory>();
mockTransferProxyFactory.Setup(x => x.NewProxy(It.IsAny <TransferProxyType>())).Returns(mockTransferProxy.Object);
DIBuilder
.Continue()
.Add(x => x.AddSingleton(mockTransferProxyFactory.Object))
.Complete();
var csvExportFileWriter = new CSVExportFileWriter(requestArgument, true);
csvExportFileWriter.PersistResult(dataRows);
var localExportPath = FilePathHelper.GetTempFolderForExport(requestArgument.ProjectID, "");
var uniqueFileName = requestArgument.FileName + "__" + requestArgument.TRexNodeID;
var zipFullPath = Path.Combine(localExportPath, uniqueFileName) + ZIP_extension;
var firstFile = Path.Combine(localExportPath, uniqueFileName, requestArgument.FileName + CSV_extension);
if (!FileSystem.Exists(firstFile))
{
Assert.True(true, $"firstFile: {firstFile} should exist");
}
if (!FileSystem.Exists(zipFullPath))
{
Assert.True(true, $"zipFullPath: {zipFullPath} should exist");
}
using (var zip = ZipFile.OpenRead(zipFullPath))
{
zip.Entries.Count.Should().Be(1);
zip.Entries[0].Name.Should().Be(requestArgument.FileName + CSV_extension);
var fileInfo = new System.IO.FileInfo(firstFile);
fileInfo.Length.Should().Be(285);
zip.Entries[0].Length.Should().Be(fileInfo.Length);
}
string header;
using (var fs = new StreamReader(firstFile)) header = fs.ReadLine();
header.Should().NotBeNullOrEmpty();
header.Should().Be("Time,CellN_FT,CellE_FT,Elevation_FT,PassCount,lastRadioLtncy,DesignName,Machine,Speed_mph,LastGPSMode,GPSAccTol_FT,TargPassCount,TotalPasses,Lift,LastCMV,TargCMV,LastMDP,TargMDP,LastRMV,LastFreq_Hz,LastAmp_mm,TargThickness_FT,MachineGear,VibeState,LastTemp,_f");
var projectDir = localExportPath.Remove(localExportPath.Length - 7);
if (Directory.Exists(projectDir))
{
Directory.Delete(projectDir, true);
}
}
/// <summary> /// Constructor for the renderer accepting all parameters necessary for its operation /// </summary> public CSVExportComputeFuncExecutor(CSVExportRequestArgument arg) => _CSVExportRequestArgument = arg;
