public bool PrepareCSV(string timeInUse = null)
{
List <string> dataPointsInUse;
if (timeInUse != null)
{
dataPointsInUse = new List <string> {
timeInUse
};
dataPointsInUse = GetDataPointsInUse(dataPointsInUse, "csvfield", "field");
}
else
{
dataPointsInUse = GetDataPointsInUse("csvfield", "field");
}
if (dataPointsInUse.Count == 0)
{
return(true);
}
CsvProcessor csvProcessor = new CsvProcessor(dataFile, dataRestURL, dataSourceFileOrURL, dataPointsInUse, node);
try {
dataRecords = csvProcessor.GetRecords();
} catch (Exception ex) {
sourceLogger.Error(ex, "Error retrieving CSV data.");
return(false);
}
return(true);
}
public IActionResult ImportPatientCredentials(ImportPatientCredentials importPatientCredentials)
{
// if all goes well, redirect to import patients with a success message
// otherwise, r
if (!CsvProcessor.IsCsv(importPatientCredentials.File))
{
ViewData["ErrorMessage"] = "Could not upload patients, the file provided was not a CSV file.";
return(View(viewName: "ImportPatients"));
}
List <string> errorMessages = new List <string>();
List <Patient> patients = CsvProcessor.ReadPatients(importPatientCredentials.File, errorMessages);
// No patients parsed then return with errors. If there are patients parsed but the user does not want to upload
// if there are errors, then simply return with errors
if (patients.Count == 0 || (errorMessages.Count > 0 && !importPatientCredentials.UploadWithErrors))
{
ViewData["ErrorMessage"] = "No patients were uploaded, check the file matches the format required.";
return(View(viewName: "ImportPatients"));
}
int successfulInserts = PatientProcessor.SavePatients(Patient.Convert(patients), errorMessages);
if (errorMessages.Count != 0)
{
ViewData["ErrorMessages"] = errorMessages;
}
ViewData["SuccessMessage"] = $"{successfulInserts} Patients were uploaded successfully.";
return(View(viewName: "ImportPatients"));
}
/// <summary>
/// Decrypts a specified csv file, which was created by XtractUniversal with column encryption enabled.
/// </summary>
/// <param name="args"></param>
/// <returns></returns>
private static async Task Main(string[] args)
{
if (!GetInfoFromArgs(args))
{
return;
}
// build metadata file name
metaDataFile = metaDataFile ?? $"{Path.GetFileNameWithoutExtension(sourceFile)}_metadata.json";
ivArray = ArrayPool <byte> .Shared.Rent(12);
await using FileStream metaDataStream = File.Open(metaDataFile, FileMode.Open, FileAccess.Read);
using StreamReader reader = new StreamReader(metaDataStream);
using CsvProcessor csvProcessor = new CsvProcessor(await reader.ReadToEndAsync());
string keyXml = await File.ReadAllTextAsync(keyFile);
using RSACryptoServiceProvider privateKey = new RSACryptoServiceProvider();
privateKey.FromXmlString(keyXml);
byte[] sessionKey = privateKey.Decrypt(csvProcessor.EncryptedSessionKey, true);
await using FileStream target = File.Open(targetFile, FileMode.Create, FileAccess.Write);
await using FileStream fs = File.Open(sourceFile, FileMode.Open, FileAccess.Read);
using (aesGcm = new AesGcm(sessionKey))
{
await csvProcessor.ProcessDataAsync(DecryptCell, fs.ReadAsync, target.WriteAsync,
CancellationToken.None);
ArrayPool <byte> .Shared.Return(ivArray);
}
}
public void CapitalCities_must_call_BulkAsync_method()
{
var mockFileSystem = new MockFileSystem();
var mockInputFile = new MockFileData("city,city_ascii,lat,lng,country,iso2,iso3,admin_name,capital,population,id\nPristina,Pristina,42.6666,21.1724,Kosovo,XK,XKS,Prishtinë,primary,,1901760068");
mockFileSystem.AddFile(@"capital_cities.csv", mockInputFile);
CapitalCityRecord record = new CapitalCityRecord()
{
City = "Pristina",
Country = "Kosovo",
Id = "1901760068",
CityAscii = "Pristina",
Population = null,
Latitude = new decimal(42.6666),
Longitude = new decimal(21.1724)
};
CapitalSearchDocument capitalSearch = new CapitalSearchDocument(record);
List <CapitalSearchDocument> cp = new List <CapitalSearchDocument>()
{
capitalSearch
};
var parameter = CsvProcessor.ProcessCsvFiles(mockFileSystem);
Assert.AreEqual(capitalSearch, parameter[0]);
}
public void LoadCSVFile()
{
// Should be able to read the CSV file and load all records for that application ID in list
CsvProcessor csvProcessor = new CsvProcessor();
List <LicenseDetails> lstRecords = csvProcessor.loadCSV("testData.csv", "374");
Assert.AreEqual(2, lstRecords.Count);
}
public void OneLicenseForTwoLaptops()
{
// Since there are two records for laptop and one laptop can be free provided there is one license.
CsvProcessor csvProcessor = new CsvProcessor();
List <LicenseDetails> lstRecords = csvProcessor.loadCSV("testData.csv", "374");
var result = csvProcessor.processRecords(lstRecords);
Assert.AreEqual(1, result);
}
public void ProcessCSVData()
{
// Should be able to process the CSV file and calculate the total license for an application ID.
CsvProcessor csvProcessor = new CsvProcessor();
List <LicenseDetails> lstRecords = csvProcessor.loadCSV("testData.csv", "375");
var result = csvProcessor.processRecords(lstRecords);
Assert.AreEqual(3, result);
}
public void ThreeLicenseForApp375()
{
// Since there is only one record for laptop and three for desktop, so only three license are required.
CsvProcessor csvProcessor = new CsvProcessor();
List <LicenseDetails> lstRecords = csvProcessor.loadCSV("testData.csv", "375");
var result = csvProcessor.processRecords(lstRecords);
Assert.AreEqual(3, result);
}
public void IgnoreDuplicateAndCalculate()
{
// Should consider only the first record and ignore the duplicate record for other entries with same computerID.
CsvProcessor csvProcessor = new CsvProcessor();
List <LicenseDetails> lstRecords = csvProcessor.loadCSV("testData.csv", "376");
var result = csvProcessor.processRecords(lstRecords);
Assert.AreEqual(2, result);
}
public void Emtpy_File_Should_Display_Nothing()
{
const string emptyFile = "emptyFile.csv";
var csvProcessor = new CsvProcessor();
csvProcessor.Process(emptyFile);
var result = csvProcessor.GetResult();
result.Should().BeEmpty();
}
public void OnlyOneLaptopIsFree()
{
// Since we have records for two laptops and two desktops. The number of license required should be 3
CsvProcessor csvProcessor = new CsvProcessor();
List <LicenseDetails> lstRecords = csvProcessor.loadCSV("testData.csv", "377");
var result = csvProcessor.processRecords(lstRecords);
Assert.AreEqual(3, result);
}
public void Person_File_Should_Display_Headerline_Correctly()
{
var csvProcessor = new CsvProcessor();
csvProcessor.Process("singleLineFile.csv");
var result = csvProcessor.GetResult();
result.Should().Be(
"|City|Age|Name|"
+ Environment.NewLine
+ "+----+---+----+");
}
public void Emtpy_File_Should_Display_Nothing()
{
const string emptyFile = "emptyFile.csv";
var csvProcessor = new CsvProcessor();
csvProcessor.Process(emptyFile);
var result = csvProcessor.GetResult();
result.Should().BeEmpty();
}
public void Person_File_Should_Display_Headerline_Correctly()
{
var csvProcessor = new CsvProcessor();
csvProcessor.Process("singleLineFile.csv");
var result = csvProcessor.GetResult();
result.Should().Be(
"|City|Age|Name|"
+ Environment.NewLine
+ "+----+---+----+");
}
public void ProcessFile_FilenameIsEmpty_ReturnsErrorStatus()
{
//Arrange
var expected = ProcessFileStatuses.Error;
var sut = new CsvProcessor();
//Act
var result = sut.ProcessFile(string.Empty);
//Assert
Assert.AreEqual(expected, result.Status);
}
public void ProcessFile_FilenameIsEmpty_ReturnsAnEmptyListOfItems()
{
//Arrange
var expected = new List <ProcessResultItem>();
var sut = new CsvProcessor(new FilesystemWrapperStub());
//Act
var actual = sut.ProcessFile(string.Empty);
//Assert
actual.Items.Should().AllBeEquivalentTo(expected);
}
public void FileNotFound()
{
CsvProcessor csvProcessor = new CsvProcessor();
try
{
csvProcessor.loadCSV("testData1.csv", "375");
}
catch (Exception ex)
{
Assert.AreEqual("File does not exist or bad data", ex.Message);
}
}
public IActionResult DisplayBatchCalculations(BatchCalculation batchCalculation)
{
// if all goes well, redirect to import patients with a success message
// otherwise, r
if (!CsvProcessor.IsCsv(batchCalculation.File))
{
ViewData["ErrorMessage"] = "Could not upload patients, the file provided was not a CSV file.";
return(View(viewName: "BatchCalculation"));
}
var errorMessages = CsvProcessor.ReadBatchPatientData(batchCalculation.File, out List <ListCalculations> calculatedPatients);
if (errorMessages.Count != 0)
{
ViewData["ErrorMessages"] = errorMessages;
}
return(View(viewName: "DisplayBatchCalculations", model: calculatedPatients));
}
public void ProcessFile_FileDoesNotExist_ReturnsAnEmptyListOfItems()
{
//Arrange
var expected = new List <ProcessResultItem>();
var fileSystemWrapperStub = new FilesystemWrapperStub();
fileSystemWrapperStub.Exists = false;
fileSystemWrapperStub.Path = "path";
fileSystemWrapperStub.Lines = new[] { "blah" };
var sut = new CsvProcessor(fileSystemWrapperStub);
//Act
var actual = sut.ProcessFile("filename");
//Assert
actual.Items.Should().AllBeEquivalentTo(expected);
}
public void TestSubcase(string outputFilename, PathFormat pathFormat,
LineEndingsNoFile lineEndings, bool autoOpen)
{
Mock <ICondition> mockCondition = new Mock <ICondition>();
Mock <IFieldSource> mockFieldSource = new Mock <IFieldSource>();
Mock <IRunInfo> mockRunInfo = new Mock <IRunInfo>();
mockCondition.Setup(c => c.ColumnCount).Returns(3);
mockCondition.Setup(c => c.ColumnHeaders).Returns(new string[] { "foo", "bar", "baz" });
mockFieldSource.Setup(f => f.ColumnCount).Returns(2);
mockFieldSource.Setup(f => f.ColumnHeaders).Returns(new string[] { "hello", "world" });
mockRunInfo.Setup(r => r.Condition).Returns(mockCondition.Object);
mockRunInfo.Setup(r => r.FieldSources).Returns(new IFieldSource[] { mockFieldSource.Object });
CsvProcessor csvProcessor = new CsvProcessor();
csvProcessor.SetParameter("Filename", GetCurrentTestResultsFilePath(outputFilename));
csvProcessor.SetParameter("PathFormat", pathFormat);
csvProcessor.SetParameter("LineEndings", lineEndings);
csvProcessor.SetParameter("AutoOpen", autoOpen);
csvProcessor.Init(mockRunInfo.Object);
FileInfo file1 = GetTestFile("BasicTextFile.txt");
FileInfo file2 = GetTestFile("TextFileWithNewlines.txt");
string[] vals1 = new string[] { "a", "b", "c", "d", "e" };
string[] vals2 = new string[] { "q", "w", "e", "r", "t" };
ProcessingResult result1;
ProcessingResult result2;
try
{
result1 = csvProcessor.Process(file1, MatchResultType.Yes, vals1,
new FileInfo[0], ProcessInput.OriginalFile, CancellationToken.None);
result2 = csvProcessor.Process(file2, MatchResultType.No, vals2,
new FileInfo[0], ProcessInput.OriginalFile, CancellationToken.None);
}
finally
{
csvProcessor.Cleanup();
}
Assert.AreEqual(ProcessingResultType.Success, result1.Type);
Assert.AreEqual(ProcessingResultType.Success, result2.Type);
}
public IFileProcessor CreateFileProcessor(FileType type)
{
IFileProcessor fileProcessor = null;
switch (type)
{
case FileType.CSV:
fileProcessor = new CsvProcessor();
break;
case FileType.JSON:
fileProcessor = new JsonProcessor();
break;
case FileType.XML:
fileProcessor = new XmlProcessor();
break;
}
return(fileProcessor);
}
public async Task RunAsync()
{
// if the index exists, let's delete it
// you probably don't want to do this kind of
// index management in a production environment
var index = await client.Indices.ExistsAsync(IndexName);
if (index.Exists)
{
await client.Indices.DeleteAsync(IndexName);
}
// let's create the index
var createResult =
await client.Indices.CreateAsync(IndexName, c => c
.Settings(s => s
.Analysis(a => a
// our custom search analyzer
.AddSearchAnalyzer()
)
)
.Map <CapitalSearchDocument>(m => m.AutoMap())
);
// let's load the data
var records = CsvProcessor.ProcessCsvFiles(fileSystem);
// we are pushing all the data in at once
var bullkResult =
await client
.BulkAsync(b => b
.Index(IndexName)
.CreateMany(records)
);
}
public IActionResult Index(ImportClinicianCredentials clinicianCredentials)
{
ClinicianManagerModel clinicianManagerModel = new ClinicianManagerModel();
if (clinicianCredentials.File == null)
{
ViewData["ErrorMessage"] = "Could not upload clinicians, no file was provided.";
clinicianManagerModel.Clinicians = Clinician.Convert(ClinicianProcessor.GetAllClinicians());
return(View(clinicianManagerModel));
}
if (!CsvProcessor.IsCsv(clinicianCredentials.File))
{
ViewData["ErrorMessage"] = "Could not upload clinicians, the file provided was not a CSV file.";
clinicianManagerModel.Clinicians = Clinician.Convert(ClinicianProcessor.GetAllClinicians());
return(View(clinicianManagerModel));
}
var errorMessages = CsvProcessor.GetClinicianCredentials(clinicianCredentials.File, out List <Clinician> clinicians);
if (clinicians.Count == 0 || (errorMessages.Count > 0 && !clinicianCredentials.UploadWithErrors))
{
ViewData["ErrorMessage"] = "No clinicians were uploaded, check the file matches the format required.";
clinicianManagerModel.Clinicians = Clinician.Convert(ClinicianProcessor.GetAllClinicians());
return(View(clinicianManagerModel));
}
int successfulInserts = ClinicianProcessor.SaveClinicians(Clinician.Convert(clinicians), errorMessages);
if (errorMessages.Count != 0)
{
ViewData["ErrorMessages"] = errorMessages;
}
ViewData["SuccessMessage"] = $"{successfulInserts} Clinicians were uploaded successfully.";
clinicianManagerModel.Clinicians = Clinician.Convert(ClinicianProcessor.GetAllClinicians());
return(View(clinicianManagerModel));
}
public IActionResult Index(IFormFile[] files)
{
IEnumerable <IFormFile> csvFiles = files.Where(file => Path.GetExtension(file.FileName) == ".csv");
var model = new FilesProcessingViewModel();
if (csvFiles.Any())
{
using (var processor = new CsvProcessor(_context, _logger))
{
var filesForProcessing = csvFiles.Select(formFile => new InputFile
{
FileName = formFile.FileName,
OpenStream = formFile.OpenReadStream
});
model.Result = processor.ProcessFiles(filesForProcessing);
}
}
else
{
model.Result = "There isn't csv-files";
}
return(View("FilesProcessing", model));
}
private static void ProcessFormat1(string pathSource, string pathOutput, string pathRef = null)
{
var refDic = new SortedDictionary <string, string>();
if (pathRef != null)
{
Console.Write($"Reading {pathRef} ...");
refDic = new CsvProcessor().ParseIntoDictionaryFromFile(pathRef);
Console.WriteLine("Done.");
}
var sw = new Stopwatch();
sw.Start();
Console.Write("Reading total number of lines in the CSV file: ");
var totalCount = File.ReadLines(pathSource).Count();
Console.WriteLine(totalCount);
var fileNameList = new List <string>();
var index = 0;
var errorCount = 0;
using (var writer = new CSVFilesWriteDispatcher(pathOutput))
using (var sr = new StreamReader(pathSource))
{
string line = null;
var firstLine = true;
while ((line = sr.ReadLine()) != null)
{
index++;
if (firstLine)
{
firstLine = false;
continue;
}
var item = refDic.Count == 0? new ReportModel(line) : new ReportModel(line, refDic);
if (item.IsValid)
{
writer.WriteToCsvFilesAccordingToMonth(item);
}
else
{
errorCount++;
}
if (index % 10000 == 0)
{
DrawTextProgressBar(index, totalCount);
}
}
DrawTextProgressBar(index, totalCount);
Console.WriteLine("Generating Report...");
fileNameList = writer.GetNamesOfGeneratedFile();
Console.WriteLine("Done.");
}
sw.Stop();
Console.WriteLine("Total Elapsed Time: " + sw.Elapsed);
Console.WriteLine("Error Count: " + errorCount);
sw.Restart();
Console.WriteLine("Generating Report ...");
GenerateFileReport(fileNameList, pathOutput);
sw.Stop();
Console.WriteLine("Total Elapsed Time: " + sw.Elapsed);
}
public HomeController()
{
//TODO: This can be an injected server
this.csvProcessor = new CsvProcessor();
}
public FileProcessController()
{
_fileUploader = new FileUploader();
_csvProcessor = new CsvProcessor();
_processFileResultMapper = new ProcessFileResultMapper();
}
public static async Task Run(
[BlobTrigger("<yourStorageBlob>/{name}", Connection = "AzureWebJobsStorage")]
Stream encryptedFile,
string name,
ILogger log)
{
// Saving the start time for diagnostics
DateTime startTime = DateTime.Now;
try
{
// The function triggers for every uploaded file.
// We just exclude the metadata file, to have maximum flexibility.
// Please adjust the trigger or filter for files to ignore to have the function match your purpose.
if (name.Contains(".json") || name.Contains("metadata"))
{
log.LogInformation("Metadata file upload. Exiting function.");
return;
}
ivArray = ArrayPool <byte> .Shared.Rent(12);
// preparing decryption
log.LogInformation("Loading account and container info...");
string conString = Environment.GetEnvironmentVariable("AzureWebJobsStorage");
sourceContainerName = Environment.GetEnvironmentVariable("SourceContainer");
targetContainerName = Environment.GetEnvironmentVariable("TargetContainer");
privateKeyFileName = Environment.GetEnvironmentVariable("PrivateKeyFileName");
metadataFileName = $"{Path.GetFileNameWithoutExtension(name)}_metadata.json";
// Remove this check if you are not using a connection string
if (string.IsNullOrWhiteSpace(conString))
{
throw new InvalidOperationException("No connection string was specified.");
}
if (string.IsNullOrWhiteSpace(privateKeyFileName))
{
throw new InvalidOperationException("No private key file was specified.");
}
if (string.IsNullOrWhiteSpace(sourceContainerName))
{
throw new InvalidOperationException("No source container was specified.");
}
if (string.IsNullOrWhiteSpace(targetContainerName))
{
throw new InvalidOperationException("No target container was specified.");
}
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(conString);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
sourceContainer = blobClient.GetContainerReference(sourceContainerName);
targetContainer = blobClient.GetContainerReference(targetContainerName);
log.LogInformation($"C# Blob trigger function Processed blob\n Name: {name}\n Size: {encryptedFile.Length} Bytes");
if (encryptedFile.Length == 0)
{
for (int retriesRemaining = 3; retriesRemaining > 0; retriesRemaining--)
{
if (encryptedFile.Length > 0)
{
break;
}
else
{
log.LogInformation("No data on the stream yet. Retrying in two seconds.");
await Task.Delay(2000);
}
}
if (encryptedFile.Length == 0)
{
log.LogInformation("No data received.");
return;
}
}
log.LogInformation("Loading metadata...");
log.LogInformation($"Expected metadata file name: {metadataFileName}");
using CsvProcessor csvProcessor = new CsvProcessor(await GetMetaDataAsync());
log.LogInformation("Decrypting session key...");
using RSACryptoServiceProvider privateKey = new RSACryptoServiceProvider();
privateKey.FromXmlString(await GetPrivateKeyAsync());
// decryption AES session key
byte[] sessionKey = privateKey.Decrypt(csvProcessor.EncryptedSessionKey, true);
log.LogInformation("Opening target stream...");
CloudBlockBlob plainTextBlob = targetContainer.GetBlockBlobReference(name);
await using CloudBlobStream uploadStream = await plainTextBlob.OpenWriteAsync();
// process and decrypt the data.
log.LogInformation("Decrypting data...");
using (aesGcm = new AesGcm(sessionKey))
{
await csvProcessor.ProcessDataAsync(DecryptCell, encryptedFile.ReadAsync, uploadStream.WriteAsync,
CancellationToken.None);
}
log.LogInformation("Wrapping up upload to destination blob.");
await uploadStream.CommitAsync();
}
catch (Exception e)
{
log.LogError(e.ToString());
}
finally
{
// cleanup some resources
if (ivArray != null)
{
ArrayPool <byte> .Shared.Return(ivArray);
}
log.LogInformation($"Function started at {startTime} terminated.");
}
}
private void UploadButton_Click(object sender, EventArgs e)
{
CsvProcessor.OpenFile(pathLabel, loadingGif, loadingLabel, workspaceDataGridView);
}
private void SaveButton_Click(object sender, EventArgs e)
{
CsvProcessor.SaveCsv(workspaceDataGridView);
}
private static void ApplyTemplate(string contentFolder, string siteFolder, FileInfo fi)
{
if (Verbose) Console.WriteLine("Processing "+ fi.Extension + " " + fi.Name);
if (fi.Extension == ".html") {
HtmlProcessor processor = new HtmlProcessor();
processor.Consume(contentFolder, siteFolder, fi.Name, fi.Extension);
FileUtils.WriteFile(processor.SiteFile, processor.Template, processor.Content);
SearchProcessor.TagSearchFile(processor.Content, contentFolder, siteFolder, fi);
} else if (fi.Extension == ".xml") {
string title = BijouUtils.ParsePageTitle(siteFolder+"/bogus.xxx");
XmlProcessor processor = new XmlProcessor();
processor.XslArgs = XmlProcessor.BuildXsltArgumentList(title);
processor.Consume(contentFolder, siteFolder, fi.Name, fi.Extension);
FileUtils.WriteFile(processor.SiteFile, processor.Content);
} else if (fi.Extension == ".csv") {
CsvProcessor processor = new CsvProcessor();
processor.Consume(contentFolder, siteFolder, fi.Name, fi.Extension);
FileUtils.WriteFile(processor.SiteFile, processor.Template, processor.Content);
} else if (fi.Extension == ".md") {
MdProcessor processor = new MdProcessor();
processor.Consume(contentFolder, siteFolder, fi.Name, fi.Extension);
FileUtils.WriteFile(processor.SiteFile, processor.Template, processor.Content);
} else if (fi.Extension == ".rss") {
RssProcessor processor = new RssProcessor();
processor.Consume(contentFolder, siteFolder, fi.Name, fi.Extension);
FileUtils.WriteFile(processor.SiteFile, processor.Template, processor.Content);
FileUtils.HtmlClone(contentFolder, siteFolder, fi.Name, fi.Extension, processor.Clone);
} else if (fi.Extension == ".ics") {
IcsProcessor processor = new IcsProcessor();
processor.Consume(contentFolder, siteFolder, fi.Name, fi.Extension);
FileUtils.WriteFile(processor.SiteFile, processor.Template, processor.Content);
FileUtils.HtmlClone(contentFolder, siteFolder, fi.Name, fi.Extension, processor.Clone);
}
}
private readonly int writeThreshold = 6 * (int)Math.Pow(2, 20); // 5 MB
#endregion
/// <summary>
/// You can modify this code as you wish.
///
/// Make sure not to change the decryption api interface logic to ensure the successful decryption of your data.
/// </summary>
/// <param name="input"></param>
/// <param name="context"></param>
/// <returns></returns>
/// <exception cref="InvalidOperationException"></exception>
/// <exception cref="FileNotFoundException"></exception>
public async Task <bool> FunctionHandler(S3EventNotification input, ILambdaContext context)
{
try
{
ivArray = ArrayPool <byte> .Shared.Rent(12);
this.targetFileName = this.sourceFileName = input.Records[0].S3.Object.Key;
LambdaLogger.Log("Loading and checking source/destination buckets, file name...");
this.sourceBucketName = Environment.GetEnvironmentVariable("sourcebucket");
this.targetBucketName = Environment.GetEnvironmentVariable("targetbucket");
this.keyId = Environment.GetEnvironmentVariable("privatekeyid");
// validate information
ValidateEnvironment();
LambdaLogger.Log(Environment.GetEnvironmentVariable("AWS_REGION"));
LambdaLogger.Log("Loading ciphertext...");
GetObjectRequest readRequest = new GetObjectRequest
{
BucketName = this.sourceBucketName,
Key = this.sourceFileName,
};
this.client = new AmazonS3Client(RegionEndpoint.USEast1);
using GetObjectResponse response = await this.client.GetObjectAsync(readRequest);
if (response.HttpStatusCode != HttpStatusCode.OK)
{
throw new FileNotFoundException("Could not retrieve file from source bucket.");
}
LambdaLogger.Log("Loading metadata...");
using CsvProcessor csvProcessor = new CsvProcessor(await GetMetaDataAsync());
// decrypt aes session key
byte[] sessionKey = await DecryptSessionKey(csvProcessor.EncryptedSessionKey);
LambdaLogger.Log(
$"Preparing multipart upload with a minimal part size of {this.writeThreshold.ToString()} bytes");
this.outputStream = new MemoryStream(this.partSize);
await InitPartUploadAsync();
LambdaLogger.Log("Decrypting...");
using (this.aesGcm = new AesGcm(sessionKey))
{
await csvProcessor.ProcessDataAsync(DecryptCell, response.ResponseStream.ReadAsync, WritePartAsync,
CancellationToken.None);
}
LambdaLogger.Log("Completing multipart upload...");
if (this.outputStream.Length > 0)
{
await WritePartInternalAsync();
}
await CompleteMultipartUploadAsync();
return(true);
}
catch (Exception ex)
{
LambdaLogger.Log($"Exception in PutS3Object: {ex}");
if (!string.IsNullOrWhiteSpace(this.uploadId))
{
// Abort the upload.
AbortMultipartUploadRequest abortRequest = new AbortMultipartUploadRequest
{
BucketName = this.targetBucketName,
Key = this.targetFileName,
UploadId = this.uploadId
};
await this.client.AbortMultipartUploadAsync(abortRequest);
}
return(false);
}
finally
{
// cleanup some resources
ArrayPool <byte> .Shared.Return(ivArray);
this.client?.Dispose();
}
}