public async Task LogFileTotalRows(FileTransformationLogRecord logItem)
{
try
{
var readingFileLogId = await(from fileLog in _context.DataFeedFileLoadingLog
join runLog in _context.DataFeedRunLog on fileLog.FileReadingLogId equals runLog.FileReadingLogId
where fileLog.FilePath == logItem.FilePath
select fileLog.DataFeedFileLoadingLogId).FirstOrDefaultAsync();
var entity = _context.DataFeedFileLoadingLog.AsNoTracking()
.FirstOrDefault(x => x.DataFeedFileLoadingLogId.Equals(readingFileLogId));
if (entity != null)
{
entity.TotalRows = logItem == null ? 0 : logItem.TotalRows;
_context.Entry(entity).State = EntityState.Modified;
await _context.SaveChangesAsync();
}
}
catch (Exception e)
{
Console.WriteLine(e);
throw;
}
}
private async Task ProcessFile(string filePath)
{
try
{
var failedRows = new List <string>();
var successfulRows = new List <TalendResponseObject>();
var file = await _fileManager.GetFileAsync(filePath);
using (var client = new HttpClient())
{
using (var formData = new MultipartFormDataContent())
{
formData.Add(new ByteArrayContent(file));
//TODO: deal with PostAsync andResult. Do not block Task.
var response = client.PostAsync(_talendUrl, formData).Result;
if (response.IsSuccessStatusCode)
{
var responseContent = response.Content.ReadAsStringAsync().Result;
(successfulRows, failedRows) = TalendResponseParser.ParseTalendResponse(responseContent);
await client.PostAsync(_settings.LookupApiUrl,
new StringContent(JsonConvert.SerializeObject(successfulRows), Encoding.UTF8,
"application/json"));
var logItem = new FileTransformationLogRecord
{
DataFeedId = DataFeedId,
RunId = RunId,
FilePath = filePath,
InvalidRows = failedRows,
TotalRows = successfulRows.Count + failedRows.Count
};
await client.PostAsync(_settings.DataTransformationApiUrl, new StringContent(
JsonConvert.SerializeObject(logItem), Encoding.UTF8,
"application/json"));
}
else
{
_logger.LogError("Something went wrong in reformat process! {File} transformation went wrong.", filePath);
throw new Exception($"{filePath} failed to transform.");
}
}
}
}
catch (Exception e)
{
Console.WriteLine(e);
throw;
}
}
public async Task LogDataTransformation(FileTransformationLogRecord logItem)
{
var readingFileLogId = await(from fileLog in _context.DataFeedFileLoadingLog
join runLog in _context.DataFeedRunLog on fileLog.FileReadingLogId equals runLog.FileReadingLogId
where fileLog.FilePath == logItem.FilePath
select fileLog.DataFeedFileLoadingLogId).FirstOrDefaultAsync();
var items = logItem.InvalidRows.Select(x => new DataTransformationLog
{
DataTransformationLogId = Guid.NewGuid(),
DataFeedFileLoadingLogId = readingFileLogId,
ErrorRecordText = x
});
await _context.DataTransformationLog.AddRangeAsync(items);
await _context.SaveChangesAsync();
}
public async Task Post(FileTransformationLogRecord logItem)
{
await _repo.LogDataTransformation(logItem);
await _repo.LogFileTotalRows(logItem);
}
