private async Task ProcessFile(string fileItem)
{
//Move the file from Src folder to Processing folder
var file = _jobStorage.MoveFileToProcessing(fileItem);
try
{
//Parse and serialize csv file
var products = _productsCsvParser.Parse(file);
var jsonString = JsonSerializer.Serialize(products, new JsonSerializerOptions
{
WriteIndented = true
});
//get path to destination folder
string destination = _jobStorage.GetPathToProcessedFolder();
string filename = $"{destination}\\{Guid.NewGuid()}.json";
//store file as json in the destination folder
await _jsonFileStorageService.StoreAsJson(filename, jsonString);
_jobStorage.RemoveFileFromProcessing(file.Substring(file.LastIndexOf("\\") + 1));
}
catch (Exception)
{
string filename = file.Substring(file.LastIndexOf("\\") + 1);
_jobStorage.MoveFileToFailed(filename);
}
}
public async Task Handle(FileUploadedEvent notification, CancellationToken cancellationToken)
{
_logger.LogInformation("JsonTransformer strated.");
//Move the file from Src folder to Processing folder
var file = _jobStorage.MoveFileToProcessing(notification.FileName);
_logger.LogInformation($"Processing {file}");
try
{
//Parse and serialize csv file
var products = _productsCsvParser.Parse(file);
var jsonString = JsonSerializer.Serialize(products, new JsonSerializerOptions {
WriteIndented = true
});
//get path to destination folder
string destination = _jobStorage.GetPathToProcessedFolder();
string filename = $"{destination}\\{Guid.NewGuid()}.json";
//store file as json in the destination folder
await _jsonFileStorageService.StoreAsJson(filename, jsonString);
_jobStorage.RemoveFileFromProcessing(file.Substring(file.LastIndexOf("\\") + 1));
}
catch (Exception e)
{
_logger.LogError($"Failed processing {file} , moving file to failed folder", e);
string filename = file.Substring(file.LastIndexOf("\\") + 1);
_jobStorage.MoveFileToFailed(filename);
}
}
public void ParseNormalOne()
{
IEnumerable <IEnumerable <string> > parsedData = csvParser.Parse(normalText);
Assert.IsNotNull(parsedData);
int numerOfLinesParsed = 0;
foreach (IEnumerable <string> innerLine in parsedData)
{
Assert.IsNotNull(innerLine);
Assert.AreEqual(16, innerLine.Count());
numerOfLinesParsed++;
}
Assert.AreEqual(4, numerOfLinesParsed);
}
public void Import(string csv)
{
var productsCsv = _csvParser.Parse(csv);
var products = productsCsv.Select(productCsv => _productFactory.Create(productCsv));
_productRepository.SaveAll(products);
_productJsonRepository.SaveAll(products);
}
public List <T> Load(UniversityInfo universityInfo)
{
if (universityInfo == null)
{
throw new ArgumentNullException(nameof(universityInfo));
}
return(_csvParser.Parse(GetPath(universityInfo.Uri.Host)));
}
public void parse_should_throw_exception_for_file_not_found()
{
//Given
_csvParser = new CsvParser();
//When
Action action = () => _csvParser.Parse <Employee>(@"TestFiles\invalidtest.csv");
//Then
action.Should().Throw <FileNotFoundException>();
}
public void parse_should_throw_missing_mapping_exception_for_no_mapper()
{
//Given
_csvParser = new CsvParser();
//When
Action action = () => _csvParser.Parse <InvalidTest>(@"TestFiles\employee.csv");
//then
action.Should().Throw <MissingMappingException>();
}
public void parse_should_parse_provided_file()
{
//Given
_csvParser = new CsvParser();
//when
var result = _csvParser.Parse <Employee>(@"TestFiles\employee.csv");
//then
result.Should().NotBeNullOrEmpty();
result.Count().Should().Be(4);
}
public List <UniversityInfo> Load() => _csvParser.Parse(FileName);
public async Task <ImportResult> ImportAsync(string csvPath,
ImportMethod method,
CancellationToken?token,
IProgress <int> progress)
{
IEnumerable <TModel> original = (LoadAsync == null) ? Enumerable.Empty <TModel>() : await LoadAsync();
if (token?.IsCancellationRequested ?? false)
{
return(null);
}
progress?.Report(20); // 進捗：取込対象データ読込完了
var unitOfWork = new UnitOfWorkWithReference <TModel, TIdentity>(
original, createIdentity);
var worker = new ImportWorker <TModel>(RowDef);
worker.LoginUserId = UserId;
worker.LoginUserCode = UserCode;
worker.LoginCompanyId = CompanyId;
worker.LoginCompanyCode = CompanyCode;
await InitializeWorker?.Invoke(worker);
var valid = true;
var csv = Parser.Parse(csvPath);
RowDef.SetupFields();
var fieldDefs = RowDef.Fields
.Where(f => !f.Ignored)
.OrderBy(f => f.FieldNumber)
.ToArray();
foreach (var fields in csv.Skip(RowDef.StartLineNumber))
{
worker.NewRecord(fields);
if (fields.Length < fieldDefs.Length)
{
foreach (var field in fieldDefs.Skip(fields.Length))
{
Errors.Add(new WorkingReport
{
LineNo = worker.RecordCount,
FieldNo = field.FieldIndex,
FieldName = field.FieldName,
Value = string.Empty,
Message = $"{field.FieldName}がありません。",
});
}
}
valid &= RowDef.Do(worker); // 書式チェック＆エラー情報収集
if (worker.Reports.Any())
{
Errors.AddRange(worker.Reports);
worker.Reports.Clear();
}
}
progress?.Report(40); // 進捗：CSVファイル読込完了
// DB関連チェック
var validate = new CodeToIdWorker <TModel>(worker); // importの結果を引き渡す
valid &= RowDef.Do(validate); // DBチェック＆エラー情報収集
Errors.AddRange(validate.Reports);
await AdditionalWorker?.Invoke(worker);
var additionalValidate = new AdditionalValidationWorker <TModel>(worker)
{
ImportMethod = method,
};
valid &= RowDef.Do(additionalValidate); // DBチェック＆エラー情報収集
Errors.AddRange(additionalValidate.Reports);
foreach (var lineNo in Errors.Where(x => x.LineNo.HasValue).Select(x => x.LineNo.Value).Distinct())
{
worker.Models.Remove(lineNo);
}
// DB重複チェック
if (!method.ValidateDuplicated(unitOfWork, worker))
{
worker.Reports.ForEach(r => worker.Models.Remove(r.LineNo.Value));
Errors.AddRange(worker.Reports);
worker.Reports.Clear();
}
// ファイル内キー重複チェック
var uniqueKeys = new Dictionary <TIdentity, int>();
var duplicatedLines = new List <int>();
foreach (var pair in validate.Models)
{
// 必須項目が空欄の時は、Nullまたはゼロになっている。
// キー項目が部分的にでも空欄の場合、重複チェックをパスする。
var identity = createIdentity(pair.Value);
if (ContainsNull(identity))
{
identity = default(TIdentity);
}
if (identity == null || identity.Equals(default(TIdentity)))
{
continue;
}
if (uniqueKeys.ContainsKey(identity))
{
switch (RowDef.DuplicateAdoption)
{
case DuplicateAdoption.BothAreErrors:
var duplicated = uniqueKeys[identity];
if (!duplicatedLines.Contains(duplicated))
{
duplicatedLines.Add(duplicated);
}
duplicatedLines.Add(pair.Key);
break;
case DuplicateAdoption.First:
duplicatedLines.Add(pair.Key);
break;
}
}
else
{
uniqueKeys.Add(identity, pair.Key);
}
if (token.HasValue && token.Value.IsCancellationRequested)
{
return(null);
}
}
progress?.Report(60); // 進捗：データベース関連チェック完了
switch (RowDef.TreatDuplicateAs)
{
case TreatDuplicateAs.Error:
duplicatedLines.ForEach(lineNo =>
{
Errors.Add(new WorkingReport() // キー重複
{
LineNo = lineNo,
FieldName = RowDef.KeyFields?.FirstOrDefault().FieldName ?? string.Empty,
Message = "重複しているため、インポートできません。",
Value = createIdentity(validate.Models[lineNo]).ToString(),
});
});
break;
case TreatDuplicateAs.Ignore:
// エラーにはせず、ここで取込対象から取り除く。
duplicatedLines.ForEach(lineNo => worker.Ignore(lineNo));
break;
}
if (token.HasValue && token.Value.IsCancellationRequested)
{
return(null);
}
// エラーデータを更新対象から取り除く
var errorLines = Errors
.GroupBy(report => report.LineNo)
.Select(g => g.Key)
.ToList();
if (errorLines.Any(lineNo => !lineNo.HasValue))
{
validate.Models.Clear();
}
else
{
errorLines.ForEach(lineNo => validate.Models.Remove(lineNo.Value));
}
if (token.HasValue && token.Value.IsCancellationRequested)
{
return(null);
}
progress?.Report(80); // 進捗：すべてのデータチェック完了
ImportResult result = null;
if (method.Import(unitOfWork, worker) && RegisterAsync != null)
{
result = await RegisterAsync(unitOfWork);
PostImportHanlder?.Invoke(worker, result);
Debug.Assert(result != null,
$"{nameof(RegisterAsync)}()が{nameof(ImportResult)}を返しませんでした。");
}
else // 登録処理をしなかった、または取込可能件数がゼロ
{
result = new ImportResult
{
ProcessResult = new ProcessResult {
Result = true
},
InsertCount = 0,
UpdateCount = 0,
DeleteCount = 0,
};
}
Errors.AddRange(worker.Reports);
if (Errors.Any() && !string.IsNullOrWhiteSpace(ErrorLogPath))
{
OutputErrorLog(ErrorLogPath, Errors, csvPath);
}
result.ValidItemCount = worker.Models.Count;
result.InvalidItemCount = worker.RecordCount - result.ValidItemCount;
UnitOfWork = unitOfWork;
progress?.Report(100); // 進捗：取込完了
return(result);
}
public IEnumerable <ImgwClimateMeteoData> GetMeteoData(Stream stream)
{
return(_csvParser.Parse(stream).Select(c => c.Result));
}