public static Step <TInput, TOutput> UseFlatFileItemReader <TInput, TOutput>(this Step <TInput, TOutput> step,
string resourceUrl, IFieldSetMapper <TInput> fieldMapper, int linesToSkip, string[] headers, char token = DelimitedLineTokenizer.DEFAULT_TOKEN)
{
FlatFileItemReader <TInput> flatFileItemReader = CreateFlatFileReader(resourceUrl, fieldMapper, new DelimitedLineTokenizer(headers, token));
flatFileItemReader.LinesToSkip = linesToSkip;
return(step.SetReader(flatFileItemReader));
}
public void FlatFileTestTransactionalTest()
{
var writer = new FlatFileItemWriter <Person>
{
Resource = new FileSystemResource(_testPath2),
LineAggregator = new LineAggregator(),
HeaderWriter = new HeaderWriter()
};
var reader = new FlatFileItemReader <Person>
{
Resource = new FileSystemResource(_testPath2),
LinesToSkip = 2,
LineMapper = new LineMapper()
};
var executionContext = new ExecutionContext();
writer.Open(executionContext);
try
{
for (int i = 0; i < CountTransactions; i++)
{
using (TransactionScope scope = TransactionScopeManager.CreateScope())
{
writer.Write(GetPersons(i));
if (i == CountTransactions - 1) //SIMULATE FAILURE
{
throw new Exception("Bailing out ... should rollback ...");
}
scope.Complete();
}
}
}
catch (Exception e)
{
Logger.Error(e, "An unexpected exception occured :");
}
writer.Close();
var persons = new List <Person>();
reader.Open(executionContext);
Person person;
while ((person = reader.Read()) != null)
{
persons.Add(person);
}
reader.Close();
Assert.AreEqual((CountTransactions - 1) * CountObjects, persons.Count);
for (var i = 0; i < persons.Count; i++)
{
Assert.AreEqual(i % CountObjects, persons[i].Id);
Assert.IsTrue(persons[i].Name.StartsWith("Person" + i % CountObjects));
}
}
public void MapIsCalledAsManyTimesAsChunkSize(int chunkSize, int expected)
{
var fileService = new Mock<IFileService>();
fileService.Setup(f => f.ReadLines(0, chunkSize)).Returns(Enumerable.Range(0, chunkSize).Select(s => "item read"));
var itemReader = new FlatFileItemReader<string>(_lineMapper.Object, fileService.Object);
itemReader.Read(0, chunkSize);
_lineMapper.Verify(m => m.MapToModel(It.IsAny<string>()), Times.Exactly(expected));
}
public void LineMapperIsNotCalledIfItemIsNullOrEmpty()
{
var fileService = new Mock<IFileService>();
fileService.Setup(f => f.ReadLines(0, 1)).Returns(Enumerable.Empty<string>());
var itemReader = new FlatFileItemReader<string>(_lineMapper.Object, fileService.Object);
itemReader.Read(0, 1);
_lineMapper.Verify(m => m.MapToModel(It.IsAny<string>()), Times.Never());
}
public void LineMapperIsNotCalledIfItemIsNullOrEmpty()
{
var fileService = new Mock <IFileService>();
fileService.Setup(f => f.ReadLines(0, 1)).Returns(Enumerable.Empty <string>());
var itemReader = new FlatFileItemReader <string>(_lineMapper.Object, fileService.Object);
itemReader.Read(0, 1);
_lineMapper.Verify(m => m.MapToModel(It.IsAny <string>()), Times.Never());
}
public void MapIsCalledAsManyTimesAsChunkSize(int chunkSize, int expected)
{
var fileService = new Mock <IFileService>();
fileService.Setup(f => f.ReadLines(0, chunkSize)).Returns(Enumerable.Range(0, chunkSize).Select(s => "item read"));
var itemReader = new FlatFileItemReader <string>(_lineMapper.Object, fileService.Object);
itemReader.Read(0, chunkSize);
_lineMapper.Verify(m => m.MapToModel(It.IsAny <string>()), Times.Exactly(expected));
}
public void SkipsLinesOnlyWhenReadingForFirstTime(int linesToSkip, int expected)
{
const int CHUNK_SIZE = 10;
var fileService = new Mock<IFileService>();
fileService.Setup(f => f.ReadLines(0, CHUNK_SIZE)).Returns(Enumerable.Range(0, CHUNK_SIZE).Select(s => "item read"));
var itemReader = new FlatFileItemReader<string>(_lineMapper.Object, fileService.Object)
{
LinesToSkip = linesToSkip
};
itemReader.Read(0, CHUNK_SIZE);
_lineMapper.Verify(m => m.MapToModel(It.IsAny<string>()), Times.Exactly(expected));
}
public void FlatFileTestTest()
{
if (System.IO.File.Exists(_testPath))
{
System.IO.File.Delete(_testPath);
}
Assert.IsFalse(System.IO.File.Exists(_testPath));
var writer = new FlatFileItemWriter <Person>
{
Resource = new FileSystemResource(_testPath),
LineAggregator = new LineAggregator(),
HeaderWriter = new HeaderWriter()
};
var reader = new FlatFileItemReader <Person>
{
Resource = new FileSystemResource(_testPath),
LinesToSkip = 2,
LineMapper = new LineMapper()
};
var executionContext = new ExecutionContext();
writer.Open(executionContext);
writer.Write(GetPersons());
writer.Close();
Assert.IsTrue(System.IO.File.Exists(_testPath));
Assert.IsTrue(new FileInfo(_testPath).Length > 0);
var persons = new List <Person>();
reader.Open(executionContext);
Person person;
while ((person = reader.Read()) != null)
{
persons.Add(person);
}
reader.Close();
Assert.AreEqual(CountObjects, persons.Count);
for (var i = 0; i < CountObjects; i++)
{
Assert.AreEqual(i, persons[i].Id);
Assert.AreEqual("Person" + i, persons[i].Name);
}
}
public void SkipsLinesOnlyWhenReadingForFirstTime(int linesToSkip, int expected)
{
const int CHUNK_SIZE = 10;
var fileService = new Mock <IFileService>();
fileService.Setup(f => f.ReadLines(0, CHUNK_SIZE)).Returns(Enumerable.Range(0, CHUNK_SIZE).Select(s => "item read"));
var itemReader = new FlatFileItemReader <string>(_lineMapper.Object, fileService.Object)
{
LinesToSkip = linesToSkip
};
itemReader.Read(0, CHUNK_SIZE);
_lineMapper.Verify(m => m.MapToModel(It.IsAny <string>()), Times.Exactly(expected));
}
public void FlatFileTestRestart()
{
System.IO.File.Copy("TestData/FlatFile/output/Test.txt", _testpath3, true);
IResource fileResource = new FileSystemResource(new FileInfo(_testpath3));
Assert.IsTrue(fileResource.Exists());
var writer = new FlatFileItemWriter <Person>
{
Resource = new FileSystemResource(_testpath3),
LineAggregator = new LineAggregator(),
HeaderWriter = new HeaderWriter()
};
var reader = new FlatFileItemReader <Person>
{
Resource = new FileSystemResource(_testpath3),
LinesToSkip = 2,
LineMapper = new LineMapper()
};
var executionContext = new ExecutionContext();
executionContext.PutLong("FlatFileItemWriter.current.count", 133L); // Last position in Test.txt
executionContext.PutLong("FlatFileItemWriter.written", 10L); // Record already written in Test.txt
writer.Open(executionContext);
writer.Write(GetPersons());
writer.Close();
Assert.IsTrue(System.IO.File.Exists(_testpath3));
Assert.IsTrue(new FileInfo(_testpath3).Length > 0);
var persons = new List <Person>();
reader.Open(executionContext);
Person person;
while ((person = reader.Read()) != null)
{
persons.Add(person);
}
reader.Close();
Assert.AreEqual(CountObjects + 10, persons.Count); // The final record number must be 20+10.
}
