private void ParseEntries(IEnumerable<FileInfo> files, LogEntry lastEntry, out long count)
{
count = 0;
// Only allow as many parallel threads as the size of the connection pool.
var options = new ParallelOptions { MaxDegreeOfParallelism = this.db.MaxConcurrentConnections };
var subCounts = new Dictionary<string, long>();
Parallel.ForEach(files, options, (file) =>
{
long subCount;
// Read entries from file and apply transformations.
var entries = this.ParseEntries(file, this.transformations).Where(AllowInsert(lastEntry));
// Write entries to the database.
this.db.Write(entries, out subCount);
subCounts.Add(file.FullName, subCount);
});
// Set count to sum of all entry counts per file.
count = subCounts.Values.Sum();
}
/// <summary>
/// Don't insert already existing records (determined by <c>LogRow</c> in latest inserted file).
/// </summary>
/// <param name="lastEntry"></param>
/// <returns></returns>
private static Func<LogEntry, bool> AllowInsert(LogEntry lastEntry)
{
if (lastEntry == null)
return (entry) => true;
return (entry) =>
entry.LogFilename != lastEntry.LogFilename || entry.LogRow > lastEntry.LogRow;
}
public void ParseEntries(IEnumerable<string> importedFileNames, LogEntry lastEntry, out long count)
{
if (importedFileNames == null)
throw new ArgumentNullException("importedFileNames");
if (lastEntry == null)
throw new ArgumentNullException("lastEntry");
IEnumerable<FileInfo> files = this.fileService.GetFiles(importedFileNames.ToArray(), lastEntry);
this.ParseEntries(files, lastEntry, out count);
}
public IEnumerable<FileInfo> GetFiles(IEnumerable<string> importedFileNames, LogEntry lastEntry)
{
if (importedFileNames == null)
throw new ArgumentNullException("importedFileNames");
if (lastEntry == null)
throw new ArgumentNullException("lastEntry");
return (from f in this.files
where f.FullName == lastEntry.LogFilename || !importedFileNames.Contains(f.FullName)
orderby f.Name
select f);
}
private LogEntry CreateEntry(IDictionary<int, string> fields, string[] values)
{
var result = new LogEntry();
foreach (int index in fields.Keys)
{
// Lesser values, than fields specified...
if (values.Length <= index)
continue;
this.adapter.SetValue(result, fields[index], values[index]);
}
return result;
}
private void TransformEntry(LogEntry entry, ITransformation[] transformations)
{
foreach (ITransformation transformation in transformations)
{
if (transformation == null)
continue;
transformation.Apply(entry);
}
}
