internal FilteredRecordReader(
IRecordReader reader,
Predicate<IRecord> condition)
: base(reader)
{
_condition = condition;
}
/// <summary>
/// Launches the engine.
/// </summary>
/// <param name="inputFilePaths">List of full input files to be processed.</param>
/// <param name="recordReader">Instance that reads the record from the input files.</param>
/// <param name="expression">Instance of the expression used to matched against the record.</param>
public override void LaunchEngine(String[] inputFilePaths, IRecordReader recordReader, IRecordMatchExpression expression)
{
this.recordWriter = CreateRecordWriter();
var engine = new Engine();
engine.FileOpened += this.FileOpenedHandler;
engine.FileRead += this.FileReadHandler;
engine.CheckForCancellation = this.CheckForCancellation;
this.worker = new BackgroundWorker();
this.worker.WorkerSupportsCancellation = true;
this.worker.DoWork += (sender, e) =>
{
engine.Execute(
inputFilePaths,
this.uiLogManager,
new FileReaderFactory(),
recordReader,
expression,
this.recordWriter,
this.statisticsManager,
this.statisticsManager);
if (engine.CheckForCancellation())
{
e.Cancel = true;
}
};
this.worker.RunWorkerCompleted += BackgroundWorkCompleted;
this.worker.RunWorkerAsync();
}
public IgcReader(Stream stream, IRecordReader recordReader)
{
if (recordReader == null)
throw new ArgumentNullException("recordReader");
this.recordReader = recordReader;
this.reader = new StreamReader(stream);
}
public void Dump(IRecordReader<string, string> reader, string title)
{
DumpHeader(title);
foreach (var k in reader.GetKeys())
foreach (var v in reader.GetValues(k))
WriteLine(k + " => " + v);
DumpFooter(title);
}
static IEnumerable<Record> ReadDocument(Stream stream, IRecordReader recordReader)
{
using (var reader = new IgcReader(stream, recordReader))
{
Record record;
while (reader.Read(out record))
yield return record;
}
yield break;
}
public FixedLengthRecordParser(TextReader reader, FixedLengthSchema schema, FixedLengthOptions options)
{
if (String.IsNullOrEmpty(options.RecordSeparator))
{
this.recordReader = new FixedLengthRecordReader(reader, schema.TotalWidth);
}
else
{
this.recordReader = new SeparatorRecordReader(reader, options.RecordSeparator);
}
}
/// <summary>
/// Launches the engine.
/// </summary>
/// <param name="inputFilePaths">List of full input files to be processed.</param>
/// <param name="recordReader">Instance that reads the record from the input files.</param>
/// <param name="expression">Instance of the expression used to matched against the record.</param>
public override void LaunchEngine(String[] inputFilePaths, IRecordReader recordReader, IRecordMatchExpression expression)
{
var recordWriter = CreateRecordWriter();
var engine = new Engine();
engine.FileOpened += this.FileOpenedHandler;
engine.FileRead += this.FileReadHandler;
engine.CheckForCancellation = this.CheckForCancellation;
// Get the UI thread now because this method is guaranteed to be running on that thread (since this
// method is called from a winform control handler method)
this.mainTaskScheduler = TaskScheduler.FromCurrentSynchronizationContext();
// Create a new cancellation source and token - the previous one may have been cancelled already.
this.cancellationTokenSource = new CancellationTokenSource();
this.cancellationToken = this.cancellationTokenSource.Token;
var task = Task.Factory.StartNew(() =>
{
engine.Execute(
inputFilePaths,
this.uiLogManager,
new FileReaderFactory(),
recordReader,
expression,
recordWriter,
this.statisticsManager,
this.statisticsManager);
}, this.cancellationToken);
var finishedTask = task.ContinueWith((antecedent) =>
{
if (antecedent.Exception != null)
{
this.uiLogManager.WriteMessagesToLogs("Job FAILED.");
var ae = antecedent.Exception.Flatten();
Int32 count = 1;
foreach (Exception e in ae.InnerExceptions)
{
this.uiLogManager.WriteMessagesToLogs(String.Format("EXCEPTION {0}: {1}", count++, e.Message));
this.uiLogManager.WriteMessagesToLogs("STACK: " + e.StackTrace);
}
}
else if (this.cancellationToken.IsCancellationRequested)
{
this.uiLogManager.WriteMessagesToLogs("CANCELLED.");
}
recordWriter.Close();
this.uiLogManager.Close();
this.mainForm.JobFinished();
}, TaskScheduler.FromCurrentSynchronizationContext());
}
public static void Copy(IRecordReader<string, string> reader, IRecordWriter<string, string> writer)
{
var keys = new List<string>(reader.GetKeys());
for (int i = 0; i < keys.Count; i++)
{
var key = keys[i];
if (i % 100 == 0 && i > 0)
Console.WriteLine("Copy key " + i + "/" + keys.Count);
var values = reader.GetValues(key);
foreach (var value in values)
writer.Write(key, value);
}
}
/// <summary>
/// Read a Record handler forward, optionally skipping n lines and starting at a record number > 0
/// </summary>
/// <param name="reader">Reader to get records</param>
/// <param name="forwardLines">Number of lines to skip before reading</param>
/// <param name="startLine">Lines already read from file</param>
internal ForwardReader(IRecordReader reader, int forwardLines, int startLine)
{
mReader = reader;
mFowardLines = forwardLines;
mLineNumber = startLine;
mFowardStrings = new string[mFowardLines + 1];
mRemaingLines = mFowardLines + 1;
for (int i = 0; i < mFowardLines + 1; i++) {
mFowardStrings[i] = mReader.ReadRecordString();
mLineNumber++;
if (mFowardStrings[i] == null) {
mRemaingLines = i;
break;
}
}
}
public void Execute(
String[] filePaths,
ILogManager logManager,
IStreamReaderFactory streamReaderFactory,
IRecordReader recordReader,
IRecordMatchExpression expression,
IRecordWriter recordWriter,
IStatisticsCollector statisticsCollector,
IStatisticsReporter statisticsReporter)
{
logManager.VerifyThatObjectIsNotNull("Parameter 'logManager' is null.");
try
{
this.logManager = logManager;
statisticsCollector.VerifyThatObjectIsNotNull("Parameter 'statisticsCollector' is null.");
this.statisticsCollector = statisticsCollector;
statisticsReporter.VerifyThatObjectIsNotNull("Parameter 'statisticsReporter' is null.");
this.statisticsReporter = statisticsReporter;
this.logManager.WriteMessagesToLogs("Run Started...");
Action<IStreamReader, Record> writeMatchedRecordMethod, writeUnmatchedRecordMethod;
this.DetermineOutputMethods(recordWriter, out writeMatchedRecordMethod, out writeUnmatchedRecordMethod);
this.Process(filePaths, streamReaderFactory, recordReader, expression, writeMatchedRecordMethod, writeUnmatchedRecordMethod);
recordWriter.Close();
this.statisticsReporter.WriteToLog(this.logManager);
this.logManager.WriteMessagesToLogs("Run Finished.");
}
catch (Exception exception)
{
logManager.WriteMessageToApplicationLog("EXCEPTION: " + exception.Message);
logManager.WriteMessageToApplicationLog("STACK: " + exception.StackTrace);
throw exception;
}
}
/// <summary>
/// Initializes a new instance of the SingleChildren class.
/// </summary>
/// <param name="recordReader">The recordReader that can read the children from the data stream.</param>
/// <param name="mapper">The mapper that puts the children into the parent.</param>
public SingleChildren(IRecordReader <TChild> recordReader, SingleChildMapper <TParent, TChild> mapper)
{
_recordReader = recordReader;
_mapper = mapper;
}
/// <summary>
/// Create a new instance of query executor
/// </summary>
/// <param name="connectionString"></param>
public QueryExecutor(string connectionString)
{
_connectionManager = new ConnectionManager(connectionString);
_recordReader = new RecordReader();
}
/// <summary>
/// Create a new instance of query executor
/// </summary>
/// <param name="connectionString"></param>
public QueryExecutor(string connectionString)
{
_connectionManager = new ConnectionManager(connectionString);
_recordReader = new RecordReader();
}
internal static CCRecordSet ClassifyAndUpdate(Stream oldFileStream, Stream newFileStream, IRecordReader reader, C3Configuration config)
{
CCRecordSet oldRecordSet = CCRecordSet.FromStream(oldFileStream, config);
List <CCRecord> newRecords = reader.ReadFromStream(newFileStream, config);
return(ClassifyAndUpdate(oldRecordSet, newRecords, reader, config));
}
/// <summary>
/// Initializes a new instance of the QueryReader class.
/// </summary>
/// <param name="recordReader">A mapping that can be used to read a single record from the stream.</param>
protected QueryReader(IRecordReader <T> recordReader)
{
RecordReader = recordReader;
_children = new List <Children <T> >();
}
/// <include file='MultiRecordEngine.docs.xml' path='doc/ReadStream/*'/>
public object[] ReadStream(IRecordReader reader)
{
if (reader == null)
{
throw new ArgumentNullException("reader", "The reader of the Stream can´t be null");
}
if (mRecordSelector == null)
{
throw new BadUsageException(
"The Recordselector can´t be null, please pass a not null Selector in the constructor.");
}
ResetFields();
mHeaderText = String.Empty;
mFooterText = String.Empty;
var resArray = new ArrayList();
using (var freader = new ForwardReader(reader, mMultiRecordInfo[0].IgnoreLast)) {
freader.DiscardForward = true;
mLineNumber = 1;
var completeLine = freader.ReadNextLine();
var currentLine = completeLine;
if (MustNotifyProgress) // Avoid object creation
{
OnProgress(new ProgressEventArgs(0, -1));
}
int currentRecord = 0;
if (mMultiRecordInfo[0].IgnoreFirst > 0)
{
for (int i = 0; i < mMultiRecordInfo[0].IgnoreFirst && currentLine != null; i++)
{
mHeaderText += currentLine + StringHelper.NewLine;
currentLine = freader.ReadNextLine();
mLineNumber++;
}
}
bool byPass = false;
var line = new LineInfo(currentLine)
{
mReader = freader
};
while (currentLine != null)
{
try {
mTotalRecords++;
currentRecord++;
line.ReLoad(currentLine);
var skip = false;
Type currType = null;
try {
currType = mRecordSelector(this, currentLine);
}
catch (Exception ex) {
throw new Exception("Selector failed to process correctly", ex);
}
if (currType != null)
{
var info = (RecordInfo)mRecordInfoHash[currType];
if (info == null)
{
throw new BadUsageException("A record is of type '" + currType.Name +
"' which this engine is not configured to handle. Try adding this type to the constructor.");
}
var record = info.Operations.CreateRecordHandler();
if (MustNotifyProgress) // Avoid object creation
{
OnProgress(new ProgressEventArgs(currentRecord, -1));
}
BeforeReadEventArgs <object> e = null;
if (MustNotifyRead) // Avoid object creation
{
e = new BeforeReadEventArgs <object>(this, record, currentLine, LineNumber);
skip = OnBeforeReadRecord(e);
if (e.RecordLineChanged)
{
line.ReLoad(e.RecordLine);
}
}
if (skip == false)
{
var values = new object[info.FieldCount];
if (info.Operations.StringToRecord(record, line, values))
{
if (MustNotifyRead) // Avoid object creation
{
skip = OnAfterReadRecord(currentLine, record, e.RecordLineChanged, LineNumber);
}
if (skip == false)
{
resArray.Add(record);
}
}
}
}
}
catch (Exception ex) {
switch (mErrorManager.ErrorMode)
{
case ErrorMode.ThrowException:
byPass = true;
throw;
case ErrorMode.IgnoreAndContinue:
break;
case ErrorMode.SaveAndContinue:
var err = new ErrorInfo {
mLineNumber = freader.LineNumber,
mExceptionInfo = ex,
mRecordString = completeLine
};
// err.mColumnNumber = mColumnNum;
mErrorManager.AddError(err);
break;
}
}
finally {
if (byPass == false)
{
currentLine = freader.ReadNextLine();
completeLine = currentLine;
mLineNumber = freader.LineNumber;
}
}
}
if (mMultiRecordInfo[0].IgnoreLast > 0)
{
mFooterText = freader.RemainingText;
}
}
return(resArray.ToArray());
}
/// <summary> /// Launches the engine. /// </summary> /// <param name="inputFilePaths">List of full input files to be processed.</param> /// <param name="recordReader">Instance that reads the record from the input files.</param> /// <param name="expression">Instance of the expression used to matched against the record.</param> public abstract void LaunchEngine(String[] inputFilePaths, IRecordReader recordReader, IRecordMatchExpression expression);
public T Deserialize(IRecordReader recordReader)
{
Generator.Method.Invoke(recordReader, out var instance);
return(instance);
}
public ProgressLoggedRecordReader(IRecordReader innerRecordReader)
{
_innerRecordReader = innerRecordReader;
}
/// <inheritdoc/>
public override bool Equals(IRecordReader other)
{
return(true);
}
/// <summary>
/// Initializes a new instance of the PostProcessRecordReader class.
/// </summary>
/// <param name="baseReader">The base record reader to use to read the record.</param>
/// <param name="postRead">The code to execute after reading the record.</param>
public PostProcessRecordReader(IRecordReader <T> baseReader, Func <IDataReader, T, T> postRead)
{
_baseReader = baseReader;
_postRead = postRead;
}
/// <summary>
/// A dynamic text parser.
/// </summary>
public Parser(IRecordReader recordReader)
{
_recordReader = recordReader;
}
/// <summary>
/// Initializes a new instance of the SingleReader class.
/// </summary>
/// <param name="mapping">The mapping to use to read objects from each record.</param>
public SingleReader(IRecordReader <T> mapping) : base(mapping)
{
}
/// <summary>
/// Initializes a new instance of the ListReaderAdapter class.
/// </summary>
/// <param name="recordReader">The one-to-one mapping to use to read objects from each record.</param>
public ListReaderAdapter(IRecordReader <T1> recordReader) : base(recordReader)
{
}
/// <summary>
/// Initializes a new instance of the ListReader class.
/// </summary>
/// <param name="recordReader">The one-to-one mapping to use to read objects from each record.</param>
public ListReader(IRecordReader <T> recordReader) : base(recordReader)
{
}
public InputBuffer(IRecordReader reader = null)
{
Reader = reader;
currentRecord = Record.Min;
}
/// <inheritdoc/> public abstract bool Equals(IRecordReader other);
/// <inheritdoc/>
public bool Equals(IRecordReader other)
{
return(other is MultiReader <TBase>);
}
//internal ForwardReader(IRecordReader reader)
// : this(reader, 0, 0)
//{ }
/// <summary>
/// Read a Record handler forward, optionally skipping n lines and starting at a record number > 0
/// </summary>
/// <param name="reader">Reader to get records</param>
/// <param name="forwardLines">NUmber of lines to skip before reading</param>
internal ForwardReader(IRecordReader reader, int forwardLines)
: this(reader, forwardLines, 0)
{
}
/// <summary>
/// Extends the reader by reading another set of records.
/// </summary>
/// <typeparam name="T1">The type of objects in the first set of results.</typeparam>
/// <typeparam name="T2">The type of objects in the second set of results.</typeparam>
/// <typeparam name="T3">The type of objects in the third set of results.</typeparam>
/// <typeparam name="T4">The type of objects in the fourth set of results.</typeparam>
/// <typeparam name="T5">The type of objects in the fifth set of results.</typeparam>
/// <typeparam name="T6">The type of objects in the sixth set of results.</typeparam>
/// <param name="previous">The previous reader.</param>
/// <param name="recordReader">The mapping that defines the layout of the records in each row.</param>
/// <returns>A reader that reads a Results object with multiple results.</returns>
public static ResultsReader <T1, T2, T3, T4, T5, T6> Then <T1, T2, T3, T4, T5, T6>(
this IQueryReader <Results <T1, T2, T3, T4, T5> > previous,
IRecordReader <T6> recordReader)
{
return(new ResultsReader <T1, T2, T3, T4, T5, T6>(previous, recordReader));
}
/// <summary>
/// Reads a record using the input formatter if required.
/// </summary>
/// <param name="reader">the reader to read from</param>
/// <returns>the read record</returns>
private T ReadRecord(IRecordReader <T> reader)
{
var record = reader.Read();
return(InputFormatter == null || record == null ? record : InputFormatter.Format(record));
}
/// <summary>
/// Extends the reader by reading another set of records.
/// </summary>
/// <typeparam name="T1">The type of objects in the first set of results.</typeparam>
/// <typeparam name="T2">The type of objects in the second set of results.</typeparam>
/// <typeparam name="T3">The type of objects in the third set of results.</typeparam>
/// <typeparam name="T4">The type of objects in the fourth set of results.</typeparam>
/// <typeparam name="T5">The type of objects in the fifth set of results.</typeparam>
/// <typeparam name="T6">The type of objects in the sixth set of results.</typeparam>
/// <typeparam name="T7">The type of objects in the seventh set of results.</typeparam>
/// <typeparam name="T8">The type of objects in the eighth set of results.</typeparam>
/// <typeparam name="T9">The type of objects in the nineth set of results.</typeparam>
/// <typeparam name="T10">The type of objects in the tenth set of results.</typeparam>
/// <typeparam name="T11">The type of objects in the eleventh set of results.</typeparam>
/// <typeparam name="T12">The type of objects in the twelfth set of results.</typeparam>
/// <typeparam name="T13">The type of objects in the thirteenth set of results.</typeparam>
/// <typeparam name="T14">The type of objects in the fourteenth set of results.</typeparam>
/// <typeparam name="T15">The type of objects in the fifteenth set of results.</typeparam>
/// <typeparam name="T16">The type of objects in the sixteenth set of results.</typeparam>
/// <param name="previous">The previous reader.</param>
/// <param name="recordReader">The mapping that defines the layout of the records in each row.</param>
/// <returns>A reader that reads a Results object with multiple results.</returns>
public static ResultsReader <T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16> Then <T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16>(
this IQueryReader <Results <T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15> > previous,
IRecordReader <T16> recordReader)
{
return(new ResultsReader <T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16>(previous, recordReader));
}
/// <summary>
/// Extends the reader by reading another set of records.
/// </summary>
/// <typeparam name="T1">The type of objects in the first set of results.</typeparam>
/// <typeparam name="T2">The type of objects in the second set of results.</typeparam>
/// <param name="previous">The previous reader.</param>
/// <param name="recordReader">The mapping that defines the layout of the records in each row.</param>
/// <returns>A reader that reads a Results object with multiple results.</returns>
public static ResultsReader <T1, T2> Then <T1, T2>(
this IQueryReader <Results <T1> > previous,
IRecordReader <T2> recordReader)
{
return(new ResultsReader <T1, T2>(previous, recordReader));
}
/// <summary>
/// Converts an IDataReader to a list of objects, using the specified record reader.
/// </summary>
/// <typeparam name="T">The type of object to return.</typeparam>
/// <param name="reader">The data reader.</param>
/// <param name="recordReader">The record reader to use.</param>
/// <returns>A list of objects.</returns>
public static IList <T> ToList <T>(this IDataReader reader, IRecordReader <T> recordReader)
{
return(reader.AsEnumerable(recordReader).ToList());
}
/// <summary>
/// Defines a reader that returns a list of records.
/// </summary>
/// <typeparam name="T">The type of object in the list of records.</typeparam>
/// <param name="recordReader">The mapping that defines the layout of the records in each row.</param>
/// <returns>A reader that reads a list of objects.</returns>
public static ListReader <T> Returns <T>(IRecordReader <T> recordReader = null)
{
return(new ListReader <T>(recordReader ?? OneToOne <T> .Records));
}
internal ForwardReader(IRecordReader reader)
: this(reader, 0, 0)
{
}
static IEnumerable<Record> GetHeaderRecords(Stream stream, IRecordReader recordReader)
{
using (var reader = new IgcReader(stream, recordReader))
{
Record record;
if (!reader.Read(out record) || record.RecordType != RecordType.A)
throw new FileLoadException("Failed to read the A-record");
while (reader.Read(out record))
yield return record;
}
}
/// <summary>
/// Defines a reader that returns a Results object.
/// </summary>
/// <typeparam name="T">The type of object in the first set of results.</typeparam>
/// <param name="recordReader">The mapping that defines the layout of the records in each row.</param>
/// <returns>A reader that reads a Results object.</returns>
public static ResultsReader <T> ReturnsResults <T>(IRecordReader <T> recordReader = null)
{
return(new ResultsReader <T>(recordReader ?? OneToOne <T> .Records));
}
//internal ForwardReader(IRecordReader reader)
// : this(reader, 0, 0)
//{ }
/// <summary>
/// Read a Record handler forward, optionally skipping n lines and starting at a record number > 0
/// </summary>
/// <param name="reader">Reader to get records</param>
/// <param name="forwardLines">NUmber of lines to skip before reading</param>
internal ForwardReader(IRecordReader reader, int forwardLines)
: this(reader, forwardLines, 0)
{
}
/// <summary>
/// Defines a reader that returns a single record.
/// </summary>
/// <typeparam name="T">The type of object in the list of records.</typeparam>
/// <param name="recordReader">The mapping that defines the layout of the records in each row.</param>
/// <returns>A reader that reads a list of objects.</returns>
public static SingleReader <T> ReturnsSingle <T>(IRecordReader <T> recordReader = null)
{
return(new SingleReader <T>(recordReader ?? OneToOne <T> .Records));
}
private void Process(String[] filePaths, IStreamReaderFactory streamReaderFactory, IRecordReader recordReader, IRecordMatchExpression expression, Action<IStreamReader, Record> writeMatchedRecordMethod, Action<IStreamReader, Record> writeUnmatchedRecordMethod)
{
var messageBuilder = new StringBuilder();
foreach (String filePath in filePaths)
{
this.logManager.WriteImportantMessageToJobLog("Processing '" + filePath + "'.");
var fileReader = streamReaderFactory.CreateStreamReader(filePath);
this.OnFileOpened(fileReader.Length);
Record record;
while (!expression.HasReachedMatchQuota && (record = recordReader.ReadRecord(fileReader)) != null)
{
this.OnFileRead(fileReader.Position);
messageBuilder.Append("Record found at position " + record.Start + " with Term '" + record.Term + "'");
if (expression.IsMatch(record))
{
this.statisticsCollector.RecordIsMatched(filePath);
messageBuilder.Append(" matches with List Term '" + record.Term + "'");
writeMatchedRecordMethod(fileReader, record);
}
else
{
this.statisticsCollector.RecordIsUnmatched(filePath);
writeUnmatchedRecordMethod(fileReader, record);
}
messageBuilder.Append(".");
this.logManager.WriteMessageToJobLog(messageBuilder.ToString());
messageBuilder.Clear();
if (this.IsCancelled())
{
break;
}
}
fileReader.Close();
}
}
/// <summary>
/// Extends the reader by reading another set of records.
/// </summary>
/// <typeparam name="T1">The type of objects in the first set of results.</typeparam>
/// <typeparam name="T2">The type of objects in the second set of results.</typeparam>
/// <param name="previous">The previous reader.</param>
/// <param name="recordReader">The mapping that defines the layout of the records in each row.</param>
/// <returns>A reader that reads a Results object with multiple results.</returns>
public static ResultsReader <T1, T2> Then <T1, T2>(this ListReader <T1> previous, IRecordReader <T2> recordReader = null)
{
return(new ResultsReader <T1>(previous).Then(recordReader));
}
/// <summary>
/// Initializes a new instance of the ChildRecordReader class.
/// </summary>
/// <param name="recordReader">The reader to get the whole record.</param>
/// <param name="grouping">The function to group the results.</param>
public ChildRecordReader(IRecordReader <TRecord> recordReader, Func <IEnumerable <TRecord>, IEnumerable <IGrouping <TId, TResult> > > grouping)
{
_recordReader = recordReader;
_grouping = grouping;
}
/// <summary>
/// コンストラクタ、全要素を指定して初期化する
/// </summary>
/// <param name="commandable">実体</param>
/// <param name="recordReader"><see cref="ICodeDbDataReader"/>から<typeparamref name="TResult"/>型のレコードを列挙するオブジェクト</param>
public FuncCmd(Commandable commandable, IRecordReader <TResult> recordReader)
{
this.Commandable = commandable;
this.RecordReader = recordReader;
}
