/// <summary>
/// Launches the engine.
/// </summary>
/// <param name="inputFilePaths">List of full input files to be processed.</param>
/// <param name="recordReader">Instance that reads the record from the input files.</param>
/// <param name="expression">Instance of the expression used to matched against the record.</param>
public override void LaunchEngine(String[] inputFilePaths, IRecordReader recordReader, IRecordMatchExpression expression)
{
this.recordWriter = CreateRecordWriter();
var engine = new Engine();
engine.FileOpened += this.FileOpenedHandler;
engine.FileRead += this.FileReadHandler;
engine.CheckForCancellation = this.CheckForCancellation;
this.worker = new BackgroundWorker();
this.worker.WorkerSupportsCancellation = true;
this.worker.DoWork += (sender, e) =>
{
engine.Execute(
inputFilePaths,
this.uiLogManager,
new FileReaderFactory(),
recordReader,
expression,
this.recordWriter,
this.statisticsManager,
this.statisticsManager);
if (engine.CheckForCancellation())
{
e.Cancel = true;
}
};
this.worker.RunWorkerCompleted += BackgroundWorkCompleted;
this.worker.RunWorkerAsync();
}
public void TestSetup()
{
factory = Substitute.For <IStreamStateFactory>();
factory.Create(Arg.Any <string>()).Returns(_ => state);
state = Substitute.For <IStreamState>();
state.BufferPool.Returns(_ => pool);
state.Statistics.Returns(_ => stats);
state.RecordWriter.Returns(_ => writer);
state.SendSignal.Returns(new AsyncManualResetEvent(false));
buffer = Substitute.For <IBuffer>();
writer = Substitute.For <IRecordWriter>();
stats = Substitute.For <IStatisticsCollector>();
daemon = Substitute.For <IDaemon>();
pool = Substitute.For <IBufferPool>();
pool.TryAcquire(out _)
.Returns(
info =>
{
info[0] = buffer;
return(true);
});
log = new SynchronousConsoleLog();
sink = new HerculesSink(factory, daemon, log);
}
/// <summary>
/// Default constructor.
/// </summary>
/// <param name="writer">the underlying writer</param>
/// <param name="sum">the sum to use for similar items (or <code>null</code>)</param>
/// <param name="comparer">the comparer to use for records</param>
/// <param name="outputFormatter">the formatter to use when writing the records (or <code>null</code>)</param>
public SumWriter(IRecordWriter <T> writer, ISum <T> sum, IComparer <T> comparer, IFormatter <T> outputFormatter)
{
_writer = writer;
_sum = sum;
_comparer = comparer;
_outputFormatter = outputFormatter;
}
public void TestSetup()
{
baseWriter = Substitute.For <IRecordWriter>();
statistics = Substitute.For <IStatisticsCollector>();
signal = new AsyncManualResetEvent(false);
signalingWriter = new SignalingWriter(baseWriter, statistics, signal, 100, 0.25, 0.70);
}
public RecordFileWriter(FileInfo destination, Func <ISimpleLogger, IRecordWriter> factory)
{
Destination = destination;
var stream = new SimpleLoggerStreamWriter(File.Create(Destination.FullName));
_stream = stream;
_writer = factory(stream);
}
public void Write(IRecordWriter writer, string value)
{
if (_maximumLength != null)
value = _maximumLength.Write(value);
if (_minimumLength != null)
value = _minimumLength.Write(value);
writer.Write(value, _position, 0);
}
public SyncData(IRecordReader <T> recordReader,
IRecordWriter <T> recordWriter) : base()
{
RecordWriter = recordWriter;
RecordReader = recordReader;
RecordWriter.ErrorRaised += OnErrorRaised;
RecordReader.ErrorRaised += OnErrorRaised;
RecordReader.RecordPacketCompleted += OnReaderPacketComplete;
RecordReader.ReadCompleted += OnReadCompleteAsync;
}
public TextWrangler(string recordConfigName,
IRecordReader recordReader,
IRecordBuilder recordBuilder = null,
IFieldFormatter fieldFormatter = null,
IFieldFilterService fieldFilterService = null,
IRecordWriter recordWriter = null)
: this(RecordConfigurationBuilder.Build(recordConfigName), recordReader,
recordBuilder, fieldFormatter, fieldFilterService, recordWriter)
{
}
public void Write(IRecordWriter writer, string value)
{
if (_maximumLength != null)
value = _maximumLength.Write(value);
if (_minimumLength != null)
value = _minimumLength.Write(value);
writer.Write(value, 0, _index);
}
public StringWriter(Encoding encoding, IRecordWriter recordWriter)
{
_recordWriter = recordWriter;
_encoder = encoding.GetEncoder();
_stringBytesBuffer = new byte[encoding.GetMaxByteCount(CharsBufferSize)];
_stringSegmentBuffer = new byte[256];
_stringSegmentBuffer[255] = 0x20;
}
public void Write(IRecordWriter writer, string value)
{
if (_maximumLength != null)
{
value = _maximumLength.Write(value);
}
if (_minimumLength != null)
{
value = _minimumLength.Write(value);
}
writer.Write(value, 0, _index);
}
public static void Copy(IRecordReader<string, string> reader, IRecordWriter<string, string> writer)
{
var keys = new List<string>(reader.GetKeys());
for (int i = 0; i < keys.Count; i++)
{
var key = keys[i];
if (i % 100 == 0 && i > 0)
Console.WriteLine("Copy key " + i + "/" + keys.Count);
var values = reader.GetValues(key);
foreach (var value in values)
writer.Write(key, value);
}
}
public void Write(IRecordWriter writer, string value)
{
if (_maximumLength != null)
{
value = _maximumLength.Write(value);
}
if (_minimumLength != null)
{
value = _minimumLength.Write(value);
}
writer.Write(value, _position, 0);
}
public SignalingWriter(
IRecordWriter baseWriter,
IStatisticsCollector statistics,
AsyncManualResetEvent signal,
long sizeLimit,
double transitionSignalFraction,
double constantSignalFraction)
{
this.baseWriter = baseWriter;
this.statistics = statistics;
this.signal = signal;
transitionSignalThreshold = (long)(sizeLimit * transitionSignalFraction);
constantSignalThreshold = (long)(sizeLimit * constantSignalFraction);
}
public StreamState(
[NotNull] string name,
[NotNull] IBufferPool bufferPool,
[NotNull] IMemoryAnalyzer memoryAnalyzer,
[NotNull] IRecordWriter recordWriter,
[NotNull] IStatisticsCollector statistics,
[NotNull] AsyncManualResetEvent sendSignal)
{
Name = name ?? throw new ArgumentNullException(nameof(name));
BufferPool = bufferPool ?? throw new ArgumentNullException(nameof(bufferPool));
Statistics = statistics ?? throw new ArgumentNullException(nameof(statistics));
RecordWriter = recordWriter ?? throw new ArgumentNullException(nameof(recordWriter));
SendSignal = sendSignal ?? throw new ArgumentNullException(nameof(sendSignal));
MemoryAnalyzer = memoryAnalyzer ?? throw new ArgumentNullException(nameof(memoryAnalyzer));
}
/// <summary>
/// Merges the temporary files to the final output file
/// </summary>
/// <param name="buffers">the buffers to read the temporary files</param>
/// <param name="writer">the writer for the output file</param>
private static void Merge(IEnumerable <RecordReaderBuffer <T> > buffers, IRecordWriter <T> writer)
{
// Buffers are stored in a priority queue to have the buffers with the
// lowest record (with respect to Comparer) as the first buffer
var queue = new PriorityQueue <RecordReaderBuffer <T> >(buffers);
while (queue.Count > 0)
{
var buffer = queue.Poll();
var record = buffer.Read();
writer.Write(record);
// If the buffer has still records, we put it back in the queue
// so that is is correctly sorted
if (buffer.HasNext())
{
queue.Add(buffer);
}
}
}
public void Execute(
String[] filePaths,
ILogManager logManager,
IStreamReaderFactory streamReaderFactory,
IRecordReader recordReader,
IRecordMatchExpression expression,
IRecordWriter recordWriter,
IStatisticsCollector statisticsCollector,
IStatisticsReporter statisticsReporter)
{
logManager.VerifyThatObjectIsNotNull("Parameter 'logManager' is null.");
try
{
this.logManager = logManager;
statisticsCollector.VerifyThatObjectIsNotNull("Parameter 'statisticsCollector' is null.");
this.statisticsCollector = statisticsCollector;
statisticsReporter.VerifyThatObjectIsNotNull("Parameter 'statisticsReporter' is null.");
this.statisticsReporter = statisticsReporter;
this.logManager.WriteMessagesToLogs("Run Started...");
Action<IStreamReader, Record> writeMatchedRecordMethod, writeUnmatchedRecordMethod;
this.DetermineOutputMethods(recordWriter, out writeMatchedRecordMethod, out writeUnmatchedRecordMethod);
this.Process(filePaths, streamReaderFactory, recordReader, expression, writeMatchedRecordMethod, writeUnmatchedRecordMethod);
recordWriter.Close();
this.statisticsReporter.WriteToLog(this.logManager);
this.logManager.WriteMessagesToLogs("Run Finished.");
}
catch (Exception exception)
{
logManager.WriteMessageToApplicationLog("EXCEPTION: " + exception.Message);
logManager.WriteMessageToApplicationLog("STACK: " + exception.StackTrace);
throw exception;
}
}
public TextWrangler(RecordConfiguration recordConfiguration,
IRecordReader recordReader,
IRecordBuilder recordBuilder = null,
IFieldFormatter fieldFormatter = null,
IFieldFilterService fieldFilterService = null,
IRecordWriter recordWriter = null)
{
_recordConfiguration = recordConfiguration ?? throw new ArgumentNullException(nameof(recordReader));
_recordReader = recordReader ?? throw new ArgumentNullException(nameof(recordReader));
// Validate the configuration
RecordConfigurationValidator.Instance.Validate(_recordConfiguration, _fieldFilterService);
_recordBuilder = recordBuilder ?? TextWranglerConfig.DefaultRecordBuilder;
_fieldFormatter = fieldFormatter ?? TextWranglerConfig.DefaultFieldFormatter;
_fieldFilterService = fieldFilterService ?? TextWranglerConfig.DefaultFieldFilterService;
_recordWriter = recordWriter ?? TextWranglerConfig.DefaultRecordWriter;
_logger = LogManager.GetLogger(GetType().Name);
}
private void DetermineOutputMethods(IRecordWriter recordWriter, out Action<IStreamReader, Record> writeMatchedRecordMethod, out Action<IStreamReader, Record> writeUnmatchedRecordMethod)
{
if (recordWriter.DoWriteMatchedRecords && recordWriter.DoWriteUnmatchedRecords)
{
writeMatchedRecordMethod = recordWriter.WriteMatchedRecord;
writeUnmatchedRecordMethod = recordWriter.WriteUnmatchedRecord;
}
else if (recordWriter.DoWriteMatchedRecords)
{
writeMatchedRecordMethod = recordWriter.WriteMatchedRecord;
writeUnmatchedRecordMethod = this.WriteNothing;
}
else if (recordWriter.DoWriteUnmatchedRecords)
{
writeMatchedRecordMethod = this.WriteNothing;
writeUnmatchedRecordMethod = recordWriter.WriteUnmatchedRecord;
}
else
{
writeMatchedRecordMethod = writeUnmatchedRecordMethod = this.WriteNothing;
}
}
public void Write(IRecordWriter writer, string value)
{
value = String.Format("{0}={1}", _name, value); // is this a behavior? It could tell us how to write the value and how to match & parse on read
writer.Write(value, _position);
}
public void Write(object record, IRecordWriter writer)
{
string value = Property.Get(record);
Field.Write(writer, value);
}
public ReportingWriter(IRecordWriter baseWriter, IStatisticsCollector statistics)
{
this.baseWriter = baseWriter;
this.statistics = statistics;
}
public void WriteFileHeader(SpssOptions options, IEnumerable <Variable> variables)
{
_options = options;
_compress = options.Compressed;
_bias = options.Bias;
_variables = variables.ToArray();
// SPSS file header
var headerRecords = new List <IRecord>
{
new HeaderRecord(options)
};
// Process all variable info
var variableLongNames = new Dictionary <string, string>();
var veryLongStrings = new Dictionary <string, int>();
var displayInfoList = new List <VariableDisplayInfo>(_variables.Length);
SetVariables(headerRecords, variableLongNames, veryLongStrings, displayInfoList);
// Integer & encoding info
var intInfoRecord = new MachineIntegerInfoRecord(_options.HeaderEncoding);
headerRecords.Add(intInfoRecord);
// Integer & encoding info
var fltInfoRecord = new MachineFloatingPointInfoRecord();
headerRecords.Add(fltInfoRecord);
// Variable Display info, beware that the number of variables here must match the count of named variables
// (exclude the string continuation, include VLS segments)
var varDisplRecord = new VariableDisplayParameterRecord(displayInfoList.Count);
for (int index = 0; index < displayInfoList.Count; index++)
{
varDisplRecord[index] = displayInfoList[index];
}
headerRecords.Add(varDisplRecord);
// Variable Long names (as info record)
if (variableLongNames.Any())
{
var longNameRecord = new LongVariableNamesRecord(variableLongNames, _options.HeaderEncoding);
headerRecords.Add(longNameRecord);
}
if (veryLongStrings.Any())
{
var veryLongStringsRecord = new VeryLongStringRecord(veryLongStrings, _options.HeaderEncoding);
headerRecords.Add(veryLongStringsRecord);
}
// Char encoding info record (for data)
var charEncodingRecord = new CharacterEncodingRecord(_options.DataEncoding);
headerRecords.Add(charEncodingRecord);
// End of the info records
headerRecords.Add(new DictionaryTerminationRecord());
// Write all of header, variable and info records
foreach (var headerRecord in headerRecords)
{
headerRecord.WriteRecord(_writer);
}
if (_compress)
{
_recordWriter = new CompressedRecordWriter(_writer, _bias, double.MinValue);
}
else
{
throw new NotImplementedException("Uncompressed SPSS data writing is not yet implemented. Please set compressed to true");
}
_stringWriter = new StringWriter(_options.DataEncoding, _recordWriter);
}
public RecordWriterMapperCollector( IRecordWriter<string, string> output )
{
_output = output;
}
public void TestSetup()
{
baseWriter = Substitute.For <IRecordWriter>();
statistics = Substitute.For <IStatisticsCollector>();
reportingWriter = new ReportingWriter(baseWriter, statistics);
}
public void Write(IRecordWriter writer, string value)
{
value = String.Format("{0}={1}", _name, value); // is this a behavior? It could tell us how to write the value and how to match & parse on read
writer.Write(value, _position);
}
public void Write(object record, IRecordWriter writer)
{
string value = Property.Get(record);
Field.Write(writer, value);
}
/// <summary>
/// Opens the writer for this output file.
/// </summary>
/// <param name="recordAccessorFactory">
/// The record accessor factory to use to open the record writer.
/// </param>
public void OpenWriter(IRecordAccessorFactory <byte[]> recordAccessorFactory)
{
_writer = recordAccessorFactory.CreateWriter(Output.Create());
}
public RecordWriterReducerCollector(IRecordWriter<string, string> writer)
{
_writer = writer;
}
