/// <summary> /// Launches the engine. /// </summary> /// <param name="inputFilePaths">List of full input files to be processed.</param> /// <param name="recordReader">Instance that reads the record from the input files.</param> /// <param name="expression">Instance of the expression used to matched against the record.</param> public override void LaunchEngine(String[] inputFilePaths, IRecordReader recordReader, IRecordMatchExpression expression) { this.recordWriter = CreateRecordWriter(); var engine = new Engine(); engine.FileOpened += this.FileOpenedHandler; engine.FileRead += this.FileReadHandler; engine.CheckForCancellation = this.CheckForCancellation; this.worker = new BackgroundWorker(); this.worker.WorkerSupportsCancellation = true; this.worker.DoWork += (sender, e) => { engine.Execute( inputFilePaths, this.uiLogManager, new FileReaderFactory(), recordReader, expression, this.recordWriter, this.statisticsManager, this.statisticsManager); if (engine.CheckForCancellation()) { e.Cancel = true; } }; this.worker.RunWorkerCompleted += BackgroundWorkCompleted; this.worker.RunWorkerAsync(); }
public void TestSetup() { factory = Substitute.For <IStreamStateFactory>(); factory.Create(Arg.Any <string>()).Returns(_ => state); state = Substitute.For <IStreamState>(); state.BufferPool.Returns(_ => pool); state.Statistics.Returns(_ => stats); state.RecordWriter.Returns(_ => writer); state.SendSignal.Returns(new AsyncManualResetEvent(false)); buffer = Substitute.For <IBuffer>(); writer = Substitute.For <IRecordWriter>(); stats = Substitute.For <IStatisticsCollector>(); daemon = Substitute.For <IDaemon>(); pool = Substitute.For <IBufferPool>(); pool.TryAcquire(out _) .Returns( info => { info[0] = buffer; return(true); }); log = new SynchronousConsoleLog(); sink = new HerculesSink(factory, daemon, log); }
/// <summary> /// Default constructor. /// </summary> /// <param name="writer">the underlying writer</param> /// <param name="sum">the sum to use for similar items (or <code>null</code>)</param> /// <param name="comparer">the comparer to use for records</param> /// <param name="outputFormatter">the formatter to use when writing the records (or <code>null</code>)</param> public SumWriter(IRecordWriter <T> writer, ISum <T> sum, IComparer <T> comparer, IFormatter <T> outputFormatter) { _writer = writer; _sum = sum; _comparer = comparer; _outputFormatter = outputFormatter; }
public void TestSetup() { baseWriter = Substitute.For <IRecordWriter>(); statistics = Substitute.For <IStatisticsCollector>(); signal = new AsyncManualResetEvent(false); signalingWriter = new SignalingWriter(baseWriter, statistics, signal, 100, 0.25, 0.70); }
public RecordFileWriter(FileInfo destination, Func <ISimpleLogger, IRecordWriter> factory) { Destination = destination; var stream = new SimpleLoggerStreamWriter(File.Create(Destination.FullName)); _stream = stream; _writer = factory(stream); }
public void Write(IRecordWriter writer, string value) { if (_maximumLength != null) value = _maximumLength.Write(value); if (_minimumLength != null) value = _minimumLength.Write(value); writer.Write(value, _position, 0); }
public SyncData(IRecordReader <T> recordReader, IRecordWriter <T> recordWriter) : base() { RecordWriter = recordWriter; RecordReader = recordReader; RecordWriter.ErrorRaised += OnErrorRaised; RecordReader.ErrorRaised += OnErrorRaised; RecordReader.RecordPacketCompleted += OnReaderPacketComplete; RecordReader.ReadCompleted += OnReadCompleteAsync; }
public TextWrangler(string recordConfigName, IRecordReader recordReader, IRecordBuilder recordBuilder = null, IFieldFormatter fieldFormatter = null, IFieldFilterService fieldFilterService = null, IRecordWriter recordWriter = null) : this(RecordConfigurationBuilder.Build(recordConfigName), recordReader, recordBuilder, fieldFormatter, fieldFilterService, recordWriter) { }
public void Write(IRecordWriter writer, string value) { if (_maximumLength != null) value = _maximumLength.Write(value); if (_minimumLength != null) value = _minimumLength.Write(value); writer.Write(value, 0, _index); }
public StringWriter(Encoding encoding, IRecordWriter recordWriter) { _recordWriter = recordWriter; _encoder = encoding.GetEncoder(); _stringBytesBuffer = new byte[encoding.GetMaxByteCount(CharsBufferSize)]; _stringSegmentBuffer = new byte[256]; _stringSegmentBuffer[255] = 0x20; }
public void Write(IRecordWriter writer, string value) { if (_maximumLength != null) { value = _maximumLength.Write(value); } if (_minimumLength != null) { value = _minimumLength.Write(value); } writer.Write(value, 0, _index); }
public static void Copy(IRecordReader<string, string> reader, IRecordWriter<string, string> writer) { var keys = new List<string>(reader.GetKeys()); for (int i = 0; i < keys.Count; i++) { var key = keys[i]; if (i % 100 == 0 && i > 0) Console.WriteLine("Copy key " + i + "/" + keys.Count); var values = reader.GetValues(key); foreach (var value in values) writer.Write(key, value); } }
public void Write(IRecordWriter writer, string value) { if (_maximumLength != null) { value = _maximumLength.Write(value); } if (_minimumLength != null) { value = _minimumLength.Write(value); } writer.Write(value, _position, 0); }
public SignalingWriter( IRecordWriter baseWriter, IStatisticsCollector statistics, AsyncManualResetEvent signal, long sizeLimit, double transitionSignalFraction, double constantSignalFraction) { this.baseWriter = baseWriter; this.statistics = statistics; this.signal = signal; transitionSignalThreshold = (long)(sizeLimit * transitionSignalFraction); constantSignalThreshold = (long)(sizeLimit * constantSignalFraction); }
public StreamState( [NotNull] string name, [NotNull] IBufferPool bufferPool, [NotNull] IMemoryAnalyzer memoryAnalyzer, [NotNull] IRecordWriter recordWriter, [NotNull] IStatisticsCollector statistics, [NotNull] AsyncManualResetEvent sendSignal) { Name = name ?? throw new ArgumentNullException(nameof(name)); BufferPool = bufferPool ?? throw new ArgumentNullException(nameof(bufferPool)); Statistics = statistics ?? throw new ArgumentNullException(nameof(statistics)); RecordWriter = recordWriter ?? throw new ArgumentNullException(nameof(recordWriter)); SendSignal = sendSignal ?? throw new ArgumentNullException(nameof(sendSignal)); MemoryAnalyzer = memoryAnalyzer ?? throw new ArgumentNullException(nameof(memoryAnalyzer)); }
/// <summary> /// Merges the temporary files to the final output file /// </summary> /// <param name="buffers">the buffers to read the temporary files</param> /// <param name="writer">the writer for the output file</param> private static void Merge(IEnumerable <RecordReaderBuffer <T> > buffers, IRecordWriter <T> writer) { // Buffers are stored in a priority queue to have the buffers with the // lowest record (with respect to Comparer) as the first buffer var queue = new PriorityQueue <RecordReaderBuffer <T> >(buffers); while (queue.Count > 0) { var buffer = queue.Poll(); var record = buffer.Read(); writer.Write(record); // If the buffer has still records, we put it back in the queue // so that is is correctly sorted if (buffer.HasNext()) { queue.Add(buffer); } } }
public void Execute( String[] filePaths, ILogManager logManager, IStreamReaderFactory streamReaderFactory, IRecordReader recordReader, IRecordMatchExpression expression, IRecordWriter recordWriter, IStatisticsCollector statisticsCollector, IStatisticsReporter statisticsReporter) { logManager.VerifyThatObjectIsNotNull("Parameter 'logManager' is null."); try { this.logManager = logManager; statisticsCollector.VerifyThatObjectIsNotNull("Parameter 'statisticsCollector' is null."); this.statisticsCollector = statisticsCollector; statisticsReporter.VerifyThatObjectIsNotNull("Parameter 'statisticsReporter' is null."); this.statisticsReporter = statisticsReporter; this.logManager.WriteMessagesToLogs("Run Started..."); Action<IStreamReader, Record> writeMatchedRecordMethod, writeUnmatchedRecordMethod; this.DetermineOutputMethods(recordWriter, out writeMatchedRecordMethod, out writeUnmatchedRecordMethod); this.Process(filePaths, streamReaderFactory, recordReader, expression, writeMatchedRecordMethod, writeUnmatchedRecordMethod); recordWriter.Close(); this.statisticsReporter.WriteToLog(this.logManager); this.logManager.WriteMessagesToLogs("Run Finished."); } catch (Exception exception) { logManager.WriteMessageToApplicationLog("EXCEPTION: " + exception.Message); logManager.WriteMessageToApplicationLog("STACK: " + exception.StackTrace); throw exception; } }
public TextWrangler(RecordConfiguration recordConfiguration, IRecordReader recordReader, IRecordBuilder recordBuilder = null, IFieldFormatter fieldFormatter = null, IFieldFilterService fieldFilterService = null, IRecordWriter recordWriter = null) { _recordConfiguration = recordConfiguration ?? throw new ArgumentNullException(nameof(recordReader)); _recordReader = recordReader ?? throw new ArgumentNullException(nameof(recordReader)); // Validate the configuration RecordConfigurationValidator.Instance.Validate(_recordConfiguration, _fieldFilterService); _recordBuilder = recordBuilder ?? TextWranglerConfig.DefaultRecordBuilder; _fieldFormatter = fieldFormatter ?? TextWranglerConfig.DefaultFieldFormatter; _fieldFilterService = fieldFilterService ?? TextWranglerConfig.DefaultFieldFilterService; _recordWriter = recordWriter ?? TextWranglerConfig.DefaultRecordWriter; _logger = LogManager.GetLogger(GetType().Name); }
private void DetermineOutputMethods(IRecordWriter recordWriter, out Action<IStreamReader, Record> writeMatchedRecordMethod, out Action<IStreamReader, Record> writeUnmatchedRecordMethod) { if (recordWriter.DoWriteMatchedRecords && recordWriter.DoWriteUnmatchedRecords) { writeMatchedRecordMethod = recordWriter.WriteMatchedRecord; writeUnmatchedRecordMethod = recordWriter.WriteUnmatchedRecord; } else if (recordWriter.DoWriteMatchedRecords) { writeMatchedRecordMethod = recordWriter.WriteMatchedRecord; writeUnmatchedRecordMethod = this.WriteNothing; } else if (recordWriter.DoWriteUnmatchedRecords) { writeMatchedRecordMethod = this.WriteNothing; writeUnmatchedRecordMethod = recordWriter.WriteUnmatchedRecord; } else { writeMatchedRecordMethod = writeUnmatchedRecordMethod = this.WriteNothing; } }
public void Write(IRecordWriter writer, string value) { value = String.Format("{0}={1}", _name, value); // is this a behavior? It could tell us how to write the value and how to match & parse on read writer.Write(value, _position); }
public void Write(object record, IRecordWriter writer) { string value = Property.Get(record); Field.Write(writer, value); }
public ReportingWriter(IRecordWriter baseWriter, IStatisticsCollector statistics) { this.baseWriter = baseWriter; this.statistics = statistics; }
public void WriteFileHeader(SpssOptions options, IEnumerable <Variable> variables) { _options = options; _compress = options.Compressed; _bias = options.Bias; _variables = variables.ToArray(); // SPSS file header var headerRecords = new List <IRecord> { new HeaderRecord(options) }; // Process all variable info var variableLongNames = new Dictionary <string, string>(); var veryLongStrings = new Dictionary <string, int>(); var displayInfoList = new List <VariableDisplayInfo>(_variables.Length); SetVariables(headerRecords, variableLongNames, veryLongStrings, displayInfoList); // Integer & encoding info var intInfoRecord = new MachineIntegerInfoRecord(_options.HeaderEncoding); headerRecords.Add(intInfoRecord); // Integer & encoding info var fltInfoRecord = new MachineFloatingPointInfoRecord(); headerRecords.Add(fltInfoRecord); // Variable Display info, beware that the number of variables here must match the count of named variables // (exclude the string continuation, include VLS segments) var varDisplRecord = new VariableDisplayParameterRecord(displayInfoList.Count); for (int index = 0; index < displayInfoList.Count; index++) { varDisplRecord[index] = displayInfoList[index]; } headerRecords.Add(varDisplRecord); // Variable Long names (as info record) if (variableLongNames.Any()) { var longNameRecord = new LongVariableNamesRecord(variableLongNames, _options.HeaderEncoding); headerRecords.Add(longNameRecord); } if (veryLongStrings.Any()) { var veryLongStringsRecord = new VeryLongStringRecord(veryLongStrings, _options.HeaderEncoding); headerRecords.Add(veryLongStringsRecord); } // Char encoding info record (for data) var charEncodingRecord = new CharacterEncodingRecord(_options.DataEncoding); headerRecords.Add(charEncodingRecord); // End of the info records headerRecords.Add(new DictionaryTerminationRecord()); // Write all of header, variable and info records foreach (var headerRecord in headerRecords) { headerRecord.WriteRecord(_writer); } if (_compress) { _recordWriter = new CompressedRecordWriter(_writer, _bias, double.MinValue); } else { throw new NotImplementedException("Uncompressed SPSS data writing is not yet implemented. Please set compressed to true"); } _stringWriter = new StringWriter(_options.DataEncoding, _recordWriter); }
public RecordWriterMapperCollector( IRecordWriter<string, string> output ) { _output = output; }
public void TestSetup() { baseWriter = Substitute.For <IRecordWriter>(); statistics = Substitute.For <IStatisticsCollector>(); reportingWriter = new ReportingWriter(baseWriter, statistics); }
/// <summary> /// Opens the writer for this output file. /// </summary> /// <param name="recordAccessorFactory"> /// The record accessor factory to use to open the record writer. /// </param> public void OpenWriter(IRecordAccessorFactory <byte[]> recordAccessorFactory) { _writer = recordAccessorFactory.CreateWriter(Output.Create()); }
public RecordWriterReducerCollector(IRecordWriter<string, string> writer) { _writer = writer; }