// write this chunk to disk. This is where reduction happens if it happens. private void _WriteSortedRecords() { if (!Directory.Exists(_tempDir)) { Directory.CreateDirectory(_tempDir); } InternalRecordSource output = _recordsToWrite; // set up reduction filter. The whole reason we have a sorterReducer // and not just a sorter is so that we can reduce before we write our // temp files to disk. if (_internalReductionEnabled) { ReduceFilter rf = new ReduceFilter(); RecordFilterDriver filterDriver = new RecordFilterDriver(rf); filterDriver.AddInput(output); output = filterDriver; } RecordFileWriter recordFileWriter = new RecordFileWriter(GetTempFilename(_numTempFiles)); _numTempFiles++; try { recordFileWriter.Write(output); } finally { _recordsToWrite.Close(); } }
public void Write() { TStorageType outputType = TStorageType.FlatFile; if (_outputUri == null) { throw new Exception("must use constructor with OutputUri parameter when using this method"); } outputType = _outputUri.StorageType; Console.Error.Write("[" + outputType + ":" + _outputUri.FilePath + "]\tBegin Write\t"); Console.Error.WriteLine(DateTime.Now); Console.Error.Flush(); switch (outputType) { case TStorageType.TStore: TMSNStoreWriter twriter = new TMSNStoreWriter(_output, _outputUri.FilePath); twriter.KeyGroupSize = TStoreGroupSize; twriter.Write(); break; case TStorageType.FlatFile: _WriteFlatFile(_outputUri.FilePath); break; case TStorageType.RecordFile: RecordFileWriter writer = new RecordFileWriter(_outputUri.FilePath); writer.Write(_output); break; } _output.Close(); Console.Error.Write("[" + outputType + ":" + _outputUri.FilePath + "]\tEnd Write\t"); Console.Error.WriteLine(DateTime.Now); Console.Error.Flush(); }