示例#1
0
 /// <summary>
 /// 
 /// </summary>
 /// <param name="filepath"></param>
 public RecordFileReader(string filepath)
 {
     _internalRecordFileReader = new InternalRecordFileReader(filepath);
 }
示例#2
0
        private void _Finish()
        {
            if (_doThreading) {
                _SortAndWriteThreaded();
                // wait on all previous threads, sorts and writes
                foreach (Thread t in _threads) {
                    t.Join();
                }
                _threads.Clear();
            }

            else {
                // don't write the last one to disk
                _recordsToSort.Sort();
                _sortedRecords = _recordsToSort;
            }

            int numToMerge = _numTempFiles;

            if (_sortedRecords != null && _sortedRecords.NumRecords != 0) {
                numToMerge++;
            }

            if (numToMerge > 1) {
                _merger = new SortedRecordMerger();

                if (_sortedRecords != null && _sortedRecords.NumRecords != 0) {
                    _merger.AddInput(_sortedRecords);
                }

                for (int i = 0; i < _numTempFiles; i++) {
                    InternalRecordSource source = new InternalRecordFileReader(GetTempFilename(i));
                    _merger.AddInput(source);
                }

                _output = _merger;
            }

            else {
                _output = _sortedRecords;
            }

            // set up reduction filter for reduction across merged sources
            if (_internalReductionEnabled) {
                ReduceFilter reducer = new ReduceFilter();
                RecordFilterDriver filterDriver = new RecordFilterDriver(reducer);
                filterDriver.AddInput(_output);
                _output = filterDriver;
            }

            // this is kind of a hack till i figure out how these should be set
            TotalRecordBytesEstimate = _output.TotalRecordBytesEstimate;
            TotalRecordsEstimate = _output.TotalRecordsEstimate;
        }
示例#3
0
        /// <summary>
        /// Creates a record source out of 3 possible input types:
        /// 1) A searchable TStore
        /// 2) A streamable RecordsFile
        /// 3) A streamable FlatFile
        /// </summary>
        /// <param name="inputType">The type of input</param>
        /// <param name="pathInfo">The path to the input</param>
        /// <returns>A RecordSource for further processing</returns>
        public RecordSource Input(TStoreInputType inputType, string pathInfo)
        {
            InternalRecordSource source = null;

            switch (inputType) {
                case TStoreInputType.RecordFile:
                    source = new InternalRecordFileReader(pathInfo);
                    break;

                case TStoreInputType.TStore:
                    TMSNStoreReader reader = new TMSNStoreReader(pathInfo, true);
                    source = reader.RecordSource;
                    break;

                case TStoreInputType.FlatFile:
                    source = new FlatFileMapper(pathInfo);
                    break;

                case TStoreInputType.Directory:
                    source = new DirectoryMapper(pathInfo);
                    break;
            }

            source.ProcessTreeComment = "[" + pathInfo + "]";

            RecordSource source2Bmapped = new RecordSource(this);
            source2Bmapped.InternalSource = _AddLogging(source);

            return source2Bmapped;
        }