예제 #1
0
        /// <summary>
        /// SortAscending is a parameter of the constructor because we have to know immediately
        /// what our properties are so we can pass them on to whoever asks of us.
        /// </summary>
        public SortableRecordMemory(bool sortAscending, int maxMemorySize)
        {
            _recordMemoryStream = new MemoryStream();
            _recordMemoryWriter = new VariableLengthBinaryWriter(_recordMemoryStream);
            _recordBytes = new byte[_recordBytesSize];
            _maxRecordBytes = maxMemorySize;
            _internalSortAscending = sortAscending;

            // means our sorting properties are not propagated from inputs to outputs.
            _passThruInputSorting = false;

            // the sorting properties, like (segmenting, bucketing, reduced, ...) have to reflect
            // immediately what the component is going to do even before it has done it.  So even
            // before the records are sorted we set these properties because the world wants to
            // know what's true of our output.

            _sorting.IsSorted = true; // tells the world we sort.
            _sorting.IsSortedAscending = sortAscending;

            _startTick = DateTime.UtcNow.Ticks;
        }
예제 #2
0
        internal bool WriteRecordFileMaxSize(InternalRecordSource input, long maxFileSize)
        {
            // initialize
            if (_outputStream == null) {
                OpenOutputStream();
                _outputWriter = new VariableLengthBinaryWriter(_outputStream);
                input.WriteProperties(_outputStream);
            }

            bool done = false;
            while (!done && input.MoveNext()) {
                DataRecord record = input.CurrentRecord;
                // output key
                _outputWriter.WriteVariableLength((uint)record.KeyBytes.Length);
                _outputWriter.Write(record.KeyBytes);

                // output data
                if (record.Data != null) {
                    _outputWriter.WriteVariableLength((uint)record.Data.Length);
                    _outputWriter.Write(record.Data);
                }
                else {
                    _outputWriter.WriteVariableLength((uint)0);
                }

                if (maxFileSize != -1 && _outputStream.Position >= maxFileSize) {
                    done = true;
                }
            }

            _outputStream.Close();
            _outputStream = null;

            // we return notDone to our caller, i.e. we return true if there is more
            // to be written, false if we wrote it all.
            return done;
        }
예제 #3
0
        /// <summary>
        /// 
        /// </summary>
        /// <param name="record"></param>
        public void WriteRecord(DataRecord record)
        {
            if (_outputStream == null) {
                OpenOutputStream();
                _outputWriter = new VariableLengthBinaryWriter(_outputStream);
                EmptyInternalSource e = new EmptyInternalSource();

                e.CurrentRecord = record; // updates the record info
                e.WriteProperties(_outputStream);
            }

            _outputWriter.WriteVariableLength((uint)record.KeyBytes.Length);
            _outputWriter.Write(record.KeyBytes);

            // output data
            if (record.Data != null) {
                _outputWriter.WriteVariableLength((uint)record.Data.Length);
                _outputWriter.Write(record.Data);
            }
            else {
                _outputWriter.WriteVariableLength((uint)0);
            }

            // slow do elsewhere _totalRecordBytesEstimate += rawRecord.Length;
            _totalRecordsEstimate++;
        }
예제 #4
0
        internal void Write(InternalRecordSource input)
        {
            while (input.MoveNext()) {
                DataRecord record = input.CurrentRecord;

                // we need to do a moveNext on our input in order for RecordInstance
                // to be valid.
                if (_outputStream == null) {
                    OpenOutputStream();
                    input.WriteProperties(_outputStream);
                    _outputWriter = new VariableLengthBinaryWriter(_outputStream);
                }

                // output key
                _outputWriter.WriteVariableLength((uint)record.KeyBytes.Length);
                _outputWriter.Write(record.KeyBytes);

                // output data
                if (record.Data != null) {
                    _outputWriter.WriteVariableLength((uint)record.Data.Length);
                    _outputWriter.Write(record.Data);
                }
                else {
                    _outputWriter.WriteVariableLength((uint)0);
                }

                // slow. do elsewhere _totalRecordBytesEstimate = _outputStream.Position;
                _totalRecordsEstimate++;
            }

            _totalRecordBytesEstimate = _outputStream.Position;
            InternalRecordSource.WriteEstimates(_outputStream, _totalRecordBytesEstimate, _totalRecordsEstimate);
            _outputStream.Close();
            _outputStream = null;
        }
예제 #5
0
        private void _Construct(InternalRecordSource input, string outputDir, string fileSetName)
        {
            if (input != null && !input.Sorting.IsSorted) {
                throw new Exception("can't output non-sorted input as TMSNStore");
            }

            _input = input;
            _outputDir = outputDir;
            _fileSetName = fileSetName;

            // if directory doesn't exist make it
            if (!Directory.Exists(outputDir)) {
                Directory.CreateDirectory(outputDir);
            }

            string keyFile = Path.Combine(outputDir, _fileSetName);
            string indexFile = Path.Combine(outputDir, _fileSetName + "-index");

            // create fileSetName file
            _keyFileStream = new FileStream(keyFile, FileMode.Create, FileAccess.Write, FileShare.None, 1024 * 1024 * 16);
            _keyFileWriter = new VariableLengthBinaryWriter(_keyFileStream);

            // create index file
            _indexFileStream = new FileStream(indexFile, FileMode.Create, FileAccess.Write, FileShare.None, 1024 * 1024 * 16);
            _indexFileWriter = new BinaryWriter(_indexFileStream);
        }