Example #1
0
        public void Write()
        {
            TStorageType outputType = TStorageType.FlatFile;

            if (_outputUri == null) {
                throw new Exception("must use constructor with OutputUri parameter when using this method");
            }

            outputType = _outputUri.StorageType;

            Console.Error.Write("[" + outputType + ":" + _outputUri.FilePath + "]\tBegin Write\t");
            Console.Error.WriteLine(DateTime.Now);
            Console.Error.Flush();

            switch (outputType) {
                case TStorageType.TStore:
                    TMSNStoreWriter twriter = new TMSNStoreWriter(_output, _outputUri.FilePath);
                    twriter.KeyGroupSize = TStoreGroupSize;
                    twriter.Write();
                    break;

                case TStorageType.FlatFile:
                    _WriteFlatFile(_outputUri.FilePath);
                    break;

                case TStorageType.RecordFile:
                    RecordFileWriter writer = new RecordFileWriter(_outputUri.FilePath);
                    writer.Write(_output);
                    break;
            }

            _output.Close();

            Console.Error.Write("[" + outputType + ":" + _outputUri.FilePath + "]\tEnd Write\t");
            Console.Error.WriteLine(DateTime.Now);
            Console.Error.Flush();
        }
Example #2
0
        /// <summary>
        /// Finalizes the writing of a TStore to disk.
        /// </summary>
        public void Write()
        {
            // we use the same disk structure to store the cache
            TMSNStoreWriter keyCacheWriter = new TMSNStoreWriter(_outputDir, true);
            uint cacheKeyGroupSize = 32; // we fix this one
            keyCacheWriter.KeyGroupSize = cacheKeyGroupSize;

            // if we are in the push mode use the sorterReducer that the records were pushed into.
            if (_input == null) {
                _input = _sorterReducer;
            }
            // Note, _input can be null if the user never pushed anything onto this writer.
            if (_input != null)
            {
                while (_input.MoveNext())
                {
                    // initialize file lengths.  We wait till after the first MoveNext so props are set
                    if (_numWritten == 0)
                    {
                        TotalRecordBytesEstimate = _input.TotalRecordBytesEstimate;
                        TotalRecordsEstimate = _input.TotalRecordsEstimate;
                        Initialize();

                        keyCacheWriter.TotalRecordsEstimate = TotalRecordsEstimate / (long)KeyGroupSize;
                        keyCacheWriter.TotalRecordBytesEstimate = TotalRecordBytesEstimate / (long)KeyGroupSize;
                        keyCacheWriter.Initialize();
                    }

                    DataRecord record = _input.CurrentRecord;

                    // write first key of group to cache
                    if (_numWritten % KeyGroupSize == 0)
                    {
                        // write the key of the first group member to the cache.
                        keyCacheWriter.AddRecord(record.KeyBytes, record.Data, false);
                    }

                    // write to main store
                    AddRecord(record.KeyBytes, record.Data, true);
                }

                if (_numWritten == 0) return;

                // if needed, pad the last group written with null entries so that
                // we don't have to special-case reading the last group.
                long unfullGroupNumMembers = _numWritten % KeyGroupSize;
                if (unfullGroupNumMembers > 0)
                {
                    uint numPadEntries = (uint)(KeyGroupSize - unfullGroupNumMembers);
                    for (int i = 0; i < numPadEntries; i++)
                    {
                        _keyFileWriter.Write((byte)0); // overlap
                        _keyFileWriter.Write((byte)0); // neu
                        _keyFileWriter.Write((byte)0); // dataLen
                    }
                }

                // take the input source and write it's properties to disk
                string recordInfoFile = Path.Combine(_outputDir, "record-info");
                using (Stream sw = new FileStream(recordInfoFile, FileMode.Create))
                {
                    _input.TotalRecordsEstimate = _numWritten;
                    _input.TotalRecordBytesEstimate = _keyFileStream.Position;
                    _input.WriteProperties(sw);
                }
            }

            Finish();
            keyCacheWriter.Finish();
            if (_sorterReducer != null) _sorterReducer.Close();
        }