public override bool MoveNext()
{
if (_source == null) {
_source = (InternalRecordSource)_inputList[_currentSourceNo++];
}
_notDone = _source.MoveNext();
if (!_notDone && _currentSourceNo < 2) {
_source = (InternalRecordSource)_inputList[_currentSourceNo++];
_notDone = _source.MoveNext();
}
CurrentRecord = _source.CurrentRecord;
return _notDone;
}
public override bool MoveNext()
{
bool notDone = true;
if (_source == null) {
_source = (InternalRecordSource)_inputList[0];
notDone = _source.MoveNext();
CurrentRecord = _source.CurrentRecord;
TotalRecordBytesEstimate = _source.TotalRecordBytesEstimate;
TotalRecordsEstimate = _source.TotalRecordsEstimate;
if (_displayString == null) {
if (_source is InternalUserSource) {
_displayString = "[UserSource]";
}
else {
_displayString = "[" + _source.StorageType + ":" + _source.CurrentSourceName + "]";
}
}
Console.Error.Write(_displayString + "\tBegin Read\t" + TotalRecordsEstimate + " records");
if (_source is FlatFileMapper || _source is DirectoryMapper || _source is InternalUserSource) {
Console.Error.Write(" (estimate)");
_doEstimate = true;
}
Console.Error.WriteLine();
_logInterval = TotalRecordsEstimate / _numLogs;
if (_logInterval == 0) _logInterval = 1;
_nextLogCount = _logInterval;
_beginDateTime = DateTime.UtcNow;
}
else notDone = _source.MoveNext();
CurrentRecord = _source.CurrentRecord;
if (_recordCount == _nextLogCount) {
Console.Error.Write(_displayString + "\tBegin Read\t");
double percent = 100.0 * _recordCount / TotalRecordsEstimate;
Console.Error.Write("{0:0.00}% done\t", percent);
TimeSpan span = DateTime.UtcNow - _beginDateTime;
_beginDateTime = DateTime.UtcNow;
double recsPerSecond = _logInterval / (span.Seconds + 1); // div by 0
Console.Error.WriteLine(recsPerSecond + " recs/sec");
if (_doEstimate) {
double aveBytesPerRecord = (double)_lineBytes / (double)(_recordCount + 1); // div by 0
TotalRecordsEstimate = (long)(((double)TotalRecordBytesEstimate) / aveBytesPerRecord);
long recordsLeft = TotalRecordsEstimate - _recordCount;
_logInterval = recordsLeft / (_numLogs - 1);
// only reestimate one time
_doEstimate = false;
}
_nextLogCount += _logInterval;
}
_recordCount++;
if (!notDone) {
Console.Error.WriteLine(_displayString + "\tEnd Read\t" + DateTime.Now);
}
if (_doEstimate && notDone) {
_lineBytes += _source.CurrentRecord.Key.Length;
}
return notDone;
}
private void _AddInput(InternalRecordSource input)
{
if (!input.Sorting.IsSorted) {
throw new Exception("can't merge non-sortable record source");
}
bool notEmpty = input.MoveNext();
if (!notEmpty) return;
int current = _numSources++;
_heapArray[current] = input;
int parent = _Parent(current);
// while the current is less than the parent
while ((current != 0) &&
_MergeCompare(_heapArray[current].CurrentRecord.KeyBytes,
_heapArray[parent].CurrentRecord.KeyBytes)) {
_Swap(current, parent);
current = _Parent(current);
parent = _Parent(current);
}
}
internal bool WriteRecordFileMaxSize(InternalRecordSource input, long maxFileSize)
{
// initialize
if (_outputStream == null) {
OpenOutputStream();
_outputWriter = new VariableLengthBinaryWriter(_outputStream);
input.WriteProperties(_outputStream);
}
bool done = false;
while (!done && input.MoveNext()) {
DataRecord record = input.CurrentRecord;
// output key
_outputWriter.WriteVariableLength((uint)record.KeyBytes.Length);
_outputWriter.Write(record.KeyBytes);
// output data
if (record.Data != null) {
_outputWriter.WriteVariableLength((uint)record.Data.Length);
_outputWriter.Write(record.Data);
}
else {
_outputWriter.WriteVariableLength((uint)0);
}
if (maxFileSize != -1 && _outputStream.Position >= maxFileSize) {
done = true;
}
}
_outputStream.Close();
_outputStream = null;
// we return notDone to our caller, i.e. we return true if there is more
// to be written, false if we wrote it all.
return done;
}
internal void Write(InternalRecordSource input)
{
while (input.MoveNext()) {
DataRecord record = input.CurrentRecord;
// we need to do a moveNext on our input in order for RecordInstance
// to be valid.
if (_outputStream == null) {
OpenOutputStream();
input.WriteProperties(_outputStream);
_outputWriter = new VariableLengthBinaryWriter(_outputStream);
}
// output key
_outputWriter.WriteVariableLength((uint)record.KeyBytes.Length);
_outputWriter.Write(record.KeyBytes);
// output data
if (record.Data != null) {
_outputWriter.WriteVariableLength((uint)record.Data.Length);
_outputWriter.Write(record.Data);
}
else {
_outputWriter.WriteVariableLength((uint)0);
}
// slow. do elsewhere _totalRecordBytesEstimate = _outputStream.Position;
_totalRecordsEstimate++;
}
_totalRecordBytesEstimate = _outputStream.Position;
InternalRecordSource.WriteEstimates(_outputStream, _totalRecordBytesEstimate, _totalRecordsEstimate);
_outputStream.Close();
_outputStream = null;
}
/// <summary>
/// Advances iteration.
/// </summary>
/// <returns>False if at end of iteration, true otherwise.</returns>
public override bool MoveNext()
{
if (_leftSource == null) {
_leftSource = (InternalRecordSource)_inputList[0];
_rightSource = (InternalRecordSource)_inputList[1];
// advance the right side from the start so that
// a record is there waiting to be compared to
_rightNotDone = _rightSource.MoveNext();
if (_rightNotDone) _currentRightRecord = _rightSource.CurrentRecord;
}
_leftNotDone = _leftSource.MoveNext();
if (!_leftNotDone) return false;
_currentLeftRecord = _leftSource.CurrentRecord;
// Don't need a match to create a joiner
if (_joiner == null) {
_joiner = new RecordJoiner(_leftSource, _rightSource, _tableColumnSeparator);
}
// advance the right side
int diff = -1;
bool firstTime = true;
while (diff < 0) {
// the first time we test we check against the currentRightSource
// (i.e. don't go in this block) since we allow dups on the left side.
if (!firstTime && _rightNotDone) {
_rightSource.MoveNextHint = _currentLeftRecord.Key;
_rightNotDone = _rightSource.MoveNext();
if (!_rightNotDone) _currentRightRecord = null;
else _currentRightRecord = _rightSource.CurrentRecord;
}
if (_currentRightRecord != null) {
diff = TMSNStoreUtils.Utf8BytesCompare(_currentLeftRecord.KeyBytes, _currentRightRecord.KeyBytes);
}
else diff = 1; // break out of loop
firstTime = false;
}
CurrentRecord = _joiner.Join(_currentLeftRecord, _currentRightRecord, (diff == 0));
return true;
}
/// <summary>
/// Advances Iteration.
/// </summary>
/// <returns>False if at end of iteration, true otherwise.</returns>
public override bool MoveNext()
{
if (_leftSource == null) {
_leftSource = (InternalRecordSource)_inputList[0];
_rightSource = (InternalRecordSource)_inputList[1];
_rightNotDone = _rightSource.MoveNext();
if (!_rightNotDone) return false;
//_currentRightRecord = _rightSource.CurrentRecord;
}
while (true) {
// advance the left side first
_leftNotDone = _leftSource.MoveNext();
if (!_leftNotDone) return false;
// advance the right side
int diff = -1;
bool firstTime = true;
while (diff < 0) {
// the first time we test we check against the currentRightSource since
// we allow dups on the left side.
if (!firstTime && _rightNotDone) {
_rightSource.MoveNextHint = _leftSource.CurrentRecord.Key;
_rightNotDone = _rightSource.MoveNext();
if (!_rightNotDone) return false;
}
diff = TMSNStoreUtils.Utf8BytesCompare(_leftSource.CurrentRecord.KeyBytes, _rightSource.CurrentRecord.KeyBytes);
firstTime = false;
}
// if there's a match, join the right to the left
if (diff == 0) {
if (_joiner == null) {
_joiner = new RecordJoiner(_leftSource, _rightSource, _tableColumnSeparator);
}
CurrentRecord = _joiner.Join(_leftSource.CurrentRecord, _rightSource.CurrentRecord, true);
return true;
}
}
}