public void ProcessRecord(DataRecord record, RecordAccepter accepter)
{
DataRecord returnRecord = null;
// eat the first record
if (_prevInputRecord == null) {
_prevInputRecord = record;
return; // give nothing to the accepter
}
_currentInputRecord = record;
// we get called a final time with null so we can output any state we have
int diff;
if (record != null) {
//diff = _prevInputRecord.Key.CompareTo(_currentInputRecord.Key);
diff = TMSNStoreUtils.Utf8BytesCompare(_prevInputRecord.KeyBytes, _currentInputRecord.KeyBytes);
}
else diff = -1; // force a non-match
// the keys match
if (diff == 0) {
// if reduction hasn't occured yet then this is the first time thru here.
// use the prevInputRecord as an accumulator
if (!_reductionOccured) {
_accumulatorRecord = _prevInputRecord;
_reductionOccured = true;
}
((IReducableRecord)_accumulatorRecord).ReduceData((IReducableRecord)_currentInputRecord);
returnRecord = null; // no record to return yet
}
// the keys don't match
else {
// if no reduction occured, the prev record needs to get out
if (!_reductionOccured) {
returnRecord = (DataRecord)_prevInputRecord;
}
// reduction occured in the _accumulatorRecord output it
else {
// set up for next time around
_reductionOccured = false; //
returnRecord = (DataRecord)_accumulatorRecord;
}
}
// advance
_prevInputRecord = _currentInputRecord;
if (returnRecord != null) accepter.AddRecord(returnRecord);
}
/// <summary>
/// Adds a record to the accepter (FIFO).
/// </summary>
/// <param name="record">The record to be added.</param>
public void AddRecord(DataRecord record)
{
if (_inputIndex >= _fifoMemorySize) {
//byte[][] oldKey = _keyFifo;
//byte[][] oldData = _dataFifo;
DataRecord[] oldFifo = _recordFifo;
// double the size (could be more efficient if we paid attention
// to the input/output pointers but since the generally records won't
// have a very high amplification factor this shouldn't be a big deal).
_fifoMemorySize += _fifoMemorySize;
//_keyFifo = new byte[_fifoMemorySize][];
//_dataFifo = new byte[_fifoMemorySize][];
_recordFifo = new DataRecord[_fifoMemorySize];
for (int i = 0; i < oldFifo.Length; i++) {
//_keyFifo[i] = oldKey[i];
//_dataFifo[i] = oldData[i];
_recordFifo[i] = oldFifo[i];
}
}
//_keyFifo[_inputIndex] = record.KeyBytes;
//_dataFifo[_inputIndex] = record.Data;
_recordFifo[_inputIndex] = record;
_inputIndex++;
_numRecords++;
// we grab the first inputted record. This will be used as the output
// record instance we change the key and data on.
//if (_currentRecord == null) {
// _currentRecord = record;
//}
}
public virtual void ProcessRecord(DataRecord record, RecordAccepter accepter)
{
}
public void ProcessRecord(DataRecord record, RecordAccepter accepter)
{
if (_numRecords >= _recordLimit) {
accepter.IsDone = true;
}
else {
accepter.AddRecord(record);
_numRecords++;
}
}
private bool NullMoveNext()
{
_record = new DataRecord();
if (_nullFixed != null) {
_record.KeyBytes = _nullFixed;
}
else {
_record.KeyBytes = Guid.NewGuid().ToByteArray();
}
CurrentRecord = _record;
return true;
}
public override bool MoveNext()
{
string line = null;
if (_flatFileReader != null) {
line = _flatFileReader.ReadLine();
}
while (line == null && _currentFileNo < _files.Length - 1) {
Stream stream = null;
try {
stream = ZStreamIn.Open(_files[++_currentFileNo]);
}
catch {
Console.Error.WriteLine("Error opening: " + _files[_currentFileNo] + ". Skipping.");
continue;
}
if (_flatFileReader != null) {
_flatFileReader.Close(); // close prev
}
_flatFileReader = new StreamReader(stream);
line = _flatFileReader.ReadLine();
}
if (line == null) return false;
_record = new DataRecord();
//_record.Key = line;
_record.KeySpaceNormalized = line;
CurrentRecord = _record;
return true;
}
public bool AddRecord(DataRecord record)
{
// special case this InternalRecordSource since we don't add a source to it.
if (_recordInfo.RecordType == null) {
_recordInfo.Update(record);
}
int dataLen = 0;
byte[] data = record.Data;
if (data != null) dataLen = data.Length;
long recordSize = record.KeyBytes.Length + dataLen;
// if buffer too small double size
if (recordSize > (_recordBytesSize - _recordBytesUsed)) {
// if we've reach max size we can't grow anymore. Return failure
if (_recordBytesSize >= _maxRecordBytes) return false;
// double memory upto max size
long tempSize = (long)_recordBytesSize + (long)_recordBytesSize;
tempSize = Math.Min(tempSize, (long)_maxRecordBytes); // maxRecordBytes << 2GB (BlockCopy doesnt work with long)
_recordBytesSize = (int)tempSize;
// if still not big enuf - fail.
if (recordSize > (_recordBytesSize - _recordBytesUsed)) return false;
// else get the memory.
byte[] oldRecordBytes = _recordBytes;
_recordBytes = new byte[_recordBytesSize];
Buffer.BlockCopy(oldRecordBytes, 0, _recordBytes, 0, oldRecordBytes.Length);
}
int recordOffset = _recordBytesUsed;
// copy in the key
Buffer.BlockCopy(record.KeyBytes, 0, _recordBytes, _recordBytesUsed, record.KeyBytes.Length);
_recordBytesUsed += record.KeyBytes.Length;
// copy in the data
if (data != null) {
Buffer.BlockCopy(data, 0, _recordBytes, _recordBytesUsed, dataLen);
_recordBytesUsed += dataLen;
}
// if we're out of entries double it.
if (_inputIndex >= _recordEntries.Length) {
RecordEntry[] oldEntries = _recordEntries;
_recordEntries = new RecordEntry[_recordEntries.Length + _recordEntries.Length];
for (int i = 0; i < oldEntries.Length; i++) {
_recordEntries[i].ByteOffset = oldEntries[i].ByteOffset;
_recordEntries[i].KeyLen = oldEntries[i].KeyLen;
_recordEntries[i].DataLen = oldEntries[i].DataLen;
}
}
_recordEntries[_inputIndex].ByteOffset = recordOffset;
_recordEntries[_inputIndex].KeyLen = record.KeyBytes.Length;
_recordEntries[_inputIndex].DataLen = dataLen;
_inputIndex++;
_numRecords++;
TotalRecordBytesEstimate += recordSize;
TotalRecordsEstimate++;
return true;
}
public static ConstructorInfo GetConstructor(DataRecord record)
{
Type recordType = record.GetType();
ConstructorInfo constructor = recordType.GetConstructor(new Type[0]);
return constructor;
}
// for wrapping non-table records for table conversion
public ColumnWrapper(DataRecord record, string sourceName, char separator, bool hasHeaders)
{
_separator = separator;
_isTableRecord = false;
if (hasHeaders) {
// only allow headers for flat files (they have Key=line Data=null)
if (record.Data != null) {
Console.WriteLine("Only legal to specify headers for flat file input");
Environment.Exit(1);
}
// ignore the wrapper produced columnNames, this record has them
// as it's key. No Data columns allowed. Must come from file.
_columnNames = record.Key.Split(_separator);
_keyColumnNos = new int[_columnNames.Length];
for (int i = 0; i < _keyColumnNos.Length; i++) _keyColumnNos[i] = i;
}
// no headers make up column names
else {
_SetColumnNames(record, sourceName);
}
}
// for wrapping records to be joined together. Either table or non-table records
public ColumnWrapper(DataRecord record, string sourceName, char separator)
{
if (record is TableRecord) {
_isTableRecord = true;
_columnNames = ((TableRecord)record).ColumnNames;
_keyColumnNos = ((TableRecord)record).KeyColumnNos;
return;
}
_separator = separator;
_isTableRecord = false;
_SetColumnNames(record, sourceName);
}
public override void ProcessRecord(DataRecord record, RecordAccepter accepter)
{
// caseNum 0: null conversion (key expression == incoming key)
// caseNum 1: table record
// caseNum 2: everything else
// So, the thing is a null conversion is costly, potentially super
// costly. Instead of trying to rebuild the whole tree without
// conversion (which brought with it potential sorting nodes as
// well) we will error and message the user to remove the null
// conversion.
// initialize the wrapper
if (_wrapper == null) {
bool hasHeaders = false;
int caseNum = 0;
HintMessageToConsole();
// case 1: table record.
if (record is TableRecord) {
TableRecord t = record as TableRecord;
_wrapper = new ColumnWrapper(t);
caseNum = 1; // table record
}
else {
caseNum = 2; // everything else
hasHeaders = ColumnWrapper.HasHeaders(KeyExpression);
_wrapper = new ColumnWrapper(record, _sourceName, _columnSeparator, hasHeaders);
// if columnNames provided by filter user
if (_columnNames != null) {
if (_columnNames.Length != _wrapper.ColumnNames.Length) {
Console.WriteLine("too few column names provided");
Environment.Exit(1);
}
}
else {
_columnNames = _wrapper.ColumnNames; // use default
}
}
// making table records is costly. If the key columns have not
// changed there is no reason to do the conversion. (since
// sorting and joining work the same for all records).
_keyColumnNos = _wrapper.GetColumnNos(KeyExpression);
int[] currentKeyColumnNos = _wrapper.KeyColumnNos;
// _keyColumnNos == the requested new key columns
if (_keyColumnNos.Length == currentKeyColumnNos.Length) {
for (int i = 0; i < _keyColumnNos.Length; i++) {
if (_keyColumnNos[i] != currentKeyColumnNos[i])
_caseNum = caseNum;
}
}
else _caseNum = caseNum;
// we special case flat files converting to tables allowing null conversions
// since when they define headers ToTable is evaluating and dropping them
// from the input.
if (hasHeaders) {
_caseNum = 2;
return; // eat up this record containing headers
}
}
switch (_caseNum) {
case 0: // null conversion see comments above
Console.WriteLine("Null-table conversions are costly (i.e. key expression equal to incoming key).");
Console.WriteLine("Remove unnecessary 'ToTable(" + KeyExpression + ")' from expression.");
Environment.Exit(1);
break;
case 1: // table record
TableRecord outRecord = record as TableRecord;
outRecord.KeyColumnNos = _keyColumnNos;
accepter.AddRecord(outRecord);
break;
case 2: // everything else
_wrapper.SetRecord(record);
// ignore null records: no key no data
if (record.Key.Length == 0 && record.Data == null)
return;
outRecord = new TableRecord(_columnNames, _columnSeparator);
//if (_outRecord == null) {
// _outRecord = new TableRecord(_columnNames, _columnSeparator);
//}
outRecord.KeyColumnNos = _keyColumnNos;
_stringBuilder.Length = 0;
for (int i = 0; i < _wrapper.ColumnNames.Length; i++) {
if (i != 0) _stringBuilder.Append('\t');
_wrapper.AppendColumn(i, _stringBuilder);
}
outRecord.DelimitedColumns = _stringBuilder.ToString();
accepter.AddRecord(outRecord);
break;
}
}
public override void ProcessRecord(DataRecord record, RecordAccepter accepter)
{
if (_wrapper == null) {
bool hasHeaders = false;
HintMessageToConsole();
if (record is TableRecord) {
_wrapper = new ColumnWrapper(record as TableRecord);
_keyColumnNos = _wrapper.GetColumnNos(KeyExpression);
}
else {
hasHeaders = ColumnWrapper.HasHeaders(KeyExpression);
_wrapper = new ColumnWrapper(record, _sourceName, _separator, hasHeaders);
_keyColumnNos = _wrapper.GetColumnNos(KeyExpression);
}
if (hasHeaders) return; // eat up this record containing headers
}
//if (_outRecord == null) {
// _outRecord = new DataRecord();
//}
DataRecord outRecord = new DataRecord();
_wrapper.SetRecord(record);
// build the key
_stringBuilder.Length = 0;
for (int i = 0; i < _keyColumnNos.Length; i++) {
if (i != 0) _stringBuilder.Append(_separator);
_wrapper.AppendColumn(_keyColumnNos[i], _stringBuilder);
}
outRecord.Key = _stringBuilder.ToString();
accepter.AddRecord(outRecord);
}
public override void ProcessRecord(DataRecord record, RecordAccepter accepter)
{
if (_wrapper == null) {
bool hasHeaders = false;
HintMessageToConsole();
if (record is TableRecord) {
_wrapper = new ColumnWrapper(record as TableRecord);
_keyColumnNos = _wrapper.GetColumnNos(KeyExpression);
}
else {
hasHeaders = ColumnWrapper.HasHeaders(KeyExpression);
_wrapper = new ColumnWrapper(record, _sourceName, _separator, hasHeaders);
_keyColumnNos = _wrapper.GetColumnNos(KeyExpression);
}
// if the countColumnExpression ends in a bang! we allow ulong.Parse errors.
// otherwise we abort on an error.
if (_countExpression != null && _countExpression.EndsWith("!")) {
_interpretAsZero = true;
_countExpression = _countExpression.TrimEnd('!');
}
if (_countExpression != null) {
int[] countColumnNos = _wrapper.GetColumnNos(_countExpression);
if (countColumnNos.Length != 1) {
Console.WriteLine("Illegal Count Column expression");
Environment.Exit(1);
}
_countColumnNo = countColumnNos[0];
if (_countColumnNo > _wrapper.ColumnNames.Length - 1) {
Console.WriteLine("Illegal Count Column expression");
Environment.Exit(1);
}
}
// if countRecord and count column is last column.
if (record is CountRecord && _countColumnNo == _wrapper.ColumnNames.Length - 1) {
_caseNum = 0;
}
// if no expression given use 1
else if (_countExpression == null) {
_caseNum = 1;
}
else _caseNum = 2;
if (hasHeaders) return; // eat up this record containing headers
}
// cases:
// 0 : record = CountRecord && countColumn refers to the count.
// 1 : countExpression == null. Just 1-count the keys
// 2 : everything else
// not sure if this is the best way to ignore blank lines coming in.
if (record.Key.Length == 0) return;
CountRecord outRecord = new CountRecord();
_wrapper.SetRecord(record);
// build the key
_stringBuilder.Length = 0;
for (int i = 0; i < _keyColumnNos.Length; i++) {
if (i != 0) _stringBuilder.Append(_separator);
_wrapper.AppendColumn(_keyColumnNos[i], _stringBuilder);
}
outRecord.Key = _stringBuilder.ToString();
// we special case 0, because then we can avoid converting from ulong to string
// and back to ulong.
switch (_caseNum) {
case 0:
outRecord.Count = ((CountRecord)record).Count;
break;
case 1:
outRecord.Count = 1;
break;
case 2:
_stringBuilder.Length = 0;
_wrapper.AppendColumn(_countColumnNo, _stringBuilder);
try {
outRecord.Count = ulong.Parse(_stringBuilder.ToString());
}
catch {
if (!_interpretAsZero) {
Console.WriteLine("Illegal ulong string '{0}'.\nTo interpret as zero: count column expression = ${1}!", _stringBuilder.ToString(), _countColumnNo+1);
Environment.Exit(-1);
}
outRecord.Count = 0;
_numParseErrors++;
//return; // abort this record
}
break;
}
accepter.AddRecord(outRecord);
}
public TableRecord Join(DataRecord left, DataRecord right, bool match)
{
_leftWrapper.SetRecord(left);
_rightWrapper.SetRecord(right);
TableRecord outRecord = new TableRecord(_outputColumnNames, _separator);
outRecord.KeyColumnNos = _leftWrapper.KeyColumnNos;
_delimitedColumns.Length = 0; // reset
// add the left columns
for (int i = 0; i < _leftWrapper.ColumnNames.Length; i++) {
if (i != 0) _delimitedColumns.Append(_separator);
_leftWrapper.AppendColumn(i, _delimitedColumns);
}
if (!match) {
_delimitedColumns.Append(_emptyRightColumns);
outRecord.DelimitedColumns = _delimitedColumns.ToString();
return outRecord;
}
// we matched.
// add the right columns
int j = 0;
for (int i = 0; i < _rightWrapper.ColumnNames.Length; i++) {
if (i == _rightWrapper.KeyColumnNos[j]) { // don't add keys
if (_rightWrapper.KeyColumnNos.Length > j + 1) j++;
}
else {
_delimitedColumns.Append(_separator);
_rightWrapper.AppendColumn(i, _delimitedColumns);
}
}
outRecord.DelimitedColumns = _delimitedColumns.ToString();
return outRecord;
}
internal bool MoveNext()
{
_outputIndex++; // advance the index to the next record to output.
if (_outputIndex < _inputIndex) {
//_currentRecord.KeyBytes = _keyFifo[_outputIndex];
//_currentRecord.Data = _dataFifo[_outputIndex];
_currentRecord = _recordFifo[_outputIndex];
_numRecords--;
// reset indices
if (_numRecords == 0) {
_inputIndex = 0;
_outputIndex = -1;
}
return true;
}
return false;
}
/// <summary>
///
/// </summary>
/// <param name="record"></param>
public void WriteRecord(DataRecord record)
{
if (_outputStream == null) {
OpenOutputStream();
_outputWriter = new VariableLengthBinaryWriter(_outputStream);
EmptyInternalSource e = new EmptyInternalSource();
e.CurrentRecord = record; // updates the record info
e.WriteProperties(_outputStream);
}
_outputWriter.WriteVariableLength((uint)record.KeyBytes.Length);
_outputWriter.Write(record.KeyBytes);
// output data
if (record.Data != null) {
_outputWriter.WriteVariableLength((uint)record.Data.Length);
_outputWriter.Write(record.Data);
}
else {
_outputWriter.WriteVariableLength((uint)0);
}
// slow do elsewhere _totalRecordBytesEstimate += rawRecord.Length;
_totalRecordsEstimate++;
}
public static DataRecord CreateRecordInstance(DataRecord record)
{
ConstructorInfo constructor = TMSNStoreUtils.GetConstructor(record);
DataRecord newRecord = (DataRecord)constructor.Invoke(null);
return newRecord;
}
public void SetRecord(DataRecord record)
{
if (record == null) return;
if (_isTableRecord) {
TableRecord tableRecord = record as TableRecord;
_delimitedColumns = tableRecord.DelimitedColumns;
_AdvanceColumn(true); // sets up iteration
}
// else non-table so convert into columns using separator over key and data
else {
// for non-table records we set delimitedColumns equal to the key.
// In this way the columns of the key are available and if the data
// columns aren't needed they aren't computed. The record has been
// saved away for this computation.
_nonTableRecord = record;
_AdvanceColumn(true); // sets up iteration
}
}
public void Update(DataRecord recordInstance)
{
_type = recordInstance.GetType();
if (recordInstance is TableRecord) {
_tableColumnNames = ((TableRecord)recordInstance).ColumnNames;
KeyColumnNos = ((TableRecord)recordInstance).KeyColumnNos;
}
}
private void _SetColumnNames(DataRecord record, string sourceName)
{
string[] keyCols = record.Key.Split(_separator);
string[] dataCols = new string[0];
if (record.Data != null) {
dataCols = record.DataAsString.Split(_separator);
}
_columnNames = new string[keyCols.Length + dataCols.Length];
for (int i = 0; i < keyCols.Length; i++)
// display is one-based
_columnNames[i] = sourceName + ".Key." + (i + 1).ToString();
for (int i = 0; i < dataCols.Length; i++)
// display is one-based
_columnNames[keyCols.Length + i] = sourceName + ".Data." + (i + 1).ToString();
// keyColumnNos is zero-based
_keyColumnNos = new int[keyCols.Length];
for (int i = 0; i < _keyColumnNos.Length; i++) _keyColumnNos[i] = i;
}
public void AddRecord(DataRecord record)
{
if (_recordsToSort == null) {
_recordsToSort = new SortableRecordMemory(_internalSortAscending, _sortableRecordMaxMemorySize);
}
// attempt to add the new record
bool success = _recordsToSort.AddRecord(record);
// see if we need to reduce by setting the _filter or leaving it null
if (!_reductionDetermined)
{
if (!(record is IReducableRecord))
{
_internalReductionEnabled = false;
}
_reductionDetermined = true;
}
//if (inputInstance == null) inputInstance = input.CurrentRecord;
// rats! not enough memory. Sort and write temp file, then add record to new sorter
if (!success)
{
if (_doThreading) _SortAndWriteThreaded();
else _SortAndWrite();
// try again
AddRecord(record);
}
}
/// <summary>
/// Advances iteration.
/// </summary>
/// <returns>False if at end of iteration, true otherwise.</returns>
public override bool MoveNext()
{
if (_leftSource == null) {
_leftSource = (InternalRecordSource)_inputList[0];
_rightSource = (InternalRecordSource)_inputList[1];
// advance the right side from the start so that
// a record is there waiting to be compared to
_rightNotDone = _rightSource.MoveNext();
if (_rightNotDone) _currentRightRecord = _rightSource.CurrentRecord;
}
_leftNotDone = _leftSource.MoveNext();
if (!_leftNotDone) return false;
_currentLeftRecord = _leftSource.CurrentRecord;
// Don't need a match to create a joiner
if (_joiner == null) {
_joiner = new RecordJoiner(_leftSource, _rightSource, _tableColumnSeparator);
}
// advance the right side
int diff = -1;
bool firstTime = true;
while (diff < 0) {
// the first time we test we check against the currentRightSource
// (i.e. don't go in this block) since we allow dups on the left side.
if (!firstTime && _rightNotDone) {
_rightSource.MoveNextHint = _currentLeftRecord.Key;
_rightNotDone = _rightSource.MoveNext();
if (!_rightNotDone) _currentRightRecord = null;
else _currentRightRecord = _rightSource.CurrentRecord;
}
if (_currentRightRecord != null) {
diff = TMSNStoreUtils.Utf8BytesCompare(_currentLeftRecord.KeyBytes, _currentRightRecord.KeyBytes);
}
else diff = 1; // break out of loop
firstTime = false;
}
CurrentRecord = _joiner.Join(_currentLeftRecord, _currentRightRecord, (diff == 0));
return true;
}
public override bool MoveNext()
{
if (_nullInput) {
return NullMoveNext();
}
string line = _flatFileReader.ReadLine();
if (line == null) return false;
_record = new DataRecord();
_record.KeySpaceNormalized = line;
CurrentRecord = _record;
return true;
}
public override bool MoveNext()
{
if (_source == null) {
_source = Inputs[0];
}
// we already retrieved the next record and stuck it here.
if (_nextRecord != null) {
CurrentRecord = _nextRecord;
_nextRecord = null; // reset it.
return true;
}
bool notDone = _source.MoveNext();
CurrentRecord = _source.CurrentRecord;
return notDone;
}
/// <summary>
/// Method for adding records to a TStore. (Push model).
/// </summary>
/// <param name="record">Record to be added.</param>
public void AddRecord(DataRecord record)
{
if (_sorterReducer == null) {
if (_input != null) {
throw new Exception("cannot use AddRecord method together with InternalRecordSource");
}
_sorterReducer = new RecordSorterReducer(TempDir, true, reductionEnabled);
if (maxMemorySize != -1)
_sorterReducer.MaxMemorySize = maxMemorySize;
}
_sorterReducer.AddRecord(record);
}
/// <summary>
/// Gets a record from the database accessing by key.
/// </summary>
/// <param name="key">The key of record to access.</param>
/// <param name="record">The record retrieved.</param>
public void GetRecord(string key, DataRecord record)
{
record.Data = null;
record.Key = key;
record._recordNo = -1;
byte[] keyBytes = UTF8Encoding.UTF8.GetBytes(key);
//byte[] keyBytes = _encoding.GetBytes(key); NOT THREAD SAFE
long high, low, mid; // need to express neg nos for the while constraint
int diff;
long lastGroupNo = 0;
low = 0;
high = _keyIndex.NumKeyGroups - 1;
while (high >= low) {
mid = low + (high - low) / 2;
diff = _KeyCompare(keyBytes, (uint)mid, false);
// the key is equal to the first member of the group -- done
if (diff == 0) {
record._recordNo = mid * _keyIndex.KeyGroupSize;
record.Data = _GetDataOfFirstMember((uint)mid);
return;
}
// the key is greater than -- raise the floor
else if (diff > 0) {
low = mid + 1;
lastGroupNo = mid;
}
// the key is less than -- lower the ceiling
else {
high = mid - 1;
}
}
// the key still might be in the low group because we don't check all members
int groupMemberNum;
byte[] data = _FindKeyInGroup(keyBytes, (uint)lastGroupNo, out groupMemberNum);
if (groupMemberNum != -1) {
record._recordNo = lastGroupNo * _keyIndex.KeyGroupSize + groupMemberNum;
record.Data = data;
}
}
// EACH RECORD (LINE)
public void ProcessRecord(DataRecord record, RecordAccepter accepter)
{
if (_rank < _recordArray.Length) {
_recordArray[_rank] = record;
}
else {
// pick random from 0 to rank
long l = (long)(_random.NextDouble() * (double)_rank);
if (l < _recordArray.Length) {
_recordArray[l] = record;
}
}
// output no records till done
// accepter.AddRecord(record);
_rank++;
}
public override bool MoveNext()
{
if (_leftSource == null) {
_leftSource = (InternalRecordSource)_inputList[0];
_rightSource = (InternalRecordSource)_inputList[1];
}
while (true) {
_leftNotDone = _leftSource.MoveNext();
if (!_leftNotDone) return false;
_currentLeftRecord = _leftSource.CurrentRecord;
// advance the right side
int diff = -1;
bool firstTime = true;
while (diff < 0) {
// the first time we test we check against the currentRightSource since
// we allow dups on the left side.
if (!firstTime || _currentRightRecord == null) {
if (_rightNotDone) _rightNotDone = _rightSource.MoveNext();
// passThruOnMatch == left & right
// !passThruOnMatch == left &! right
// if left & right then when right is done we're done.
// if left &! right when right is done keep going so we
// can emit all the lefts that come after the last right.
if (!_rightNotDone) {
if (_passThruOnMatch) return false;
else {
CurrentRecord = _currentLeftRecord;
return true;
}
}
_currentRightRecord = _rightSource.CurrentRecord;
}
diff = TMSNStoreUtils.Utf8BytesCompare(_currentLeftRecord.KeyBytes, _currentRightRecord.KeyBytes);
firstTime = false;
}
// if there's a match
if (diff == 0) {
if (_passThruOnMatch) {
CurrentRecord = _currentLeftRecord;
return true;
}
}
else if (!_passThruOnMatch) {
CurrentRecord = _currentLeftRecord;
return true;
}
}
}
public void ProcessRecord(DataRecord record, RecordAccepter accepter)
{
// here we load the accepter with the stats we get from the source
// and we call it a day.
foreach (string stat in _source.Statistics) {
DataRecord outRecord = new DataRecord();
outRecord.Key = stat;
accepter.AddRecord(outRecord);
}
accepter.IsDone = true;
}
/// <summary>
/// Gets a record from the database accessing by key.
/// </summary>
/// <param name="key">The key of record to access.</param>
/// <param name="record">The record retrieved.</param>
public void GetRecord(string key, DataRecord record)
{
record.Data = null;
record.Key = key;
record._recordNo = -1;
_lastRecordNo = -10; // so that sequential access knows this has intervened.
byte[] keyBytes = _encoding.GetBytes(key);
long high, low, mid; // need to express neg nos for the while constraint
int diff;
long lastGroupNo = 0;
low = 0;
high = _keyIndex.NumKeyGroups - 1;
//int cacheEntryNo = 0;
while (high >= low) {
mid = low + (high - low) / 2;
diff = _KeyCompare(keyBytes, (uint)mid, false);
// the key is equal to the first member of the group -- done
if (diff == 0) {
record._recordNo = mid * _keyIndex.KeyGroupSize;
record.Data = _GetData();
return;
}
// the key is greater than -- raise the floor
else if (diff > 0) {
low = mid + 1;
lastGroupNo = mid;
}
// the key is less than -- lower the ceiling
else {
high = mid - 1;
}
}
// the key still might be in the low group because we don't check all members
int groupMemberNum = _FindKeyInGroup(keyBytes, (uint)lastGroupNo);
if (groupMemberNum != -1) {
record._recordNo = lastGroupNo * _keyIndex.KeyGroupSize + groupMemberNum;
record.Data = _GetData();
}
}