/// <summary>
/// SortAscending is a parameter of the constructor because we have to know immediately
/// what our properties are so we can pass them on to whoever asks of us.
/// </summary>
public SortableRecordMemory(bool sortAscending, int maxMemorySize)
{
_recordMemoryStream = new MemoryStream();
_recordMemoryWriter = new VariableLengthBinaryWriter(_recordMemoryStream);
_recordBytes = new byte[_recordBytesSize];
_maxRecordBytes = maxMemorySize;
_internalSortAscending = sortAscending;
// means our sorting properties are not propagated from inputs to outputs.
_passThruInputSorting = false;
// the sorting properties, like (segmenting, bucketing, reduced, ...) have to reflect
// immediately what the component is going to do even before it has done it. So even
// before the records are sorted we set these properties because the world wants to
// know what's true of our output.
_sorting.IsSorted = true; // tells the world we sort.
_sorting.IsSortedAscending = sortAscending;
_startTick = DateTime.UtcNow.Ticks;
}
internal bool WriteRecordFileMaxSize(InternalRecordSource input, long maxFileSize)
{
// initialize
if (_outputStream == null) {
OpenOutputStream();
_outputWriter = new VariableLengthBinaryWriter(_outputStream);
input.WriteProperties(_outputStream);
}
bool done = false;
while (!done && input.MoveNext()) {
DataRecord record = input.CurrentRecord;
// output key
_outputWriter.WriteVariableLength((uint)record.KeyBytes.Length);
_outputWriter.Write(record.KeyBytes);
// output data
if (record.Data != null) {
_outputWriter.WriteVariableLength((uint)record.Data.Length);
_outputWriter.Write(record.Data);
}
else {
_outputWriter.WriteVariableLength((uint)0);
}
if (maxFileSize != -1 && _outputStream.Position >= maxFileSize) {
done = true;
}
}
_outputStream.Close();
_outputStream = null;
// we return notDone to our caller, i.e. we return true if there is more
// to be written, false if we wrote it all.
return done;
}
/// <summary>
///
/// </summary>
/// <param name="record"></param>
public void WriteRecord(DataRecord record)
{
if (_outputStream == null) {
OpenOutputStream();
_outputWriter = new VariableLengthBinaryWriter(_outputStream);
EmptyInternalSource e = new EmptyInternalSource();
e.CurrentRecord = record; // updates the record info
e.WriteProperties(_outputStream);
}
_outputWriter.WriteVariableLength((uint)record.KeyBytes.Length);
_outputWriter.Write(record.KeyBytes);
// output data
if (record.Data != null) {
_outputWriter.WriteVariableLength((uint)record.Data.Length);
_outputWriter.Write(record.Data);
}
else {
_outputWriter.WriteVariableLength((uint)0);
}
// slow do elsewhere _totalRecordBytesEstimate += rawRecord.Length;
_totalRecordsEstimate++;
}
internal void Write(InternalRecordSource input)
{
while (input.MoveNext()) {
DataRecord record = input.CurrentRecord;
// we need to do a moveNext on our input in order for RecordInstance
// to be valid.
if (_outputStream == null) {
OpenOutputStream();
input.WriteProperties(_outputStream);
_outputWriter = new VariableLengthBinaryWriter(_outputStream);
}
// output key
_outputWriter.WriteVariableLength((uint)record.KeyBytes.Length);
_outputWriter.Write(record.KeyBytes);
// output data
if (record.Data != null) {
_outputWriter.WriteVariableLength((uint)record.Data.Length);
_outputWriter.Write(record.Data);
}
else {
_outputWriter.WriteVariableLength((uint)0);
}
// slow. do elsewhere _totalRecordBytesEstimate = _outputStream.Position;
_totalRecordsEstimate++;
}
_totalRecordBytesEstimate = _outputStream.Position;
InternalRecordSource.WriteEstimates(_outputStream, _totalRecordBytesEstimate, _totalRecordsEstimate);
_outputStream.Close();
_outputStream = null;
}
private void _Construct(InternalRecordSource input, string outputDir, string fileSetName)
{
if (input != null && !input.Sorting.IsSorted) {
throw new Exception("can't output non-sorted input as TMSNStore");
}
_input = input;
_outputDir = outputDir;
_fileSetName = fileSetName;
// if directory doesn't exist make it
if (!Directory.Exists(outputDir)) {
Directory.CreateDirectory(outputDir);
}
string keyFile = Path.Combine(outputDir, _fileSetName);
string indexFile = Path.Combine(outputDir, _fileSetName + "-index");
// create fileSetName file
_keyFileStream = new FileStream(keyFile, FileMode.Create, FileAccess.Write, FileShare.None, 1024 * 1024 * 16);
_keyFileWriter = new VariableLengthBinaryWriter(_keyFileStream);
// create index file
_indexFileStream = new FileStream(indexFile, FileMode.Create, FileAccess.Write, FileShare.None, 1024 * 1024 * 16);
_indexFileWriter = new BinaryWriter(_indexFileStream);
}
