public void Write()
{
TStorageType outputType = TStorageType.FlatFile;
if (_outputUri == null) {
throw new Exception("must use constructor with OutputUri parameter when using this method");
}
outputType = _outputUri.StorageType;
Console.Error.Write("[" + outputType + ":" + _outputUri.FilePath + "]\tBegin Write\t");
Console.Error.WriteLine(DateTime.Now);
Console.Error.Flush();
switch (outputType) {
case TStorageType.TStore:
TMSNStoreWriter twriter = new TMSNStoreWriter(_output, _outputUri.FilePath);
twriter.KeyGroupSize = TStoreGroupSize;
twriter.Write();
break;
case TStorageType.FlatFile:
_WriteFlatFile(_outputUri.FilePath);
break;
case TStorageType.RecordFile:
RecordFileWriter writer = new RecordFileWriter(_outputUri.FilePath);
writer.Write(_output);
break;
}
_output.Close();
Console.Error.Write("[" + outputType + ":" + _outputUri.FilePath + "]\tEnd Write\t");
Console.Error.WriteLine(DateTime.Now);
Console.Error.Flush();
}
/// <summary>
/// Finalizes the writing of a TStore to disk.
/// </summary>
public void Write()
{
// we use the same disk structure to store the cache
TMSNStoreWriter keyCacheWriter = new TMSNStoreWriter(_outputDir, true);
uint cacheKeyGroupSize = 32; // we fix this one
keyCacheWriter.KeyGroupSize = cacheKeyGroupSize;
// if we are in the push mode use the sorterReducer that the records were pushed into.
if (_input == null) {
_input = _sorterReducer;
}
// Note, _input can be null if the user never pushed anything onto this writer.
if (_input != null)
{
while (_input.MoveNext())
{
// initialize file lengths. We wait till after the first MoveNext so props are set
if (_numWritten == 0)
{
TotalRecordBytesEstimate = _input.TotalRecordBytesEstimate;
TotalRecordsEstimate = _input.TotalRecordsEstimate;
Initialize();
keyCacheWriter.TotalRecordsEstimate = TotalRecordsEstimate / (long)KeyGroupSize;
keyCacheWriter.TotalRecordBytesEstimate = TotalRecordBytesEstimate / (long)KeyGroupSize;
keyCacheWriter.Initialize();
}
DataRecord record = _input.CurrentRecord;
// write first key of group to cache
if (_numWritten % KeyGroupSize == 0)
{
// write the key of the first group member to the cache.
keyCacheWriter.AddRecord(record.KeyBytes, record.Data, false);
}
// write to main store
AddRecord(record.KeyBytes, record.Data, true);
}
if (_numWritten == 0) return;
// if needed, pad the last group written with null entries so that
// we don't have to special-case reading the last group.
long unfullGroupNumMembers = _numWritten % KeyGroupSize;
if (unfullGroupNumMembers > 0)
{
uint numPadEntries = (uint)(KeyGroupSize - unfullGroupNumMembers);
for (int i = 0; i < numPadEntries; i++)
{
_keyFileWriter.Write((byte)0); // overlap
_keyFileWriter.Write((byte)0); // neu
_keyFileWriter.Write((byte)0); // dataLen
}
}
// take the input source and write it's properties to disk
string recordInfoFile = Path.Combine(_outputDir, "record-info");
using (Stream sw = new FileStream(recordInfoFile, FileMode.Create))
{
_input.TotalRecordsEstimate = _numWritten;
_input.TotalRecordBytesEstimate = _keyFileStream.Position;
_input.WriteProperties(sw);
}
}
Finish();
keyCacheWriter.Finish();
if (_sorterReducer != null) _sorterReducer.Close();
}