public void Test()
{
var rnd = new Random();
var sw = Stopwatch.StartNew();
var gw = Stopwatch.StartNew();
using (var fs = new FileStream(Filename, FileMode.OpenOrCreate))
{
const int iter = 1000;
for (int bytes = 100; bytes < 1000000; bytes *= 2)
{
var arr = new byte[bytes];
rnd.NextBytes(arr);
TimeSpan min = TimeSpan.FromHours(1);
TimeSpan max = TimeSpan.Zero;
gw.Restart();
for (int i = 0; i < iter; ++i)
{
fs.Write(arr, 0, arr.Length);
sw.Restart();
fs.FlushToDisk();
var elapsed = sw.Elapsed;
min = elapsed < min ? elapsed : min;
max = elapsed > max ? elapsed : max;
}
gw.Stop();
Console.WriteLine("{0} bytes, Min: {1}, Max: {2}, Avg: {3}",
bytes,
min,
max,
TimeSpan.FromTicks(gw.Elapsed.Ticks/iter));
}
}
}
private void TruncateChunkAndFillWithZeros(ChunkHeader chunkHeader, string chunkFilename, long truncateChk)
{
if (chunkHeader.IsScavenged
|| chunkHeader.ChunkStartNumber != chunkHeader.ChunkEndNumber
|| truncateChk < chunkHeader.ChunkStartPosition
|| truncateChk >= chunkHeader.ChunkEndPosition)
{
throw new Exception(
string.Format("Chunk #{0}-{1} ({2}) is not correct unscavenged chunk. TruncatePosition: {3}, ChunkHeader: {4}.",
chunkHeader.ChunkStartNumber, chunkHeader.ChunkEndNumber, chunkFilename, truncateChk, chunkHeader));
}
using (var fs = new FileStream(chunkFilename, FileMode.Open, FileAccess.ReadWrite, FileShare.Read))
{
fs.SetLength(ChunkHeader.Size + chunkHeader.ChunkSize + ChunkFooter.Size);
fs.Position = ChunkHeader.Size + chunkHeader.GetLocalLogPosition(truncateChk);
var zeros = new byte[65536];
var leftToWrite = fs.Length - fs.Position;
while (leftToWrite > 0)
{
var toWrite = (int)Math.Min(leftToWrite, zeros.Length);
fs.Write(zeros, 0, toWrite);
leftToWrite -= toWrite;
}
fs.FlushToDisk();
}
}
public static PTable MergeTo(IList<PTable> tables, string outputFile, Func<IndexEntry, bool> recordExistsAt, int cacheDepth = 16)
{
Ensure.NotNull(tables, "tables");
Ensure.NotNullOrEmpty(outputFile, "outputFile");
Ensure.Nonnegative(cacheDepth, "cacheDepth");
var fileSize = GetFileSize(tables); // approximate file size
if (tables.Count == 2)
return MergeTo2(tables, fileSize, outputFile, recordExistsAt, cacheDepth); // special case
Log.Trace("PTables merge started.");
var watch = Stopwatch.StartNew();
var enumerators = tables.Select(table => table.IterateAllInOrder().GetEnumerator()).ToList();
for (int i = 0; i < enumerators.Count; i++)
{
if (!enumerators[i].MoveNext())
{
enumerators[i].Dispose();
enumerators.RemoveAt(i);
i--;
}
}
long dumpedEntryCount = 0;
using (var f = new FileStream(outputFile, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None,
DefaultSequentialBufferSize, FileOptions.SequentialScan))
{
f.SetLength(fileSize);
f.Seek(0, SeekOrigin.Begin);
using (var md5 = MD5.Create())
using (var cs = new CryptoStream(f, md5, CryptoStreamMode.Write))
using (var bs = new BufferedStream(cs, DefaultSequentialBufferSize))
{
// WRITE HEADER
var headerBytes = new PTableHeader(Version).AsByteArray();
cs.Write(headerBytes, 0, headerBytes.Length);
var buffer = new byte[IndexEntrySize];
// WRITE INDEX ENTRIES
while (enumerators.Count > 0)
{
var idx = GetMaxOf(enumerators);
var current = enumerators[idx].Current;
if (recordExistsAt(current))
{
AppendRecordTo(bs, current.Bytes, buffer);
dumpedEntryCount += 1;
}
if (!enumerators[idx].MoveNext())
{
enumerators[idx].Dispose();
enumerators.RemoveAt(idx);
}
}
bs.Flush();
cs.FlushFinalBlock();
f.FlushToDisk();
f.SetLength(f.Position + MD5Size);
// WRITE MD5
var hash = md5.Hash;
f.Write(hash, 0, hash.Length);
f.FlushToDisk();
}
}
Log.Trace("PTables merge finished in {0} ([{1}] entries merged into {2}).",
watch.Elapsed, string.Join(", ", tables.Select(x => x.Count)), dumpedEntryCount);
return new PTable(outputFile, Guid.NewGuid(), depth: cacheDepth);
}
private void CreateWriterWorkItemForNewChunk(ChunkHeader chunkHeader, int fileSize)
{
var md5 = MD5.Create();
// create temp file first and set desired length
// if there is not enough disk space or something else prevents file to be resized as desired
// we'll end up with empty temp file, which won't trigger false error on next DB verification
var tempFilename = string.Format("{0}.{1}.tmp", _filename, Guid.NewGuid());
var tempFile = new FileStream(tempFilename, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.Read,
WriteBufferSize, FileOptions.SequentialScan);
tempFile.SetLength(fileSize);
// we need to write header into temp file before moving it into correct chunk place, so in case of crash
// we don't end up with seemingly valid chunk file with no header at all...
WriteHeader(md5, tempFile, chunkHeader);
tempFile.FlushToDisk();
tempFile.Close();
File.Move(tempFilename, _filename);
var stream = new FileStream(_filename, FileMode.Open, FileAccess.ReadWrite, FileShare.Read,
WriteBufferSize, FileOptions.SequentialScan);
stream.Position = ChunkHeader.Size;
_writerWorkItem = new WriterWorkItem(stream, null, md5);
Flush(); // persist file move result
}
private static PTable MergeTo2(IList<PTable> tables, long fileSize, string outputFile,
Func<IndexEntry, bool> recordExistsAt, int cacheDepth)
{
Log.Trace("PTables merge started (specialized for <= 2 tables).");
var watch = Stopwatch.StartNew();
var enumerators = tables.Select(table => table.IterateAllInOrder().GetEnumerator()).ToList();
long dumpedEntryCount = 0;
using (var f = new FileStream(outputFile, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None,
DefaultSequentialBufferSize, FileOptions.SequentialScan))
{
f.SetLength(fileSize);
f.Seek(0, SeekOrigin.Begin);
using (var md5 = MD5.Create())
using (var cs = new CryptoStream(f, md5, CryptoStreamMode.Write))
using (var bs = new BufferedStream(cs, DefaultSequentialBufferSize))
{
// WRITE HEADER
var headerBytes = new PTableHeader(Version).AsByteArray();
cs.Write(headerBytes, 0, headerBytes.Length);
// WRITE INDEX ENTRIES
var buffer = new byte[IndexEntrySize];
var enum1 = enumerators[0];
var enum2 = enumerators[1];
bool available1 = enum1.MoveNext();
bool available2 = enum2.MoveNext();
IndexEntry current;
while (available1 || available2)
{
if (available1 && (!available2 || enum1.Current.CompareTo(enum2.Current) > 0))
{
current = enum1.Current;
available1 = enum1.MoveNext();
}
else
{
current = enum2.Current;
available2 = enum2.MoveNext();
}
if (recordExistsAt(current))
{
AppendRecordTo(bs, current.Bytes, buffer);
dumpedEntryCount += 1;
}
}
bs.Flush();
cs.FlushFinalBlock();
f.SetLength(f.Position + MD5Size);
// WRITE MD5
var hash = md5.Hash;
f.Write(hash, 0, hash.Length);
f.FlushToDisk();
}
}
Log.Trace("PTables merge finished in {0} ([{1}] entries merged into {2}).",
watch.Elapsed, string.Join(", ", tables.Select(x => x.Count)), dumpedEntryCount);
return new PTable(outputFile, Guid.NewGuid(), depth: cacheDepth);
}