public Task WriteJumpTableAsync(CancellationToken ct)
{
ct.ThrowIfCancellationRequested();
// The jump table is the last page of the file
// - it contains the list of (offset, size) of all the levels that are in the file
// - it contains any additional attributes (that were only known after writing all the data)
// - it repeats a few important values (sequence, header crc, ...)
// - it would contain any optional signature or data that is only know after writing the data to disk, and are needed to decode the rest
// marks the start of the JT because we will need to compute the checksum later on
int startOffset = m_writer.Position;
// "JMPT"
m_writer.WriteFixed32(SnapshotFormat.JUMP_TABLE_MAGIC_NUMBER);
// Page Size (repeated)
m_writer.WriteFixed32((uint)m_pageSize);
// Sequence Number (repeated)
m_writer.WriteFixed64(m_sequence);
// Database ID (repeated)
m_writer.WriteBytes(m_uid.ToSlice());
// Header CRC (repeated)
m_writer.WriteFixed32(m_headerChecksum);
int levels = m_levels;
m_writer.WriteFixed32((uint)levels); // Level Count
for (int level = 0; level < levels; level++)
{
// Level Offset (from start of file)
m_writer.WriteFixed64((ulong)m_jumpTable[level].Key);
// Level Size (in bytes)
m_writer.WriteFixed64((ulong)m_jumpTable[level].Value);
}
//TODO: additional attributes!
m_writer.WriteFixed32(0); // 0 for now
// End Marker
m_writer.WriteFixed32(uint.MaxValue);
// Checksum
int endOffset = m_writer.Position;
uint jumpTableChecksum = SnapshotFormat.ComputeChecksum(m_writer[startOffset, endOffset]);
m_writer.WriteFixed32(jumpTableChecksum);
// optional padding to fill the rest of the page
PadPageIfNeeded(SnapshotFormat.PAGE_SIZE, 0xFE);
// we are done !
return(TaskHelpers.CompletedTask);
}
/// <summary>Write the header to the file</summary>
/// <param name="headerFlags"></param>
/// <param name="uid"></param>
/// <param name="sequence"></param>
/// <param name="count"></param>
/// <param name="timestamp"></param>
/// <param name="attributes"></param>
/// <remarks>This needs to be called before writing any level to the file</remarks>
public Task WriteHeaderAsync(SnapshotFormat.Flags headerFlags, Uuid128 uid, ulong sequence, long count, long timestamp, IDictionary<string, IFdbTuple> attributes)
{
// The header will be use on ore more "pages", to simplify the job of loading / peeking at a stream content (no need for fancy buffering, just need to read 4K pages)
// > The last page is padded with 0xAAs to detect corruption.
m_uid = uid;
m_sequence = sequence;
m_itemCount = count;
m_timestamp = timestamp;
// HEADER
// - DB_HEADER (64 bytes)
// - DB ATTRIBUTES (variable size list of k/v)
// - END_MARKER + HEADER_CRC
// - PADDING (to fill last page)
// DB Header
// "PNDB"
m_writer.WriteFixed32(SnapshotFormat.HEADER_MAGIC_NUMBER);
// v1.0
m_writer.WriteFixed16(1); // major
m_writer.WriteFixed16(0); // minor
// FLAGS
m_writer.WriteFixed64((ulong)headerFlags);
// Database ID
m_writer.WriteBytes(uid.ToSlice());
// Database Version
m_writer.WriteFixed64(sequence);
// Number of items in the database
m_writer.WriteFixed64((ulong)count);
// Database Timestamp
m_writer.WriteFixed64((ulong)timestamp);
// Page Size
m_writer.WriteFixed32(SnapshotFormat.PAGE_SIZE);
// Header Size (not known yet and will be filled in later)
int offsetToHeaderSize = m_writer.Skip(4);
// we should be at the 64 byte mark
Contract.Assert(m_writer.Position == SnapshotFormat.HEADER_METADATA_BYTES);
// DB Attributes
m_writer.WriteFixed32((uint)attributes.Count);
foreach (var kvp in attributes)
{
// Name
m_writer.WriteVarbytes(Slice.FromString(kvp.Key));
// Value
m_writer.WriteVarbytes(kvp.Value.ToSlice());
}
// Mark the end of the header
m_writer.WriteFixed32(uint.MaxValue);
// we now have the size of the header, and can fill in the blank
var headerEnd = m_writer.Position;
m_writer.Position = offsetToHeaderSize;
// write the header size (includes the CRC)
m_writer.WriteFixed32((uint)checked(headerEnd + SnapshotFormat.HEADER_CRC_SIZE));
m_writer.Position = headerEnd;
// now we can compute the actual CRC
uint headerChecksum = SnapshotFormat.ComputeChecksum(m_writer.ToSlice());
m_writer.WriteFixed32(headerChecksum);
m_headerChecksum = headerChecksum;
// optional padding to fill the rest of the page
PadPageIfNeeded(SnapshotFormat.PAGE_SIZE, 0xFD);
return TaskHelpers.CompletedTask;
}
public void ReadJumpTable(CancellationToken ct)
{
ct.ThrowIfCancellationRequested();
if (!m_hasHeader)
{
throw new InvalidOperationException("Cannot read the Jump Table without reading the Header first!");
}
// an empty database will have at least 2 pages: the header and the JT
if (m_file.Length < checked (m_pageSize << 1))
{
throw ParseError("File size ({0}) is too small to be a valid snapshot", m_file.Length);
}
// the jumptable is always in the last page of the file and is expected to fit nicely
// > file size MUST be evenly divible by page size
// > then JT offset will be file.Length - pageSize
if (m_file.Length % m_pageSize != 0)
{
throw ParseError("The file size ({0}) is not a multiple of the page size ({1}), which may be a symptom of truncation", m_file.Length, m_pageSize);
}
var jumpTableStart = m_file.Length - m_pageSize;
Contract.Assert(jumpTableStart % m_pageSize == 0);
m_dataEnd = jumpTableStart;
var reader = m_file.CreateReader(jumpTableStart, m_pageSize);
// "JMPT"
var signature = reader.ReadFixed32();
// Page Size (repeated)
var pageSizeRepeated = (int)reader.ReadFixed32();
// Sequence Number (repeated)
var sequenceRepeated = reader.ReadFixed64();
// Database ID (repeated)
var uidRepeated = new Uuid128(reader.ReadBytes(16).GetBytes());
// Header CRC (repeated)
var headerChecksumRepeated = reader.ReadFixed32();
// Sanity checks
if (signature != SnapshotFormat.JUMP_TABLE_MAGIC_NUMBER)
{
throw ParseError("Last page does not appear to be the Jump Table");
}
if (pageSizeRepeated != m_pageSize)
{
throw ParseError("Page size in Jump Table does not match the header value");
}
if (sequenceRepeated != m_sequence)
{
throw ParseError("Sequence in Jump Table does not match the header value");
}
if (uidRepeated != m_uid)
{
throw ParseError("Database ID in Jump Table does not match the header value");
}
if (headerChecksumRepeated != m_headerChecksum)
{
throw ParseError("Database ID in Jump Table does not match the header value");
}
// read the table itself
int levels = (int)reader.ReadFixed32();
if (levels < 0 || levels > 32)
{
throw ParseError("The number of levels in the snapshot does not appear to be valid");
}
var table = new LevelAddress[levels];
for (int level = 0; level < levels; level++)
{
ulong offset = reader.ReadFixed64();
ulong size = reader.ReadFixed64();
// Offset and Size cannot be negative
// Empty levels (size == 0) must have a zero offset
// Non empty levels (size > 0) must have a non zero offset that is greater than the headerSize
if ((size == 0 && offset != 0) || (size > 0 && offset < m_dataStart))
{
throw ParseError("Level in Jump Table has invalid size ({0}) or offset ({1})", size, offset);
}
if (checked (offset + size) > m_dataEnd)
{
throw ParseError("Level in Jump Table would end after the end of the file");
}
table[level].Offset = offset;
table[level].Size = size;
table[level].PaddedSize = RoundUp(size, m_pageSize);
}
// end attributes
uint attributeCount = reader.ReadFixed32();
if (attributeCount != 0)
{
throw new NotImplementedException("Footer attributes not yet implemented!");
}
// end marker
if (reader.ReadFixed32() != uint.MaxValue)
{
throw ParseError("Jump Table end marker not found");
}
// checksum
uint actualChecksum = SnapshotFormat.ComputeChecksum(reader.Base, reader.Offset);
uint checksum = reader.ReadFixed32();
if (actualChecksum != checksum)
{
throw ParseError("Jump Table checksum does not match ({0} != {1}). This may be an indication of data corruption", checksum, actualChecksum);
}
m_jumpTable = table;
m_levels = levels;
m_hasJumpTable = true;
}
public void ReadHeader(CancellationToken ct)
{
ct.ThrowIfCancellationRequested();
// minimum header prolog size is 64 but most will only a single page
// we can preallocate a full page, and we will resize it later if needed
var reader = m_file.CreateReader(0, SnapshotFormat.HEADER_METADATA_BYTES);
// "PNDB"
var signature = reader.ReadFixed32();
// v1.0
uint major = reader.ReadFixed16();
uint minor = reader.ReadFixed16();
m_version = new Version((int)major, (int)minor);
// FLAGS
m_dbFlags = (SnapshotFormat.Flags)reader.ReadFixed64();
// Database ID
m_uid = new Uuid128(reader.ReadBytes(16).GetBytes());
// Database Version
m_sequence = reader.ReadFixed64();
// Number of items in the database
m_itemCount = checked ((long)reader.ReadFixed64());
// Database Timestamp
m_timestamp = reader.ReadFixed64();
// Page Size
m_pageSize = reader.ReadFixed32();
// Header Size
m_headerSize = reader.ReadFixed32();
Contract.Assert(!reader.HasMore);
#region Sanity checks
// Signature
if (signature != SnapshotFormat.HEADER_MAGIC_NUMBER)
{
throw ParseError("Invalid magic number");
}
// Version
if (m_version.Major != 1)
{
throw ParseError("Unsupported file version (major)");
}
if (m_version.Minor > 0)
{
throw ParseError("Unsupported file version (minor)");
}
// Flags
// Page Size
if (m_pageSize != UnmanagedHelpers.NextPowerOfTwo(m_pageSize))
{
throw ParseError("Page size ({0}) is not a power of two", m_pageSize);
}
if (m_pageSize < SnapshotFormat.HEADER_METADATA_BYTES)
{
throw ParseError("Page size ({0}) is too small", m_pageSize);
}
if (m_pageSize > 1 << 20)
{
throw ParseError("Page size ({0}) is too big", m_pageSize);
}
// Header Size
if (m_headerSize < 64 + 4 + 4)
{
throw ParseError("Header size ({0}) is too small", m_headerSize);
}
if (m_headerSize > m_file.Length)
{
throw ParseError("Header size is bigger than the file itself ({0} < {1})", m_headerSize, m_file.Length);
}
if (m_headerSize > 1 << 10)
{
throw ParseError("Header size ({0}) exceeds the maximum allowed size", m_headerSize);
}
#endregion
// we know the page size and header size, read the rest...
// read the rest
reader = m_file.CreateReader(0, m_headerSize);
reader.Skip(SnapshotFormat.HEADER_METADATA_BYTES);
// parse the attributes
Contract.Assert(reader.Offset == SnapshotFormat.HEADER_METADATA_BYTES);
var attributeCount = checked ((int)reader.ReadFixed32());
if (attributeCount < 0 || attributeCount > 1024)
{
throw ParseError("Attributes count is invalid");
}
var attributes = new Dictionary <string, IFdbTuple>(attributeCount);
for (int i = 0; i < attributeCount; i++)
{
var name = reader.ReadVarbytes().ToSlice(); //TODO: max size ?
if (name.IsNullOrEmpty)
{
throw ParseError("Header attribute name is empty");
}
var data = reader.ReadVarbytes().ToSlice(); //TODO: max size + have a small scratch pad buffer for these ?
var value = FdbTuple.Unpack(data);
attributes.Add(name.ToUnicode(), value);
}
m_attributes = attributes;
// read the header en marker
var marker = reader.ReadFixed32();
if (marker != uint.MaxValue)
{
throw ParseError("Header end marker is invalid");
}
// verify the header checksum
uint actualHeaderChecksum = SnapshotFormat.ComputeChecksum(reader.Base, reader.Offset);
uint headerChecksum = reader.ReadFixed32();
m_headerChecksum = headerChecksum;
if (headerChecksum != actualHeaderChecksum)
{
throw ParseError("The header checksum does not match ({0} != {1}). This may be an indication of data corruption", headerChecksum, actualHeaderChecksum);
}
m_dataStart = RoundUp(m_headerSize, m_pageSize);
m_hasHeader = true;
}
/// <summary>Write the header to the file</summary>
/// <param name="headerFlags"></param>
/// <param name="uid"></param>
/// <param name="sequence"></param>
/// <param name="count"></param>
/// <param name="timestamp"></param>
/// <param name="attributes"></param>
/// <remarks>This needs to be called before writing any level to the file</remarks>
public Task WriteHeaderAsync(SnapshotFormat.Flags headerFlags, Uuid128 uid, ulong sequence, long count, long timestamp, IDictionary <string, IFdbTuple> attributes)
{
// The header will be use on ore more "pages", to simplify the job of loading / peeking at a stream content (no need for fancy buffering, just need to read 4K pages)
// > The last page is padded with 0xAAs to detect corruption.
m_uid = uid;
m_sequence = sequence;
m_itemCount = count;
m_timestamp = timestamp;
// HEADER
// - DB_HEADER (64 bytes)
// - DB ATTRIBUTES (variable size list of k/v)
// - END_MARKER + HEADER_CRC
// - PADDING (to fill last page)
// DB Header
// "PNDB"
m_writer.WriteFixed32(SnapshotFormat.HEADER_MAGIC_NUMBER);
// v1.0
m_writer.WriteFixed16(1); // major
m_writer.WriteFixed16(0); // minor
// FLAGS
m_writer.WriteFixed64((ulong)headerFlags);
// Database ID
m_writer.WriteBytes(uid.ToSlice());
// Database Version
m_writer.WriteFixed64(sequence);
// Number of items in the database
m_writer.WriteFixed64((ulong)count);
// Database Timestamp
m_writer.WriteFixed64((ulong)timestamp);
// Page Size
m_writer.WriteFixed32(SnapshotFormat.PAGE_SIZE);
// Header Size (not known yet and will be filled in later)
int offsetToHeaderSize = m_writer.Skip(4);
// we should be at the 64 byte mark
Contract.Assert(m_writer.Position == SnapshotFormat.HEADER_METADATA_BYTES);
// DB Attributes
m_writer.WriteFixed32((uint)attributes.Count);
foreach (var kvp in attributes)
{
// Name
m_writer.WriteVarbytes(Slice.FromString(kvp.Key));
// Value
m_writer.WriteVarbytes(kvp.Value.ToSlice());
}
// Mark the end of the header
m_writer.WriteFixed32(uint.MaxValue);
// we now have the size of the header, and can fill in the blank
var headerEnd = m_writer.Position;
m_writer.Position = offsetToHeaderSize;
// write the header size (includes the CRC)
m_writer.WriteFixed32((uint)checked (headerEnd + SnapshotFormat.HEADER_CRC_SIZE));
m_writer.Position = headerEnd;
// now we can compute the actual CRC
uint headerChecksum = SnapshotFormat.ComputeChecksum(m_writer.ToSlice());
m_writer.WriteFixed32(headerChecksum);
m_headerChecksum = headerChecksum;
// optional padding to fill the rest of the page
PadPageIfNeeded(SnapshotFormat.PAGE_SIZE, 0xFD);
return(TaskHelpers.CompletedTask);
}
