public void ReadHeader(CancellationToken ct)
{
ct.ThrowIfCancellationRequested();
// minimum header prolog size is 64 but most will only a single page
// we can preallocate a full page, and we will resize it later if needed
var reader = m_file.CreateReader(0, SnapshotFormat.HEADER_METADATA_BYTES);
// "PNDB"
var signature = reader.ReadFixed32();
// v1.0
uint major = reader.ReadFixed16();
uint minor = reader.ReadFixed16();
m_version = new Version((int)major, (int)minor);
// FLAGS
m_dbFlags = (SnapshotFormat.Flags) reader.ReadFixed64();
// Database ID
m_uid = new Uuid128(reader.ReadBytes(16).GetBytes());
// Database Version
m_sequence = reader.ReadFixed64();
// Number of items in the database
m_itemCount = checked((long)reader.ReadFixed64());
// Database Timestamp
m_timestamp = reader.ReadFixed64();
// Page Size
m_pageSize = reader.ReadFixed32();
// Header Size
m_headerSize = reader.ReadFixed32();
Contract.Assert(!reader.HasMore);
#region Sanity checks
// Signature
if (signature != SnapshotFormat.HEADER_MAGIC_NUMBER) throw ParseError("Invalid magic number");
// Version
if (m_version.Major != 1) throw ParseError("Unsupported file version (major)");
if (m_version.Minor > 0) throw ParseError("Unsupported file version (minor)");
// Flags
// Page Size
if (m_pageSize != UnmanagedHelpers.NextPowerOfTwo(m_pageSize)) throw ParseError("Page size ({0}) is not a power of two", m_pageSize);
if (m_pageSize < SnapshotFormat.HEADER_METADATA_BYTES) throw ParseError("Page size ({0}) is too small", m_pageSize);
if (m_pageSize > 1 << 20) throw ParseError("Page size ({0}) is too big", m_pageSize);
// Header Size
if (m_headerSize < 64 + 4 + 4) throw ParseError("Header size ({0}) is too small", m_headerSize);
if (m_headerSize > m_file.Length) throw ParseError("Header size is bigger than the file itself ({0} < {1})", m_headerSize, m_file.Length);
if (m_headerSize > 1 << 10) throw ParseError("Header size ({0}) exceeds the maximum allowed size", m_headerSize);
#endregion
// we know the page size and header size, read the rest...
// read the rest
reader = m_file.CreateReader(0, m_headerSize);
reader.Skip(SnapshotFormat.HEADER_METADATA_BYTES);
// parse the attributes
Contract.Assert(reader.Offset == SnapshotFormat.HEADER_METADATA_BYTES);
var attributeCount = checked((int)reader.ReadFixed32());
if (attributeCount < 0 || attributeCount > 1024) throw ParseError("Attributes count is invalid");
var attributes = new Dictionary<string, IFdbTuple>(attributeCount);
for (int i = 0; i < attributeCount; i++)
{
var name = reader.ReadVarbytes().ToSlice(); //TODO: max size ?
if (name.IsNullOrEmpty) throw ParseError("Header attribute name is empty");
var data = reader.ReadVarbytes().ToSlice(); //TODO: max size + have a small scratch pad buffer for these ?
var value = FdbTuple.Unpack(data);
attributes.Add(name.ToUnicode(), value);
}
m_attributes = attributes;
// read the header en marker
var marker = reader.ReadFixed32();
if (marker != uint.MaxValue) throw ParseError("Header end marker is invalid");
// verify the header checksum
uint actualHeaderChecksum = SnapshotFormat.ComputeChecksum(reader.Base, reader.Offset);
uint headerChecksum = reader.ReadFixed32();
m_headerChecksum = headerChecksum;
if (headerChecksum != actualHeaderChecksum)
{
throw ParseError("The header checksum does not match ({0} != {1}). This may be an indication of data corruption", headerChecksum, actualHeaderChecksum);
}
m_dataStart = RoundUp(m_headerSize, m_pageSize);
m_hasHeader = true;
}
public void ReadHeader(CancellationToken ct)
{
ct.ThrowIfCancellationRequested();
// minimum header prolog size is 64 but most will only a single page
// we can preallocate a full page, and we will resize it later if needed
var reader = m_file.CreateReader(0, SnapshotFormat.HEADER_METADATA_BYTES);
// "PNDB"
var signature = reader.ReadFixed32();
// v1.0
uint major = reader.ReadFixed16();
uint minor = reader.ReadFixed16();
m_version = new Version((int)major, (int)minor);
// FLAGS
m_dbFlags = (SnapshotFormat.Flags)reader.ReadFixed64();
// Database ID
m_uid = new Uuid128(reader.ReadBytes(16).GetBytes());
// Database Version
m_sequence = reader.ReadFixed64();
// Number of items in the database
m_itemCount = checked ((long)reader.ReadFixed64());
// Database Timestamp
m_timestamp = reader.ReadFixed64();
// Page Size
m_pageSize = reader.ReadFixed32();
// Header Size
m_headerSize = reader.ReadFixed32();
Contract.Assert(!reader.HasMore);
#region Sanity checks
// Signature
if (signature != SnapshotFormat.HEADER_MAGIC_NUMBER)
{
throw ParseError("Invalid magic number");
}
// Version
if (m_version.Major != 1)
{
throw ParseError("Unsupported file version (major)");
}
if (m_version.Minor > 0)
{
throw ParseError("Unsupported file version (minor)");
}
// Flags
// Page Size
if (m_pageSize != UnmanagedHelpers.NextPowerOfTwo(m_pageSize))
{
throw ParseError("Page size ({0}) is not a power of two", m_pageSize);
}
if (m_pageSize < SnapshotFormat.HEADER_METADATA_BYTES)
{
throw ParseError("Page size ({0}) is too small", m_pageSize);
}
if (m_pageSize > 1 << 20)
{
throw ParseError("Page size ({0}) is too big", m_pageSize);
}
// Header Size
if (m_headerSize < 64 + 4 + 4)
{
throw ParseError("Header size ({0}) is too small", m_headerSize);
}
if (m_headerSize > m_file.Length)
{
throw ParseError("Header size is bigger than the file itself ({0} < {1})", m_headerSize, m_file.Length);
}
if (m_headerSize > 1 << 10)
{
throw ParseError("Header size ({0}) exceeds the maximum allowed size", m_headerSize);
}
#endregion
// we know the page size and header size, read the rest...
// read the rest
reader = m_file.CreateReader(0, m_headerSize);
reader.Skip(SnapshotFormat.HEADER_METADATA_BYTES);
// parse the attributes
Contract.Assert(reader.Offset == SnapshotFormat.HEADER_METADATA_BYTES);
var attributeCount = checked ((int)reader.ReadFixed32());
if (attributeCount < 0 || attributeCount > 1024)
{
throw ParseError("Attributes count is invalid");
}
var attributes = new Dictionary <string, IFdbTuple>(attributeCount);
for (int i = 0; i < attributeCount; i++)
{
var name = reader.ReadVarbytes().ToSlice(); //TODO: max size ?
if (name.IsNullOrEmpty)
{
throw ParseError("Header attribute name is empty");
}
var data = reader.ReadVarbytes().ToSlice(); //TODO: max size + have a small scratch pad buffer for these ?
var value = FdbTuple.Unpack(data);
attributes.Add(name.ToUnicode(), value);
}
m_attributes = attributes;
// read the header en marker
var marker = reader.ReadFixed32();
if (marker != uint.MaxValue)
{
throw ParseError("Header end marker is invalid");
}
// verify the header checksum
uint actualHeaderChecksum = SnapshotFormat.ComputeChecksum(reader.Base, reader.Offset);
uint headerChecksum = reader.ReadFixed32();
m_headerChecksum = headerChecksum;
if (headerChecksum != actualHeaderChecksum)
{
throw ParseError("The header checksum does not match ({0} != {1}). This may be an indication of data corruption", headerChecksum, actualHeaderChecksum);
}
m_dataStart = RoundUp(m_headerSize, m_pageSize);
m_hasHeader = true;
}
/// <summary>Write the header to the file</summary>
/// <param name="headerFlags"></param>
/// <param name="uid"></param>
/// <param name="sequence"></param>
/// <param name="count"></param>
/// <param name="timestamp"></param>
/// <param name="attributes"></param>
/// <remarks>This needs to be called before writing any level to the file</remarks>
public Task WriteHeaderAsync(SnapshotFormat.Flags headerFlags, Uuid128 uid, ulong sequence, long count, long timestamp, IDictionary <string, IFdbTuple> attributes)
{
// The header will be use on ore more "pages", to simplify the job of loading / peeking at a stream content (no need for fancy buffering, just need to read 4K pages)
// > The last page is padded with 0xAAs to detect corruption.
m_uid = uid;
m_sequence = sequence;
m_itemCount = count;
m_timestamp = timestamp;
// HEADER
// - DB_HEADER (64 bytes)
// - DB ATTRIBUTES (variable size list of k/v)
// - END_MARKER + HEADER_CRC
// - PADDING (to fill last page)
// DB Header
// "PNDB"
m_writer.WriteFixed32(SnapshotFormat.HEADER_MAGIC_NUMBER);
// v1.0
m_writer.WriteFixed16(1); // major
m_writer.WriteFixed16(0); // minor
// FLAGS
m_writer.WriteFixed64((ulong)headerFlags);
// Database ID
m_writer.WriteBytes(uid.ToSlice());
// Database Version
m_writer.WriteFixed64(sequence);
// Number of items in the database
m_writer.WriteFixed64((ulong)count);
// Database Timestamp
m_writer.WriteFixed64((ulong)timestamp);
// Page Size
m_writer.WriteFixed32(SnapshotFormat.PAGE_SIZE);
// Header Size (not known yet and will be filled in later)
int offsetToHeaderSize = m_writer.Skip(4);
// we should be at the 64 byte mark
Contract.Assert(m_writer.Position == SnapshotFormat.HEADER_METADATA_BYTES);
// DB Attributes
m_writer.WriteFixed32((uint)attributes.Count);
foreach (var kvp in attributes)
{
// Name
m_writer.WriteVarbytes(Slice.FromString(kvp.Key));
// Value
m_writer.WriteVarbytes(kvp.Value.ToSlice());
}
// Mark the end of the header
m_writer.WriteFixed32(uint.MaxValue);
// we now have the size of the header, and can fill in the blank
var headerEnd = m_writer.Position;
m_writer.Position = offsetToHeaderSize;
// write the header size (includes the CRC)
m_writer.WriteFixed32((uint)checked (headerEnd + SnapshotFormat.HEADER_CRC_SIZE));
m_writer.Position = headerEnd;
// now we can compute the actual CRC
uint headerChecksum = SnapshotFormat.ComputeChecksum(m_writer.ToSlice());
m_writer.WriteFixed32(headerChecksum);
m_headerChecksum = headerChecksum;
// optional padding to fill the rest of the page
PadPageIfNeeded(SnapshotFormat.PAGE_SIZE, 0xFD);
return(TaskHelpers.CompletedTask);
}
