public void ReadHeader(CancellationToken ct) { ct.ThrowIfCancellationRequested(); // minimum header prolog size is 64 but most will only a single page // we can preallocate a full page, and we will resize it later if needed var reader = m_file.CreateReader(0, SnapshotFormat.HEADER_METADATA_BYTES); // "PNDB" var signature = reader.ReadFixed32(); // v1.0 uint major = reader.ReadFixed16(); uint minor = reader.ReadFixed16(); m_version = new Version((int)major, (int)minor); // FLAGS m_dbFlags = (SnapshotFormat.Flags) reader.ReadFixed64(); // Database ID m_uid = new Uuid128(reader.ReadBytes(16).GetBytes()); // Database Version m_sequence = reader.ReadFixed64(); // Number of items in the database m_itemCount = checked((long)reader.ReadFixed64()); // Database Timestamp m_timestamp = reader.ReadFixed64(); // Page Size m_pageSize = reader.ReadFixed32(); // Header Size m_headerSize = reader.ReadFixed32(); Contract.Assert(!reader.HasMore); #region Sanity checks // Signature if (signature != SnapshotFormat.HEADER_MAGIC_NUMBER) throw ParseError("Invalid magic number"); // Version if (m_version.Major != 1) throw ParseError("Unsupported file version (major)"); if (m_version.Minor > 0) throw ParseError("Unsupported file version (minor)"); // Flags // Page Size if (m_pageSize != UnmanagedHelpers.NextPowerOfTwo(m_pageSize)) throw ParseError("Page size ({0}) is not a power of two", m_pageSize); if (m_pageSize < SnapshotFormat.HEADER_METADATA_BYTES) throw ParseError("Page size ({0}) is too small", m_pageSize); if (m_pageSize > 1 << 20) throw ParseError("Page size ({0}) is too big", m_pageSize); // Header Size if (m_headerSize < 64 + 4 + 4) throw ParseError("Header size ({0}) is too small", m_headerSize); if (m_headerSize > m_file.Length) throw ParseError("Header size is bigger than the file itself ({0} < {1})", m_headerSize, m_file.Length); if (m_headerSize > 1 << 10) throw ParseError("Header size ({0}) exceeds the maximum allowed size", m_headerSize); #endregion // we know the page size and header size, read the rest... // read the rest reader = m_file.CreateReader(0, m_headerSize); reader.Skip(SnapshotFormat.HEADER_METADATA_BYTES); // parse the attributes Contract.Assert(reader.Offset == SnapshotFormat.HEADER_METADATA_BYTES); var attributeCount = checked((int)reader.ReadFixed32()); if (attributeCount < 0 || attributeCount > 1024) throw ParseError("Attributes count is invalid"); var attributes = new Dictionary<string, IFdbTuple>(attributeCount); for (int i = 0; i < attributeCount; i++) { var name = reader.ReadVarbytes().ToSlice(); //TODO: max size ? if (name.IsNullOrEmpty) throw ParseError("Header attribute name is empty"); var data = reader.ReadVarbytes().ToSlice(); //TODO: max size + have a small scratch pad buffer for these ? var value = FdbTuple.Unpack(data); attributes.Add(name.ToUnicode(), value); } m_attributes = attributes; // read the header en marker var marker = reader.ReadFixed32(); if (marker != uint.MaxValue) throw ParseError("Header end marker is invalid"); // verify the header checksum uint actualHeaderChecksum = SnapshotFormat.ComputeChecksum(reader.Base, reader.Offset); uint headerChecksum = reader.ReadFixed32(); m_headerChecksum = headerChecksum; if (headerChecksum != actualHeaderChecksum) { throw ParseError("The header checksum does not match ({0} != {1}). This may be an indication of data corruption", headerChecksum, actualHeaderChecksum); } m_dataStart = RoundUp(m_headerSize, m_pageSize); m_hasHeader = true; }
public void ReadHeader(CancellationToken ct) { ct.ThrowIfCancellationRequested(); // minimum header prolog size is 64 but most will only a single page // we can preallocate a full page, and we will resize it later if needed var reader = m_file.CreateReader(0, SnapshotFormat.HEADER_METADATA_BYTES); // "PNDB" var signature = reader.ReadFixed32(); // v1.0 uint major = reader.ReadFixed16(); uint minor = reader.ReadFixed16(); m_version = new Version((int)major, (int)minor); // FLAGS m_dbFlags = (SnapshotFormat.Flags)reader.ReadFixed64(); // Database ID m_uid = new Uuid128(reader.ReadBytes(16).GetBytes()); // Database Version m_sequence = reader.ReadFixed64(); // Number of items in the database m_itemCount = checked ((long)reader.ReadFixed64()); // Database Timestamp m_timestamp = reader.ReadFixed64(); // Page Size m_pageSize = reader.ReadFixed32(); // Header Size m_headerSize = reader.ReadFixed32(); Contract.Assert(!reader.HasMore); #region Sanity checks // Signature if (signature != SnapshotFormat.HEADER_MAGIC_NUMBER) { throw ParseError("Invalid magic number"); } // Version if (m_version.Major != 1) { throw ParseError("Unsupported file version (major)"); } if (m_version.Minor > 0) { throw ParseError("Unsupported file version (minor)"); } // Flags // Page Size if (m_pageSize != UnmanagedHelpers.NextPowerOfTwo(m_pageSize)) { throw ParseError("Page size ({0}) is not a power of two", m_pageSize); } if (m_pageSize < SnapshotFormat.HEADER_METADATA_BYTES) { throw ParseError("Page size ({0}) is too small", m_pageSize); } if (m_pageSize > 1 << 20) { throw ParseError("Page size ({0}) is too big", m_pageSize); } // Header Size if (m_headerSize < 64 + 4 + 4) { throw ParseError("Header size ({0}) is too small", m_headerSize); } if (m_headerSize > m_file.Length) { throw ParseError("Header size is bigger than the file itself ({0} < {1})", m_headerSize, m_file.Length); } if (m_headerSize > 1 << 10) { throw ParseError("Header size ({0}) exceeds the maximum allowed size", m_headerSize); } #endregion // we know the page size and header size, read the rest... // read the rest reader = m_file.CreateReader(0, m_headerSize); reader.Skip(SnapshotFormat.HEADER_METADATA_BYTES); // parse the attributes Contract.Assert(reader.Offset == SnapshotFormat.HEADER_METADATA_BYTES); var attributeCount = checked ((int)reader.ReadFixed32()); if (attributeCount < 0 || attributeCount > 1024) { throw ParseError("Attributes count is invalid"); } var attributes = new Dictionary <string, IFdbTuple>(attributeCount); for (int i = 0; i < attributeCount; i++) { var name = reader.ReadVarbytes().ToSlice(); //TODO: max size ? if (name.IsNullOrEmpty) { throw ParseError("Header attribute name is empty"); } var data = reader.ReadVarbytes().ToSlice(); //TODO: max size + have a small scratch pad buffer for these ? var value = FdbTuple.Unpack(data); attributes.Add(name.ToUnicode(), value); } m_attributes = attributes; // read the header en marker var marker = reader.ReadFixed32(); if (marker != uint.MaxValue) { throw ParseError("Header end marker is invalid"); } // verify the header checksum uint actualHeaderChecksum = SnapshotFormat.ComputeChecksum(reader.Base, reader.Offset); uint headerChecksum = reader.ReadFixed32(); m_headerChecksum = headerChecksum; if (headerChecksum != actualHeaderChecksum) { throw ParseError("The header checksum does not match ({0} != {1}). This may be an indication of data corruption", headerChecksum, actualHeaderChecksum); } m_dataStart = RoundUp(m_headerSize, m_pageSize); m_hasHeader = true; }
/// <summary>Write the header to the file</summary> /// <param name="headerFlags"></param> /// <param name="uid"></param> /// <param name="sequence"></param> /// <param name="count"></param> /// <param name="timestamp"></param> /// <param name="attributes"></param> /// <remarks>This needs to be called before writing any level to the file</remarks> public Task WriteHeaderAsync(SnapshotFormat.Flags headerFlags, Uuid128 uid, ulong sequence, long count, long timestamp, IDictionary <string, IFdbTuple> attributes) { // The header will be use on ore more "pages", to simplify the job of loading / peeking at a stream content (no need for fancy buffering, just need to read 4K pages) // > The last page is padded with 0xAAs to detect corruption. m_uid = uid; m_sequence = sequence; m_itemCount = count; m_timestamp = timestamp; // HEADER // - DB_HEADER (64 bytes) // - DB ATTRIBUTES (variable size list of k/v) // - END_MARKER + HEADER_CRC // - PADDING (to fill last page) // DB Header // "PNDB" m_writer.WriteFixed32(SnapshotFormat.HEADER_MAGIC_NUMBER); // v1.0 m_writer.WriteFixed16(1); // major m_writer.WriteFixed16(0); // minor // FLAGS m_writer.WriteFixed64((ulong)headerFlags); // Database ID m_writer.WriteBytes(uid.ToSlice()); // Database Version m_writer.WriteFixed64(sequence); // Number of items in the database m_writer.WriteFixed64((ulong)count); // Database Timestamp m_writer.WriteFixed64((ulong)timestamp); // Page Size m_writer.WriteFixed32(SnapshotFormat.PAGE_SIZE); // Header Size (not known yet and will be filled in later) int offsetToHeaderSize = m_writer.Skip(4); // we should be at the 64 byte mark Contract.Assert(m_writer.Position == SnapshotFormat.HEADER_METADATA_BYTES); // DB Attributes m_writer.WriteFixed32((uint)attributes.Count); foreach (var kvp in attributes) { // Name m_writer.WriteVarbytes(Slice.FromString(kvp.Key)); // Value m_writer.WriteVarbytes(kvp.Value.ToSlice()); } // Mark the end of the header m_writer.WriteFixed32(uint.MaxValue); // we now have the size of the header, and can fill in the blank var headerEnd = m_writer.Position; m_writer.Position = offsetToHeaderSize; // write the header size (includes the CRC) m_writer.WriteFixed32((uint)checked (headerEnd + SnapshotFormat.HEADER_CRC_SIZE)); m_writer.Position = headerEnd; // now we can compute the actual CRC uint headerChecksum = SnapshotFormat.ComputeChecksum(m_writer.ToSlice()); m_writer.WriteFixed32(headerChecksum); m_headerChecksum = headerChecksum; // optional padding to fill the rest of the page PadPageIfNeeded(SnapshotFormat.PAGE_SIZE, 0xFD); return(TaskHelpers.CompletedTask); }