public Task WriteJumpTableAsync(CancellationToken ct)
        {
            ct.ThrowIfCancellationRequested();

            // The jump table is the last page of the file
            // - it contains the list of (offset, size) of all the levels that are in the file
            // - it contains any additional attributes (that were only known after writing all the data)
            // - it repeats a few important values (sequence, header crc, ...)
            // - it would contain any optional signature or data that is only know after writing the data to disk, and are needed to decode the rest

            // marks the start of the JT because we will need to compute the checksum later on
            int startOffset = m_writer.Position;

            // "JMPT"
            m_writer.WriteFixed32(SnapshotFormat.JUMP_TABLE_MAGIC_NUMBER);
            // Page Size (repeated)
            m_writer.WriteFixed32((uint)m_pageSize);
            // Sequence Number (repeated)
            m_writer.WriteFixed64(m_sequence);
            // Database ID (repeated)
            m_writer.WriteBytes(m_uid.ToSlice());
            // Header CRC (repeated)
            m_writer.WriteFixed32(m_headerChecksum);

            int levels = m_levels;

            m_writer.WriteFixed32((uint)levels);                                // Level Count
            for (int level = 0; level < levels; level++)
            {
                // Level Offset (from start of file)
                m_writer.WriteFixed64((ulong)m_jumpTable[level].Key);
                // Level Size (in bytes)
                m_writer.WriteFixed64((ulong)m_jumpTable[level].Value);
            }

            //TODO: additional attributes!
            m_writer.WriteFixed32(0);             // 0 for now

            // End Marker
            m_writer.WriteFixed32(uint.MaxValue);

            // Checksum
            int  endOffset         = m_writer.Position;
            uint jumpTableChecksum = SnapshotFormat.ComputeChecksum(m_writer[startOffset, endOffset]);

            m_writer.WriteFixed32(jumpTableChecksum);

            // optional padding to fill the rest of the page
            PadPageIfNeeded(SnapshotFormat.PAGE_SIZE, 0xFE);

            // we are done !
            return(TaskHelpers.CompletedTask);
        }
		/// <summary>Write the header to the file</summary>
		/// <param name="headerFlags"></param>
		/// <param name="uid"></param>
		/// <param name="sequence"></param>
		/// <param name="count"></param>
		/// <param name="timestamp"></param>
		/// <param name="attributes"></param>
		/// <remarks>This needs to be called before writing any level to the file</remarks>
		public Task WriteHeaderAsync(SnapshotFormat.Flags headerFlags, Uuid128 uid, ulong sequence, long count, long timestamp, IDictionary<string, IFdbTuple> attributes)
		{
			// The header will be use on ore more "pages", to simplify the job of loading / peeking at a stream content (no need for fancy buffering, just need to read 4K pages)
			// > The last page is padded with 0xAAs to detect corruption.

			m_uid = uid;
			m_sequence = sequence;
			m_itemCount = count;
			m_timestamp = timestamp;

			// HEADER
			// - DB_HEADER (64 bytes)
			// - DB ATTRIBUTES (variable size list of k/v)
			// - END_MARKER + HEADER_CRC
			// - PADDING (to fill last page)

			// DB Header

			// "PNDB"
			m_writer.WriteFixed32(SnapshotFormat.HEADER_MAGIC_NUMBER);
			// v1.0
			m_writer.WriteFixed16(1); // major
			m_writer.WriteFixed16(0); // minor
			// FLAGS
			m_writer.WriteFixed64((ulong)headerFlags);
			// Database ID
			m_writer.WriteBytes(uid.ToSlice());
			// Database Version
			m_writer.WriteFixed64(sequence);
			// Number of items in the database
			m_writer.WriteFixed64((ulong)count);
			// Database Timestamp
			m_writer.WriteFixed64((ulong)timestamp);
			// Page Size
			m_writer.WriteFixed32(SnapshotFormat.PAGE_SIZE);
			// Header Size (not known yet and will be filled in later)
			int offsetToHeaderSize = m_writer.Skip(4);

			// we should be at the 64 byte mark
			Contract.Assert(m_writer.Position == SnapshotFormat.HEADER_METADATA_BYTES);

			// DB Attributes
			m_writer.WriteFixed32((uint)attributes.Count);
			foreach (var kvp in attributes)
			{
				// Name
				m_writer.WriteVarbytes(Slice.FromString(kvp.Key));

				// Value
				m_writer.WriteVarbytes(kvp.Value.ToSlice());
			}

			// Mark the end of the header
			m_writer.WriteFixed32(uint.MaxValue);

			// we now have the size of the header, and can fill in the blank
			var headerEnd = m_writer.Position;
			m_writer.Position = offsetToHeaderSize;
			// write the header size (includes the CRC)
			m_writer.WriteFixed32((uint)checked(headerEnd + SnapshotFormat.HEADER_CRC_SIZE));
			m_writer.Position = headerEnd;

			// now we can compute the actual CRC
			uint headerChecksum = SnapshotFormat.ComputeChecksum(m_writer.ToSlice());
			m_writer.WriteFixed32(headerChecksum);
			m_headerChecksum = headerChecksum;

			// optional padding to fill the rest of the page
			PadPageIfNeeded(SnapshotFormat.PAGE_SIZE, 0xFD);

			return TaskHelpers.CompletedTask;
		}
        public void ReadJumpTable(CancellationToken ct)
        {
            ct.ThrowIfCancellationRequested();

            if (!m_hasHeader)
            {
                throw new InvalidOperationException("Cannot read the Jump Table without reading the Header first!");
            }

            // an empty database will have at least 2 pages: the header and the JT
            if (m_file.Length < checked (m_pageSize << 1))
            {
                throw ParseError("File size ({0}) is too small to be a valid snapshot", m_file.Length);
            }

            // the jumptable is always in the last page of the file and is expected to fit nicely
            // > file size MUST be evenly divible by page size
            // > then JT offset will be file.Length - pageSize
            if (m_file.Length % m_pageSize != 0)
            {
                throw ParseError("The file size ({0}) is not a multiple of the page size ({1}), which may be a symptom of truncation", m_file.Length, m_pageSize);
            }

            var jumpTableStart = m_file.Length - m_pageSize;

            Contract.Assert(jumpTableStart % m_pageSize == 0);
            m_dataEnd = jumpTableStart;

            var reader = m_file.CreateReader(jumpTableStart, m_pageSize);

            // "JMPT"
            var signature = reader.ReadFixed32();
            // Page Size (repeated)
            var pageSizeRepeated = (int)reader.ReadFixed32();
            // Sequence Number (repeated)
            var sequenceRepeated = reader.ReadFixed64();
            // Database ID (repeated)
            var uidRepeated = new Uuid128(reader.ReadBytes(16).GetBytes());
            // Header CRC (repeated)
            var headerChecksumRepeated = reader.ReadFixed32();

            // Sanity checks

            if (signature != SnapshotFormat.JUMP_TABLE_MAGIC_NUMBER)
            {
                throw ParseError("Last page does not appear to be the Jump Table");
            }
            if (pageSizeRepeated != m_pageSize)
            {
                throw ParseError("Page size in Jump Table does not match the header value");
            }
            if (sequenceRepeated != m_sequence)
            {
                throw ParseError("Sequence in Jump Table does not match the header value");
            }
            if (uidRepeated != m_uid)
            {
                throw ParseError("Database ID in Jump Table does not match the header value");
            }
            if (headerChecksumRepeated != m_headerChecksum)
            {
                throw ParseError("Database ID in Jump Table does not match the header value");
            }

            // read the table itself
            int levels = (int)reader.ReadFixed32();

            if (levels < 0 || levels > 32)
            {
                throw ParseError("The number of levels in the snapshot does not appear to be valid");
            }

            var table = new LevelAddress[levels];

            for (int level = 0; level < levels; level++)
            {
                ulong offset = reader.ReadFixed64();
                ulong size   = reader.ReadFixed64();

                // Offset and Size cannot be negative
                // Empty levels (size == 0) must have a zero offset
                // Non empty levels (size > 0) must have a non zero offset that is greater than the headerSize
                if ((size == 0 && offset != 0) || (size > 0 && offset < m_dataStart))
                {
                    throw ParseError("Level in Jump Table has invalid size ({0}) or offset ({1})", size, offset);
                }
                if (checked (offset + size) > m_dataEnd)
                {
                    throw ParseError("Level in Jump Table would end after the end of the file");
                }

                table[level].Offset     = offset;
                table[level].Size       = size;
                table[level].PaddedSize = RoundUp(size, m_pageSize);
            }

            // end attributes
            uint attributeCount = reader.ReadFixed32();

            if (attributeCount != 0)
            {
                throw new NotImplementedException("Footer attributes not yet implemented!");
            }

            // end marker
            if (reader.ReadFixed32() != uint.MaxValue)
            {
                throw ParseError("Jump Table end marker not found");
            }

            // checksum
            uint actualChecksum = SnapshotFormat.ComputeChecksum(reader.Base, reader.Offset);
            uint checksum       = reader.ReadFixed32();

            if (actualChecksum != checksum)
            {
                throw ParseError("Jump Table checksum does not match ({0} != {1}). This may be an indication of data corruption", checksum, actualChecksum);
            }

            m_jumpTable    = table;
            m_levels       = levels;
            m_hasJumpTable = true;
        }
        public void ReadHeader(CancellationToken ct)
        {
            ct.ThrowIfCancellationRequested();

            // minimum header prolog size is 64 but most will only a single page
            // we can preallocate a full page, and we will resize it later if needed

            var reader = m_file.CreateReader(0, SnapshotFormat.HEADER_METADATA_BYTES);

            // "PNDB"
            var signature = reader.ReadFixed32();
            // v1.0
            uint major = reader.ReadFixed16();
            uint minor = reader.ReadFixed16();

            m_version = new Version((int)major, (int)minor);
            // FLAGS
            m_dbFlags = (SnapshotFormat.Flags)reader.ReadFixed64();
            // Database ID
            m_uid = new Uuid128(reader.ReadBytes(16).GetBytes());
            // Database Version
            m_sequence = reader.ReadFixed64();
            // Number of items in the database
            m_itemCount = checked ((long)reader.ReadFixed64());
            // Database Timestamp
            m_timestamp = reader.ReadFixed64();
            // Page Size
            m_pageSize = reader.ReadFixed32();
            // Header Size
            m_headerSize = reader.ReadFixed32();

            Contract.Assert(!reader.HasMore);

            #region Sanity checks

            // Signature
            if (signature != SnapshotFormat.HEADER_MAGIC_NUMBER)
            {
                throw ParseError("Invalid magic number");
            }

            // Version
            if (m_version.Major != 1)
            {
                throw ParseError("Unsupported file version (major)");
            }
            if (m_version.Minor > 0)
            {
                throw ParseError("Unsupported file version (minor)");
            }

            // Flags

            // Page Size
            if (m_pageSize != UnmanagedHelpers.NextPowerOfTwo(m_pageSize))
            {
                throw ParseError("Page size ({0}) is not a power of two", m_pageSize);
            }
            if (m_pageSize < SnapshotFormat.HEADER_METADATA_BYTES)
            {
                throw ParseError("Page size ({0}) is too small", m_pageSize);
            }
            if (m_pageSize > 1 << 20)
            {
                throw ParseError("Page size ({0}) is too big", m_pageSize);
            }

            // Header Size
            if (m_headerSize < 64 + 4 + 4)
            {
                throw ParseError("Header size ({0}) is too small", m_headerSize);
            }
            if (m_headerSize > m_file.Length)
            {
                throw ParseError("Header size is bigger than the file itself ({0} < {1})", m_headerSize, m_file.Length);
            }
            if (m_headerSize > 1 << 10)
            {
                throw ParseError("Header size ({0}) exceeds the maximum allowed size", m_headerSize);
            }

            #endregion

            // we know the page size and header size, read the rest...

            // read the rest
            reader = m_file.CreateReader(0, m_headerSize);
            reader.Skip(SnapshotFormat.HEADER_METADATA_BYTES);

            // parse the attributes
            Contract.Assert(reader.Offset == SnapshotFormat.HEADER_METADATA_BYTES);
            var attributeCount = checked ((int)reader.ReadFixed32());
            if (attributeCount < 0 || attributeCount > 1024)
            {
                throw ParseError("Attributes count is invalid");
            }

            var attributes = new Dictionary <string, IFdbTuple>(attributeCount);
            for (int i = 0; i < attributeCount; i++)
            {
                var name = reader.ReadVarbytes().ToSlice();                 //TODO: max size ?
                if (name.IsNullOrEmpty)
                {
                    throw ParseError("Header attribute name is empty");
                }

                var data  = reader.ReadVarbytes().ToSlice();                    //TODO: max size + have a small scratch pad buffer for these ?
                var value = FdbTuple.Unpack(data);
                attributes.Add(name.ToUnicode(), value);
            }
            m_attributes = attributes;

            // read the header en marker
            var marker = reader.ReadFixed32();
            if (marker != uint.MaxValue)
            {
                throw ParseError("Header end marker is invalid");
            }

            // verify the header checksum
            uint actualHeaderChecksum = SnapshotFormat.ComputeChecksum(reader.Base, reader.Offset);
            uint headerChecksum       = reader.ReadFixed32();
            m_headerChecksum = headerChecksum;

            if (headerChecksum != actualHeaderChecksum)
            {
                throw ParseError("The header checksum does not match ({0} != {1}). This may be an indication of data corruption", headerChecksum, actualHeaderChecksum);
            }

            m_dataStart = RoundUp(m_headerSize, m_pageSize);
            m_hasHeader = true;
        }
        /// <summary>Write the header to the file</summary>
        /// <param name="headerFlags"></param>
        /// <param name="uid"></param>
        /// <param name="sequence"></param>
        /// <param name="count"></param>
        /// <param name="timestamp"></param>
        /// <param name="attributes"></param>
        /// <remarks>This needs to be called before writing any level to the file</remarks>
        public Task WriteHeaderAsync(SnapshotFormat.Flags headerFlags, Uuid128 uid, ulong sequence, long count, long timestamp, IDictionary <string, IFdbTuple> attributes)
        {
            // The header will be use on ore more "pages", to simplify the job of loading / peeking at a stream content (no need for fancy buffering, just need to read 4K pages)
            // > The last page is padded with 0xAAs to detect corruption.

            m_uid       = uid;
            m_sequence  = sequence;
            m_itemCount = count;
            m_timestamp = timestamp;

            // HEADER
            // - DB_HEADER (64 bytes)
            // - DB ATTRIBUTES (variable size list of k/v)
            // - END_MARKER + HEADER_CRC
            // - PADDING (to fill last page)

            // DB Header

            // "PNDB"
            m_writer.WriteFixed32(SnapshotFormat.HEADER_MAGIC_NUMBER);
            // v1.0
            m_writer.WriteFixed16(1);             // major
            m_writer.WriteFixed16(0);             // minor
            // FLAGS
            m_writer.WriteFixed64((ulong)headerFlags);
            // Database ID
            m_writer.WriteBytes(uid.ToSlice());
            // Database Version
            m_writer.WriteFixed64(sequence);
            // Number of items in the database
            m_writer.WriteFixed64((ulong)count);
            // Database Timestamp
            m_writer.WriteFixed64((ulong)timestamp);
            // Page Size
            m_writer.WriteFixed32(SnapshotFormat.PAGE_SIZE);
            // Header Size (not known yet and will be filled in later)
            int offsetToHeaderSize = m_writer.Skip(4);

            // we should be at the 64 byte mark
            Contract.Assert(m_writer.Position == SnapshotFormat.HEADER_METADATA_BYTES);

            // DB Attributes
            m_writer.WriteFixed32((uint)attributes.Count);
            foreach (var kvp in attributes)
            {
                // Name
                m_writer.WriteVarbytes(Slice.FromString(kvp.Key));

                // Value
                m_writer.WriteVarbytes(kvp.Value.ToSlice());
            }

            // Mark the end of the header
            m_writer.WriteFixed32(uint.MaxValue);

            // we now have the size of the header, and can fill in the blank
            var headerEnd = m_writer.Position;

            m_writer.Position = offsetToHeaderSize;
            // write the header size (includes the CRC)
            m_writer.WriteFixed32((uint)checked (headerEnd + SnapshotFormat.HEADER_CRC_SIZE));
            m_writer.Position = headerEnd;

            // now we can compute the actual CRC
            uint headerChecksum = SnapshotFormat.ComputeChecksum(m_writer.ToSlice());

            m_writer.WriteFixed32(headerChecksum);
            m_headerChecksum = headerChecksum;

            // optional padding to fill the rest of the page
            PadPageIfNeeded(SnapshotFormat.PAGE_SIZE, 0xFD);

            return(TaskHelpers.CompletedTask);
        }