Ejemplo n.º 1
0
        internal void WriteHybridLogIncrementalMetaInfo(DeltaLog deltaLog)
        {
            var metadata = _hybridLogCheckpoint.info.ToByteArray();

            if (CommitCookie != null && CommitCookie.Length != 0)
            {
                var convertedCookie = Convert.ToBase64String(CommitCookie);
                metadata = metadata.Concat(Encoding.Default.GetBytes(convertedCookie)).ToArray();
            }
            checkpointManager.CommitLogIncrementalCheckpoint(_hybridLogCheckpointToken, _hybridLogCheckpoint.info.version, metadata, deltaLog);
        }
        /// <inheritdoc />
        public byte[] GetLogCheckpointMetadata(Guid logToken, DeltaLog deltaLog)
        {
            byte[] metadata = null;
            if (deltaLog != null)
            {
                // Try to get latest valid metadata from delta-log
                deltaLog.Reset();
                while (deltaLog.GetNext(out long physicalAddress, out int entryLength, out int type))
                {
                    if (type != 1)
                    {
                        continue;            // consider only metadata records
                    }
                    long endAddress = physicalAddress + entryLength;
                    metadata = new byte[entryLength];
                    unsafe
                    {
                        fixed(byte *m = metadata)
                        Buffer.MemoryCopy((void *)physicalAddress, m, entryLength, entryLength);
                    }
                }
                if (metadata != null)
                {
                    return(metadata);
                }
            }

            var device = deviceFactory.Get(checkpointNamingScheme.LogCheckpointMetadata(logToken));

            ReadInto(device, 0, out byte[] writePad, sizeof(int));
            int size = BitConverter.ToInt32(writePad, 0);

            byte[] body;
            if (writePad.Length >= size + sizeof(int))
            {
                body = writePad;
            }
            else
            {
                ReadInto(device, 0, out body, size + sizeof(int));
            }
            device.Dispose();
            return(new Span <byte>(body).Slice(sizeof(int)).ToArray());
        }
        /// <summary>
        /// Retrieve commit metadata for specified log checkpoint
        /// </summary>
        /// <param name="logToken">Token</param>
        /// <param name="deltaLog">Delta log</param>
        /// <returns>Metadata, or null if invalid</returns>
        public byte[] GetLogCheckpointMetadata(Guid logToken, DeltaLog deltaLog)
        {
            byte[] metadata = null;
            if (deltaLog != null)
            {
                // Get latest valid metadata from delta-log
                deltaLog.Reset();
                while (deltaLog.GetNext(out long physicalAddress, out int entryLength, out int type))
                {
                    if (type != 1)
                    {
                        continue;            // consider only metadata records
                    }
                    long endAddress = physicalAddress + entryLength;
                    metadata = new byte[entryLength];
                    unsafe
                    {
                        fixed(byte *m = metadata)
                        {
                            Buffer.MemoryCopy((void *)physicalAddress, m, entryLength, entryLength);
                        }
                    }
                }
                if (metadata != null)
                {
                    return(metadata);
                }
            }

            var dir = new DirectoryInfo(directoryConfiguration.GetHybridLogCheckpointFolder(logToken));

            if (!File.Exists(dir.FullName + Path.DirectorySeparatorChar + "completed.dat"))
            {
                return(null);
            }

            string checkpointInfoFile = directoryConfiguration.GetHybridLogCheckpointMetaFileName(logToken);

            using (var reader = new BinaryReader(new FileStream(checkpointInfoFile, FileMode.Open)))
            {
                var len = reader.ReadInt32();
                return(reader.ReadBytes(len));
            }
        }
Ejemplo n.º 4
0
        /// <inheritdoc />
        public unsafe void CommitLogIncrementalCheckpoint(Guid logToken, int version, byte[] commitMetadata, DeltaLog deltaLog)
        {
            deltaLog.Allocate(out int length, out long physicalAddress);
            if (length < commitMetadata.Length)
            {
                deltaLog.Seal(0, DeltaLogEntryType.CHECKPOINT_METADATA);
                deltaLog.Allocate(out length, out physicalAddress);
                if (length < commitMetadata.Length)
                {
                    deltaLog.Seal(0);
                    throw new Exception($"Metadata of size {commitMetadata.Length} does not fit in delta log space of size {length}");
                }
            }

            fixed(byte *ptr = commitMetadata)
            {
                Buffer.MemoryCopy(ptr, (void *)physicalAddress, commitMetadata.Length, commitMetadata.Length);
            }

            deltaLog.Seal(commitMetadata.Length, DeltaLogEntryType.CHECKPOINT_METADATA);
            deltaLog.FlushAsync().Wait();
        }
Ejemplo n.º 5
0
        /// <summary>
        /// Retrieve commit metadata for specified log checkpoint
        /// </summary>
        /// <param name="logToken">Token</param>
        /// <param name="deltaLog">Delta log</param>
        /// <param name="scanDelta"> whether or not to scan through the delta log to acquire latest entry</param>
        /// <param name="recoverTo"> version upper bound to scan for in the delta log. Function will return the largest version metadata no greater than the given version.</param>
        /// <returns>Metadata, or null if invalid</returns>
        public byte[] GetLogCheckpointMetadata(Guid logToken, DeltaLog deltaLog, bool scanDelta, long recoverTo)
        {
            byte[] metadata = null;
            if (scanDelta && deltaLog != null)
            {
                // Get latest valid metadata from delta-log
                deltaLog.Reset();
                while (deltaLog.GetNext(out long physicalAddress, out int entryLength, out DeltaLogEntryType type))
                {
                    switch (type)
                    {
                    case DeltaLogEntryType.DELTA:
                        // consider only metadata records
                        continue;

                    case DeltaLogEntryType.CHECKPOINT_METADATA:
                        metadata = new byte[entryLength];
                        unsafe
                        {
                            fixed(byte *m = metadata)
                            {
                                Buffer.MemoryCopy((void *)physicalAddress, m, entryLength, entryLength);
                            }
                        }
                        HybridLogRecoveryInfo recoveryInfo = new();
                        using (StreamReader s = new(new MemoryStream(metadata))) {
                            recoveryInfo.Initialize(s);
                            // Finish recovery if only specific versions are requested
                            if (recoveryInfo.version == recoverTo || recoveryInfo.version < recoverTo && recoveryInfo.nextVersion > recoverTo)
                            {
                                goto LoopEnd;
                            }
                        }
                        continue;

                    default:
                        throw new FasterException("Unexpected entry type");
                    }
LoopEnd:
                    break;
                }
                if (metadata != null)
                {
                    return(metadata);
                }
            }

            DirectoryInfo dir = new(directoryConfiguration.GetHybridLogCheckpointFolder(logToken));

            if (!File.Exists(dir.FullName + Path.DirectorySeparatorChar + "completed.dat"))
            {
                return(null);
            }

            string checkpointInfoFile = directoryConfiguration.GetHybridLogCheckpointMetaFileName(logToken);

            using BinaryReader reader = new(new FileStream(checkpointInfoFile, FileMode.Open));
            var len = reader.ReadInt32();

            return(reader.ReadBytes(len));
        }
        /// <inheritdoc />
        public byte[] GetLogCheckpointMetadata(Guid logToken, DeltaLog deltaLog, bool scanDelta, long recoverTo)
        {
            byte[] metadata = null;
            if (deltaLog != null && scanDelta)
            {
                // Try to get latest valid metadata from delta-log
                deltaLog.Reset();
                while (deltaLog.GetNext(out long physicalAddress, out int entryLength, out var type))
                {
                    switch (type)
                    {
                    case DeltaLogEntryType.DELTA:
                        // consider only metadata records
                        continue;

                    case DeltaLogEntryType.CHECKPOINT_METADATA:
                        metadata = new byte[entryLength];
                        unsafe
                        {
                            fixed(byte *m = metadata)
                            Buffer.MemoryCopy((void *)physicalAddress, m, entryLength, entryLength);
                        }
                        HybridLogRecoveryInfo recoveryInfo = new();
                        using (StreamReader s = new(new MemoryStream(metadata))) {
                            recoveryInfo.Initialize(s);
                            // Finish recovery if only specific versions are requested
                            if (recoveryInfo.version == recoverTo || recoveryInfo.version < recoverTo && recoveryInfo.nextVersion > recoverTo)
                            {
                                goto LoopEnd;
                            }
                        }
                        continue;

                    default:
                        throw new FasterException("Unexpected entry type");
                    }
LoopEnd:
                    break;
                }
                if (metadata != null)
                {
                    return(metadata);
                }
            }

            var device = deviceFactory.Get(checkpointNamingScheme.LogCheckpointMetadata(logToken));

            ReadInto(device, 0, out byte[] writePad, sizeof(int));
            int size = BitConverter.ToInt32(writePad, 0);

            byte[] body;
            if (writePad.Length >= size + sizeof(int))
            {
                body = writePad;
            }
            else
            {
                ReadInto(device, 0, out body, size + sizeof(int));
            }
            device.Dispose();
            return(body.AsSpan().Slice(sizeof(int), size).ToArray());
        }
Ejemplo n.º 7
0
 internal void WriteHybridLogIncrementalMetaInfo(DeltaLog deltaLog)
 {
     checkpointManager.CommitLogIncrementalCheckpoint(_hybridLogCheckpointToken, _hybridLogCheckpoint.info.version, _hybridLogCheckpoint.info.ToByteArray(), deltaLog);
 }