示例#1
0
        /// <summary>
        /// Create FASTER instance
        /// </summary>
        /// <param name="size">Size of core index (#cache lines)</param>
        /// <param name="comparer">FASTER equality comparer for key</param>
        /// <param name="variableLengthStructSettings"></param>
        /// <param name="logSettings">Log settings</param>
        /// <param name="checkpointSettings">Checkpoint settings</param>
        /// <param name="serializerSettings">Serializer settings</param>
        public FasterKV(long size, LogSettings logSettings,
                        CheckpointSettings checkpointSettings  = null, SerializerSettings <Key, Value> serializerSettings = null,
                        IFasterEqualityComparer <Key> comparer = null,
                        VariableLengthStructSettings <Key, Value> variableLengthStructSettings = null)
        {
            if (comparer != null)
            {
                this.comparer = comparer;
            }
            else
            {
                if (typeof(IFasterEqualityComparer <Key>).IsAssignableFrom(typeof(Key)))
                {
                    this.comparer = new Key() as IFasterEqualityComparer <Key>;
                }
                else
                {
                    Console.WriteLine(
                        "***WARNING*** Creating default FASTER key equality comparer based on potentially slow EqualityComparer<Key>.Default. To avoid this, provide a comparer (IFasterEqualityComparer<Key>) as an argument to FASTER's constructor, or make Key implement the interface IFasterEqualityComparer<Key>");
                    this.comparer = FasterEqualityComparer <Key> .Default;
                }
            }

            if (checkpointSettings == null)
            {
                checkpointSettings = new CheckpointSettings();
            }

            if (checkpointSettings.CheckpointDir != null && checkpointSettings.CheckpointManager != null)
            {
                throw new FasterException(
                          "Specify either CheckpointManager or CheckpointDir for CheckpointSettings, not both");
            }

            bool oldCheckpointManager = false;

            if (oldCheckpointManager)
            {
                checkpointManager = checkpointSettings.CheckpointManager ??
                                    new LocalCheckpointManager(checkpointSettings.CheckpointDir ?? "");
            }
            else
            {
                checkpointManager = checkpointSettings.CheckpointManager ??
                                    new DeviceLogCommitCheckpointManager
                                        (new LocalStorageNamedDeviceFactory(),
                                        new DefaultCheckpointNamingScheme(
                                            new DirectoryInfo(checkpointSettings.CheckpointDir ?? ".").FullName));
            }

            if (checkpointSettings.CheckpointManager == null)
            {
                disposeCheckpointManager = true;
            }

            FoldOverSnapshot = checkpointSettings.CheckPointType == core.CheckpointType.FoldOver;
            CopyReadsToTail  = logSettings.CopyReadsToTail;

            if (logSettings.ReadCacheSettings != null)
            {
                CopyReadsToTail = false;
                UseReadCache    = true;
            }

            if (Utility.IsBlittable <Key>() && Utility.IsBlittable <Value>())
            {
                if (variableLengthStructSettings != null)
                {
                    hlog = new VariableLengthBlittableAllocator <Key, Value>(logSettings, variableLengthStructSettings,
                                                                             this.comparer, null, epoch);
                    Log = new LogAccessor <Key, Value>(this, hlog);
                    if (UseReadCache)
                    {
                        readcache = new VariableLengthBlittableAllocator <Key, Value>(
                            new LogSettings
                        {
                            PageSizeBits    = logSettings.ReadCacheSettings.PageSizeBits,
                            MemorySizeBits  = logSettings.ReadCacheSettings.MemorySizeBits,
                            SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
                            MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction
                        }, variableLengthStructSettings, this.comparer, ReadCacheEvict, epoch);
                        readcache.Initialize();
                        ReadCache = new LogAccessor <Key, Value>(this, readcache);
                    }
                }
                else
                {
                    hlog = new BlittableAllocator <Key, Value>(logSettings, this.comparer, null, epoch);
                    Log  = new LogAccessor <Key, Value>(this, hlog);
                    if (UseReadCache)
                    {
                        readcache = new BlittableAllocator <Key, Value>(
                            new LogSettings
                        {
                            PageSizeBits    = logSettings.ReadCacheSettings.PageSizeBits,
                            MemorySizeBits  = logSettings.ReadCacheSettings.MemorySizeBits,
                            SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
                            MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction
                        }, this.comparer, ReadCacheEvict, epoch);
                        readcache.Initialize();
                        ReadCache = new LogAccessor <Key, Value>(this, readcache);
                    }
                }
            }
            else
            {
                WriteDefaultOnDelete = true;

                hlog = new GenericAllocator <Key, Value>(logSettings, serializerSettings, this.comparer, null, epoch);
                Log  = new LogAccessor <Key, Value>(this, hlog);
                if (UseReadCache)
                {
                    readcache = new GenericAllocator <Key, Value>(
                        new LogSettings
                    {
                        PageSizeBits    = logSettings.ReadCacheSettings.PageSizeBits,
                        MemorySizeBits  = logSettings.ReadCacheSettings.MemorySizeBits,
                        SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
                        MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction
                    }, serializerSettings, this.comparer, ReadCacheEvict, epoch);
                    readcache.Initialize();
                    ReadCache = new LogAccessor <Key, Value>(this, readcache);
                }
            }

            hlog.Initialize();

            sectorSize = (int)logSettings.LogDevice.SectorSize;
            Initialize(size, sectorSize);

            systemState         = default;
            systemState.phase   = Phase.REST;
            systemState.version = 1;
        }
示例#2
0
        /// <summary>
        /// Create FASTER instance
        /// </summary>
        /// <param name="size">Size of core index (#cache lines)</param>
        /// <param name="logSettings">Log settings</param>
        /// <param name="checkpointSettings">Checkpoint settings</param>
        /// <param name="serializerSettings">Serializer settings</param>
        /// <param name="comparer">FASTER equality comparer for key</param>
        /// <param name="variableLengthStructSettings"></param>
        public FasterKV(long size, LogSettings logSettings,
                        CheckpointSettings checkpointSettings  = null, SerializerSettings <Key, Value> serializerSettings = null,
                        IFasterEqualityComparer <Key> comparer = null,
                        VariableLengthStructSettings <Key, Value> variableLengthStructSettings = null)
        {
            if (comparer != null)
            {
                this.comparer = comparer;
            }
            else
            {
                if (typeof(IFasterEqualityComparer <Key>).IsAssignableFrom(typeof(Key)))
                {
                    if (default(Key) != null)
                    {
                        this.comparer = default(Key) as IFasterEqualityComparer <Key>;
                    }
                    else if (typeof(Key).GetConstructor(Type.EmptyTypes) != null)
                    {
                        this.comparer = Activator.CreateInstance(typeof(Key)) as IFasterEqualityComparer <Key>;
                    }
                }
                else
                {
                    this.comparer = FasterEqualityComparer.Get <Key>();
                }
            }

            if (checkpointSettings == null)
            {
                checkpointSettings = new CheckpointSettings();
            }

            if (checkpointSettings.CheckpointDir != null && checkpointSettings.CheckpointManager != null)
            {
                throw new FasterException(
                          "Specify either CheckpointManager or CheckpointDir for CheckpointSettings, not both");
            }

            bool oldCheckpointManager = false;

            if (oldCheckpointManager)
            {
                checkpointManager = checkpointSettings.CheckpointManager ??
                                    new LocalCheckpointManager(checkpointSettings.CheckpointDir ?? "");
            }
            else
            {
                checkpointManager = checkpointSettings.CheckpointManager ??
                                    new DeviceLogCommitCheckpointManager
                                        (new LocalStorageNamedDeviceFactory(),
                                        new DefaultCheckpointNamingScheme(
                                            new DirectoryInfo(checkpointSettings.CheckpointDir ?? ".").FullName), removeOutdated: checkpointSettings.RemoveOutdated);
            }

            if (checkpointSettings.CheckpointManager == null)
            {
                disposeCheckpointManager = true;
            }

            FoldOverSnapshot = checkpointSettings.CheckPointType == core.CheckpointType.FoldOver;
            CopyReadsToTail  = logSettings.CopyReadsToTail;

            if (logSettings.ReadCacheSettings != null)
            {
                CopyReadsToTail = CopyReadsToTail.None;
                UseReadCache    = true;
            }

            UpdateVarLen(ref variableLengthStructSettings);

            if ((!Utility.IsBlittable <Key>() && variableLengthStructSettings?.keyLength == null) ||
                (!Utility.IsBlittable <Value>() && variableLengthStructSettings?.valueLength == null))
            {
                WriteDefaultOnDelete = true;

                hlog = new GenericAllocator <Key, Value>(logSettings, serializerSettings, this.comparer, null, epoch);
                Log  = new LogAccessor <Key, Value>(this, hlog);
                if (UseReadCache)
                {
                    readcache = new GenericAllocator <Key, Value>(
                        new LogSettings
                    {
                        LogDevice       = new NullDevice(),
                        ObjectLogDevice = new NullDevice(),
                        PageSizeBits    = logSettings.ReadCacheSettings.PageSizeBits,
                        MemorySizeBits  = logSettings.ReadCacheSettings.MemorySizeBits,
                        SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
                        MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction
                    }, serializerSettings, this.comparer, ReadCacheEvict, epoch);
                    readcache.Initialize();
                    ReadCache = new LogAccessor <Key, Value>(this, readcache);
                }
            }
            else if (variableLengthStructSettings != null)
            {
                hlog = new VariableLengthBlittableAllocator <Key, Value>(logSettings, variableLengthStructSettings,
                                                                         this.comparer, null, epoch);
                Log = new LogAccessor <Key, Value>(this, hlog);
                if (UseReadCache)
                {
                    readcache = new VariableLengthBlittableAllocator <Key, Value>(
                        new LogSettings
                    {
                        LogDevice       = new NullDevice(),
                        PageSizeBits    = logSettings.ReadCacheSettings.PageSizeBits,
                        MemorySizeBits  = logSettings.ReadCacheSettings.MemorySizeBits,
                        SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
                        MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction
                    }, variableLengthStructSettings, this.comparer, ReadCacheEvict, epoch);
                    readcache.Initialize();
                    ReadCache = new LogAccessor <Key, Value>(this, readcache);
                }
            }
            else
            {
                hlog = new BlittableAllocator <Key, Value>(logSettings, this.comparer, null, epoch);
                Log  = new LogAccessor <Key, Value>(this, hlog);
                if (UseReadCache)
                {
                    readcache = new BlittableAllocator <Key, Value>(
                        new LogSettings
                    {
                        LogDevice       = new NullDevice(),
                        PageSizeBits    = logSettings.ReadCacheSettings.PageSizeBits,
                        MemorySizeBits  = logSettings.ReadCacheSettings.MemorySizeBits,
                        SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
                        MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction
                    }, this.comparer, ReadCacheEvict, epoch);
                    readcache.Initialize();
                    ReadCache = new LogAccessor <Key, Value>(this, readcache);
                }
            }

            hlog.Initialize();

            sectorSize = (int)logSettings.LogDevice.SectorSize;
            Initialize(size, sectorSize);

            systemState         = default;
            systemState.Phase   = Phase.REST;
            systemState.Version = 1;
        }
示例#3
0
        /// <summary>
        /// Create FASTER instance
        /// </summary>
        /// <param name="size">Size of core index (#cache lines)</param>
        /// <param name="comparer">FASTER equality comparer for key</param>
        /// <param name="functions">Callback functions</param>
        /// <param name="logSettings">Log settings</param>
        /// <param name="checkpointSettings">Checkpoint settings</param>
        /// <param name="serializerSettings">Serializer settings</param>
        public FasterKV(long size, Functions functions, LogSettings logSettings, CheckpointSettings checkpointSettings = null, SerializerSettings <Key, Value> serializerSettings = null, IFasterEqualityComparer <Key> comparer = null)
        {
            threadCtx     = new FastThreadLocal <FasterExecutionContext>();
            prevThreadCtx = new FastThreadLocal <FasterExecutionContext>();

            if (comparer != null)
            {
                this.comparer = comparer;
            }
            else
            {
                if (typeof(IFasterEqualityComparer <Key>).IsAssignableFrom(typeof(Key)))
                {
                    this.comparer = new Key() as IFasterEqualityComparer <Key>;
                }
                else
                {
                    Console.WriteLine("***WARNING*** Creating default FASTER key equality comparer based on potentially slow EqualityComparer<Key>.Default. To avoid this, provide a comparer (IFasterEqualityComparer<Key>) as an argument to FASTER's constructor, or make Key implement the interface IFasterEqualityComparer<Key>");
                    this.comparer = FasterEqualityComparer <Key> .Default;
                }
            }

            if (checkpointSettings == null)
            {
                checkpointSettings = new CheckpointSettings();
            }

            directoryConfiguration = new DirectoryConfiguration(checkpointSettings.CheckpointDir);

            FoldOverSnapshot = checkpointSettings.CheckPointType == core.CheckpointType.FoldOver;
            CopyReadsToTail  = logSettings.CopyReadsToTail;
            this.functions   = functions;

            if (logSettings.ReadCacheSettings != null)
            {
                CopyReadsToTail = false;
                UseReadCache    = true;
            }

            if (Utility.IsBlittable <Key>() && Utility.IsBlittable <Value>())
            {
                hlog = new BlittableAllocator <Key, Value>(logSettings, this.comparer, null, epoch);
                Log  = new LogAccessor <Key, Value, Input, Output, Context>(this, hlog);
                if (UseReadCache)
                {
                    readcache = new BlittableAllocator <Key, Value>(
                        new LogSettings {
                        PageSizeBits    = logSettings.ReadCacheSettings.PageSizeBits,
                        MemorySizeBits  = logSettings.ReadCacheSettings.MemorySizeBits,
                        SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
                        MutableFraction = logSettings.ReadCacheSettings.SecondChanceFraction
                    }, this.comparer, ReadCacheEvict, epoch);
                    readcache.Initialize();
                    ReadCache = new LogAccessor <Key, Value, Input, Output, Context>(this, readcache);
                }
            }
            else
            {
                hlog = new GenericAllocator <Key, Value>(logSettings, serializerSettings, this.comparer, null, epoch);
                Log  = new LogAccessor <Key, Value, Input, Output, Context>(this, hlog);
                if (UseReadCache)
                {
                    readcache = new GenericAllocator <Key, Value>(
                        new LogSettings
                    {
                        PageSizeBits    = logSettings.ReadCacheSettings.PageSizeBits,
                        MemorySizeBits  = logSettings.ReadCacheSettings.MemorySizeBits,
                        SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
                        MutableFraction = logSettings.ReadCacheSettings.SecondChanceFraction
                    }, serializerSettings, this.comparer, ReadCacheEvict, epoch);
                    readcache.Initialize();
                    ReadCache = new LogAccessor <Key, Value, Input, Output, Context>(this, readcache);
                }
            }

            hlog.Initialize();

            sectorSize = (int)logSettings.LogDevice.SectorSize;
            Initialize(size, sectorSize);

            _systemState         = default(SystemState);
            _systemState.phase   = Phase.REST;
            _systemState.version = 1;
            _checkpointType      = CheckpointType.HYBRID_LOG_ONLY;
        }