/// <summary>
/// Constructor with more options for derived classes
/// </summary>
/// <param name="filename">File name (or prefix) with path</param>
/// <param name="preallocateFile"></param>
/// <param name="deleteOnClose"></param>
/// <param name="disableFileBuffering"></param>
/// <param name="capacity">The maximum number of bytes this storage device can accommondate, or CAPACITY_UNSPECIFIED if there is no such limit </param>
/// <param name="recoverDevice">Whether to recover device metadata from existing files</param>
/// <param name="initialLogFileHandles">Optional set of preloaded safe file handles, which can speed up hydration of preexisting log file handles</param>
protected internal LocalStorageDevice(string filename,
bool preallocateFile = false,
bool deleteOnClose = false,
bool disableFileBuffering = true,
long capacity = Devices.CAPACITY_UNSPECIFIED,
bool recoverDevice = false,
IEnumerable <KeyValuePair <int, SafeFileHandle> > initialLogFileHandles = null)
: base(filename, GetSectorSize(filename), capacity)
{
Native32.EnableProcessPrivileges();
string path = new FileInfo(filename).Directory.FullName;
if (!Directory.Exists(path))
{
Directory.CreateDirectory(path);
}
this.preallocateFile = preallocateFile;
this.deleteOnClose = deleteOnClose;
this.disableFileBuffering = disableFileBuffering;
logHandles = initialLogFileHandles != null
? new SafeConcurrentDictionary <int, SafeFileHandle>(initialLogFileHandles)
: new SafeConcurrentDictionary <int, SafeFileHandle>();
if (recoverDevice)
{
RecoverFiles();
}
}
/// <summary>
/// Constructor with more options for derived classes
/// </summary>
/// <param name="filename">File name (or prefix) with path</param>
/// <param name="preallocateFile"></param>
/// <param name="deleteOnClose"></param>
/// <param name="disableFileBuffering"></param>
/// <param name="capacity">The maximum number of bytes this storage device can accommondate, or CAPACITY_UNSPECIFIED if there is no such limit </param>
/// <param name="recoverDevice">Whether to recover device metadata from existing files</param>
/// <param name="initialLogFileHandles">Optional set of preloaded safe file handles, which can speed up hydration of preexisting log file handles</param>
/// <param name="useIoCompletionPort">Whether we use IO completion port with polling</param>
protected internal LocalStorageDevice(string filename,
bool preallocateFile = false,
bool deleteOnClose = false,
bool disableFileBuffering = true,
long capacity = Devices.CAPACITY_UNSPECIFIED,
bool recoverDevice = false,
IEnumerable <KeyValuePair <int, SafeFileHandle> > initialLogFileHandles = null,
bool useIoCompletionPort = true)
: base(filename, GetSectorSize(filename), capacity)
{
#if NETSTANDARD || NET
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
throw new FasterException("Cannot use LocalStorageDevice from non-Windows OS platform, use ManagedLocalStorageDevice instead.");
}
#endif
ThrottleLimit = 120;
this.useIoCompletionPort = useIoCompletionPort;
this._disposed = false;
if (useIoCompletionPort)
{
ThreadPool.GetMaxThreads(out int workerThreads, out _);
ioCompletionPort = Native32.CreateIoCompletionPort(new SafeFileHandle(new IntPtr(-1), false), IntPtr.Zero, UIntPtr.Zero, (uint)(workerThreads + NumCompletionThreads));
for (int i = 0; i < NumCompletionThreads; i++)
{
var thread = new Thread(() => new LocalStorageDeviceCompletionWorker().Start(ioCompletionPort, _callback))
{
IsBackground = true
};
thread.Start();
}
}
if (UsePrivileges && preallocateFile)
{
Native32.EnableProcessPrivileges();
}
string path = new FileInfo(filename).Directory.FullName;
if (!Directory.Exists(path))
{
Directory.CreateDirectory(path);
}
this.preallocateFile = preallocateFile;
this.deleteOnClose = deleteOnClose;
this.disableFileBuffering = disableFileBuffering;
this.results = new ConcurrentQueue <SimpleAsyncResult>();
logHandles = initialLogFileHandles != null
? new SafeConcurrentDictionary <int, SafeFileHandle>(initialLogFileHandles)
: new SafeConcurrentDictionary <int, SafeFileHandle>();
if (recoverDevice)
{
RecoverFiles();
}
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="filename"></param>
/// <param name="preallocateFile"></param>
/// <param name="deleteOnClose"></param>
/// <param name="disableFileBuffering"></param>
public LocalStorageDevice(string filename, bool preallocateFile = false, bool deleteOnClose = false, bool disableFileBuffering = true)
: base(filename, GetSectorSize(filename))
{
Native32.EnableProcessPrivileges();
this.preallocateFile = preallocateFile;
this.deleteOnClose = deleteOnClose;
this.disableFileBuffering = disableFileBuffering;
logHandles = new SafeConcurrentDictionary <int, SafeFileHandle>();
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="filename"></param>
/// <param name="preallocateFile"></param>
/// <param name="deleteOnClose"></param>
/// <param name="disableFileBuffering"></param>
/// <param name="capacity">The maximum number of bytes this storage device can accommondate, or CAPACITY_UNSPECIFIED if there is no such limit </param>
public LocalStorageDevice(string filename,
bool preallocateFile = false,
bool deleteOnClose = false,
bool disableFileBuffering = true,
long capacity = Devices.CAPACITY_UNSPECIFIED)
: base(filename, GetSectorSize(filename), capacity)
{
Native32.EnableProcessPrivileges();
this.preallocateFile = preallocateFile;
this.deleteOnClose = deleteOnClose;
this.disableFileBuffering = disableFileBuffering;
logHandles = new SafeConcurrentDictionary <int, SafeFileHandle>();
RecoverFiles();
}
/// <summary>
/// Constructor with more options for derived classes
/// </summary>
/// <param name="filename">File name (or prefix) with path</param>
/// <param name="preallocateFile"></param>
/// <param name="deleteOnClose"></param>
/// <param name="disableFileBuffering"></param>
/// <param name="capacity">The maximum number of bytes this storage device can accommondate, or CAPACITY_UNSPECIFIED if there is no such limit </param>
/// <param name="recoverDevice">Whether to recover device metadata from existing files</param>
/// <param name="initialLogFileHandles">Optional set of preloaded safe file handles, which can speed up hydration of preexisting log file handles</param>
protected internal LocalStorageDevice(string filename,
bool preallocateFile = false,
bool deleteOnClose = false,
bool disableFileBuffering = true,
long capacity = Devices.CAPACITY_UNSPECIFIED,
bool recoverDevice = false,
IEnumerable <KeyValuePair <int, SafeFileHandle> > initialLogFileHandles = null)
: base(filename, GetSectorSize(filename), capacity)
{
#if NETSTANDARD
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
throw new FasterException("Cannot use LocalStorageDevice from non-Windows OS platform, use ManagedLocalStorageDevice instead.");
}
#endif
if (UsePrivileges && preallocateFile)
{
Native32.EnableProcessPrivileges();
}
string path = new FileInfo(filename).Directory.FullName;
if (!Directory.Exists(path))
{
Directory.CreateDirectory(path);
}
this.preallocateFile = preallocateFile;
this.deleteOnClose = deleteOnClose;
this.disableFileBuffering = disableFileBuffering;
this.results = new ConcurrentQueue <SimpleAsyncResult>();
logHandles = initialLogFileHandles != null
? new SafeConcurrentDictionary <int, SafeFileHandle>(initialLogFileHandles)
: new SafeConcurrentDictionary <int, SafeFileHandle>();
if (recoverDevice)
{
RecoverFiles();
}
}
private void InternalRecover(Guid indexToken, Guid hybridLogToken)
{
Debug.WriteLine("********* Primary Recovery Information ********");
Debug.WriteLine("Index Checkpoint: {0}", indexToken);
Debug.WriteLine("HybridLog Checkpoint: {0}", hybridLogToken);
// Assert corresponding checkpoints are safe to recover from
Debug.Assert(IsCheckpointSafe(indexToken, CheckpointType.INDEX_ONLY),
"Cannot recover from incomplete index checkpoint " + indexToken.ToString());
Debug.Assert(IsCheckpointSafe(hybridLogToken, CheckpointType.HYBRID_LOG_ONLY),
"Cannot recover from incomplete hybrid log checkpoint " + hybridLogToken.ToString());
// Recovery appropriate context information
var recoveredICInfo = new IndexCheckpointInfo();
recoveredICInfo.Recover(indexToken, directoryConfiguration);
recoveredICInfo.info.DebugPrint();
var recoveredHLCInfo = new HybridLogCheckpointInfo();
recoveredHLCInfo.Recover(hybridLogToken, directoryConfiguration);
recoveredHLCInfo.info.DebugPrint();
// Check if the two checkpoints are compatible for recovery
if (!IsCompatible(recoveredICInfo.info, recoveredHLCInfo.info))
{
throw new Exception("Cannot recover from (" + indexToken.ToString() + "," + hybridLogToken.ToString() + ") checkpoint pair!\n");
}
// Set new system state after recovery
var v = recoveredHLCInfo.info.version;
_systemState.phase = Phase.REST;
_systemState.version = (v + 1);
// Recover fuzzy index from checkpoint
RecoverFuzzyIndex(recoveredICInfo);
// Recover segment offsets for object log
if (recoveredHLCInfo.info.objectLogSegmentOffsets != null)
{
Array.Copy(recoveredHLCInfo.info.objectLogSegmentOffsets,
hlog.GetSegmentOffsets(),
recoveredHLCInfo.info.objectLogSegmentOffsets.Length);
}
// Make index consistent for version v
if (FoldOverSnapshot)
{
RecoverHybridLog(recoveredICInfo.info, recoveredHLCInfo.info);
}
else
{
RecoverHybridLogFromSnapshotFile(recoveredICInfo.info, recoveredHLCInfo.info);
}
// Read appropriate hybrid log pages into memory
RestoreHybridLog(recoveredHLCInfo.info.finalLogicalAddress);
// Recover session information
_recoveredSessions = new SafeConcurrentDictionary <Guid, long>();
foreach (var sessionInfo in recoveredHLCInfo.info.continueTokens)
{
_recoveredSessions.GetOrAdd(sessionInfo.Key, sessionInfo.Value);
}
}