/// <summary>
/// Each thread write to individual section
/// </summary>
/// <param name="token"></param>
/// <param name="random"></param>
/// <param name="start"></param>
/// <param name="end"></param>
private void TestSequentialWrite(CancellationToken token, Random random, long start, long end)
{
using (var handle = Win32NativeFileMethods.CreateFile(filePath,
Win32NativeFileAccess.GenericWrite, Win32NativeFileShare.Write, IntPtr.Zero,
Win32NativeFileCreationDisposition.OpenExisting,
testRequest.BufferedWrites ? Win32NativeFileAttributes.None : (Win32NativeFileAttributes.Write_Through | Win32NativeFileAttributes.NoBuffering),
IntPtr.Zero))
{
ValidateHandle(handle);
using (var fs = new FileStream(handle, FileAccess.ReadWrite))
{
var buffer = new byte[testRequest.ChunkSize];
var position = start;
fs.Seek(position, SeekOrigin.Begin);
var sw = new Stopwatch();
while (token.IsCancellationRequested == false)
{
random.NextBytes(buffer);
sw.Restart();
fs.Write(buffer, 0, testRequest.ChunkSize);
dataStorage.MarkWrite(testRequest.ChunkSize, sw.ElapsedMilliseconds);
position += testRequest.ChunkSize;
if (position + testRequest.ChunkSize > end)
{
fs.Seek(start, SeekOrigin.Begin);
position = start;
}
}
}
}
}
/// <summary>
/// Each thread reads sequentially from beginning of file
/// </summary>
/// <param name="token"></param>
private void TestSequentialRead(CancellationToken token)
{
using (var handle = Win32NativeFileMethods.CreateFile(filePath,
Win32NativeFileAccess.GenericRead, Win32NativeFileShare.Read, IntPtr.Zero,
Win32NativeFileCreationDisposition.OpenExisting,
testRequest.BufferedReads ? Win32NativeFileAttributes.None : Win32NativeFileAttributes.NoBuffering,
IntPtr.Zero))
{
ValidateHandle(handle);
using (var fs = new FileStream(handle, FileAccess.Read))
{
var buffer = new byte[testRequest.ChunkSize];
var position = 0;
fs.Seek(position, SeekOrigin.Begin);
var sw = new Stopwatch();
while (token.IsCancellationRequested == false)
{
sw.Restart();
fs.Read(buffer, 0, testRequest.ChunkSize);
dataStorage.MarkRead(testRequest.ChunkSize, sw.ElapsedMilliseconds);
position += testRequest.ChunkSize;
if (position + testRequest.ChunkSize > testRequest.FileSize)
{
fs.Seek(0, SeekOrigin.Begin);
position = 0;
}
}
}
}
}
/// <summary>
/// Each thread reads from any section
/// </summary>
/// <param name="token"></param>
/// <param name="random"></param>
private void TestRandomRead(CancellationToken token, Random random)
{
using (var handle = Win32NativeFileMethods.CreateFile(filePath,
Win32NativeFileAccess.GenericRead, Win32NativeFileShare.Read, IntPtr.Zero,
Win32NativeFileCreationDisposition.OpenExisting,
testRequest.BufferedReads ? Win32NativeFileAttributes.None : Win32NativeFileAttributes.NoBuffering,
IntPtr.Zero))
{
ValidateHandle(handle);
using (var fs = new FileStream(handle, FileAccess.Read))
{
var buffer = new byte[testRequest.ChunkSize];
var sw = new Stopwatch();
while (token.IsCancellationRequested == false)
{
var position = LongRandom(0, testRequest.FileSize, 4096, random);
sw.Restart();
// for with with no_buffering seek must be multiple of 4096.
fs.Seek(position, SeekOrigin.Begin);
fs.Read(buffer, 0, testRequest.ChunkSize);
dataStorage.MarkRead(testRequest.ChunkSize, sw.ElapsedMilliseconds);
}
}
}
}
public override void TestDiskIO()
{
try
{
PrepareTestFile(filePath);
if (File.Exists(journalPath))
{
File.Delete(journalPath);
}
using (TestTimeMeasure())
{
onInfo("Starting test...");
taskKillToken.ThrowIfCancellationRequested();
try
{
secondTimer = new Timer(SecondTicked, null, 1000, 1000);
using (dataHandle = Win32NativeFileMethods.CreateFile(filePath,
Win32NativeFileAccess.GenericWrite, Win32NativeFileShare.None, IntPtr.Zero,
Win32NativeFileCreationDisposition.OpenExisting,
Win32NativeFileAttributes.Write_Through,
IntPtr.Zero))
{
ValidateHandle(dataHandle);
using (dataFs = new FileStream(dataHandle, FileAccess.Write))
using (journalHandle = Win32NativeFileMethods.CreateFile(journalPath,
Win32NativeFileAccess.GenericWrite, Win32NativeFileShare.None, IntPtr.Zero,
Win32NativeFileCreationDisposition.CreateAlways,
Win32NativeFileAttributes.Write_Through | Win32NativeFileAttributes.NoBuffering,
IntPtr.Zero))
{
ValidateHandle(journalHandle);
using (journalFs = new FileStream(journalHandle, FileAccess.Write))
{
MeasurePerformance();
}
}
}
}
finally
{
if (File.Exists(journalPath))
{
IOExtensions.DeleteFile(journalPath);
}
}
}
}
catch (Exception e)
{
onInfo("Failed to perform I/O test... check logs for more information");
log.Error("Failed to perform I/O test...", e); //this could happen because of things like SAN network failure, and we really want to know about it if it happens
hasFailed = true;
errors.Add(e);
}
}
private void FsyncIfNeeded(int dataFileWriteCounter)
{
if (dataFileWriteCounter % 500 == 0)
{
dataFs.Flush(true);
journalFs.Flush(true);
Win32NativeFileMethods.FlushFileBuffers(dataHandle);
Win32NativeFileMethods.FlushFileBuffers(journalHandle);
}
}
public CodecIndexInput(FileInfo file, Func <Stream, Stream> applyCodecs)
{
try
{
this.file = file;
this.applyCodecs = applyCodecs;
if (file.Length == 0)
{
stream = applyCodecs(Stream.Null);
return;
}
fileHandle = Win32NativeFileMethods.CreateFile(file.FullName,
Win32NativeFileAccess.GenericRead,
Win32NativeFileShare.Read | Win32NativeFileShare.Write | Win32NativeFileShare.Delete,
IntPtr.Zero,
Win32NativeFileCreationDisposition.OpenExisting,
Win32NativeFileAttributes.RandomAccess,
IntPtr.Zero);
if (fileHandle.IsInvalid)
{
const int ERROR_FILE_NOT_FOUND = 2;
if (Marshal.GetLastWin32Error() == ERROR_FILE_NOT_FOUND)
{
throw new FileNotFoundException(file.FullName);
}
throw new Win32Exception(Marshal.GetLastWin32Error(), "Could not open file " + file.FullName);
}
mmf = Win32MemoryMapNativeMethods.CreateFileMapping(fileHandle.DangerousGetHandle(), IntPtr.Zero, Win32MemoryMapNativeMethods.FileMapProtection.PageReadonly,
0, 0, null);
if (mmf == IntPtr.Zero)
{
throw new Win32Exception(Marshal.GetLastWin32Error(), "Could not create file mapping for " + file.FullName);
}
basePtr = Win32MemoryMapNativeMethods.MapViewOfFileEx(mmf,
Win32MemoryMapNativeMethods.NativeFileMapAccessType.Read,
0, 0, UIntPtr.Zero, null);
if (basePtr == null)
{
throw new Win32Exception(Marshal.GetLastWin32Error(), "Could not map file " + file.FullName);
}
stream = applyCodecs(new MmapStream(file.FullName, basePtr, file.Length));
}
catch (Exception)
{
Dispose(false);
throw;
}
}
private static long GetAvailableFreeSpace(string path)
{
if (StorageEnvironmentOptions.RunningOnPosix == false)
{
if (Win32NativeFileMethods.GetDiskFreeSpaceEx(path, out _, out _, out var total) == false)
{
return(0);
}
return((long)total);
}
return(new DriveInfo(path).AvailableFreeSpace);
}
public override void TestDiskIO()
{
PrepareTestFile(filePath);
if (File.Exists(journalPath))
{
File.Delete(journalPath);
}
using (TestTimeMeasure())
{
onInfo("Starting test...");
taskKillToken.ThrowIfCancellationRequested();
try
{
secondTimer = new Timer(SecondTicked, null, 1000, 1000);
using (dataHandle = Win32NativeFileMethods.CreateFile(filePath,
Win32NativeFileAccess.GenericWrite, Win32NativeFileShare.None, IntPtr.Zero,
Win32NativeFileCreationDisposition.OpenExisting,
Win32NativeFileAttributes.Write_Through,
IntPtr.Zero))
{
ValidateHandle(dataHandle);
using (dataFs = new FileStream(dataHandle, FileAccess.Write))
using (journalHandle = Win32NativeFileMethods.CreateFile(journalPath,
Win32NativeFileAccess.GenericWrite, Win32NativeFileShare.None, IntPtr.Zero,
Win32NativeFileCreationDisposition.CreateAlways,
Win32NativeFileAttributes.Write_Through | Win32NativeFileAttributes.NoBuffering,
IntPtr.Zero))
{
ValidateHandle(journalHandle);
using (journalFs = new FileStream(journalHandle, FileAccess.Write))
{
MeasurePerformance();
}
}
}
}
finally
{
if (File.Exists(journalPath))
{
IOExtensions.DeleteFile(journalPath);
}
}
}
}
/// <summary>
/// Each thread write to individual section
/// Each thread reads from any section
/// </summary>
/// <param name="token"></param>
/// <param name="random"></param>
/// <param name="start"></param>
/// <param name="end"></param>
private void TestRandomReadWrite(CancellationToken token, Random random, long start, long end)
{
using (var readHandle = Win32NativeFileMethods.CreateFile(filePath,
Win32NativeFileAccess.GenericWrite | Win32NativeFileAccess.GenericRead,
Win32NativeFileShare.Read | Win32NativeFileShare.Write, IntPtr.Zero,
Win32NativeFileCreationDisposition.OpenExisting,
testRequest.BufferedReads ? Win32NativeFileAttributes.None : Win32NativeFileAttributes.NoBuffering,
IntPtr.Zero))
using (var writeHandle = Win32NativeFileMethods.CreateFile(filePath,
Win32NativeFileAccess.GenericWrite | Win32NativeFileAccess.GenericRead,
Win32NativeFileShare.Read | Win32NativeFileShare.Write, IntPtr.Zero,
Win32NativeFileCreationDisposition.OpenExisting,
testRequest.BufferedWrites ? Win32NativeFileAttributes.None : (Win32NativeFileAttributes.NoBuffering | Win32NativeFileAttributes.Write_Through),
IntPtr.Zero))
{
ValidateHandle(readHandle);
ValidateHandle(writeHandle);
using (var readFs = new FileStream(readHandle, FileAccess.Read))
using (var writeFs = new FileStream(readHandle, FileAccess.Write))
{
var buffer = new byte[testRequest.ChunkSize];
var sw = new Stopwatch();
while (token.IsCancellationRequested == false)
{
if (random.Next(2) > 0)
{
var position = LongRandom(0, testRequest.FileSize, 4096, random);
sw.Restart();
readFs.Seek(position, SeekOrigin.Begin);
readFs.Read(buffer, 0, testRequest.ChunkSize);
dataStorage.MarkRead(testRequest.ChunkSize, sw.ElapsedMilliseconds);
}
else
{
var position = LongRandom(start, end - testRequest.ChunkSize, 4096, random);
random.NextBytes(buffer);
sw.Restart();
writeFs.Seek(position, SeekOrigin.Begin);
writeFs.Write(buffer, 0, testRequest.ChunkSize);
dataStorage.MarkWrite(testRequest.ChunkSize, sw.ElapsedMilliseconds);
}
}
}
}
}
public bool Read(long pageNumber, byte *buffer, int count)
{
if (_readHandle == null)
{
_readHandle = Win32NativeFileMethods.CreateFile(_filename,
Win32NativeFileAccess.GenericRead,
Win32NativeFileShare.Write | Win32NativeFileShare.Read | Win32NativeFileShare.Delete,
IntPtr.Zero,
Win32NativeFileCreationDisposition.OpenExisting,
Win32NativeFileAttributes.Normal,
IntPtr.Zero);
}
long position = pageNumber * AbstractPager.PageSize;
var overlapped = new Overlapped((int)(position & 0xffffffff), (int)(position >> 32), IntPtr.Zero, null);
NativeOverlapped *nativeOverlapped = overlapped.Pack(null, null);
try
{
while (count > 0)
{
int read;
if (Win32NativeFileMethods.ReadFile(_readHandle, buffer, count, out read, nativeOverlapped) == false)
{
int lastWin32Error = Marshal.GetLastWin32Error();
if (lastWin32Error == ErrorHandleEof)
{
return(false);
}
throw new Win32Exception(lastWin32Error);
}
count -= read;
buffer += read;
position += read;
nativeOverlapped->OffsetLow = (int)(position & 0xffffffff);
nativeOverlapped->OffsetHigh = (int)(position >> 32);
}
return(true);
}
finally
{
Overlapped.Free(nativeOverlapped);
}
}
public Win32FileJournalWriter(string filename, long journalSize)
{
_filename = filename;
_handle = Win32NativeFileMethods.CreateFile(filename,
Win32NativeFileAccess.GenericWrite, Win32NativeFileShare.Read, IntPtr.Zero,
Win32NativeFileCreationDisposition.OpenAlways,
Win32NativeFileAttributes.Write_Through | Win32NativeFileAttributes.NoBuffering | Win32NativeFileAttributes.Overlapped, IntPtr.Zero);
if (_handle.IsInvalid)
{
throw new Win32Exception();
}
Win32NativeFileMethods.SetFileLength(_handle, journalSize);
NumberOfAllocatedPages = journalSize / AbstractPager.PageSize;
_manualResetEvent = new ManualResetEvent(false);
_nativeOverlapped = (NativeOverlapped *)Marshal.AllocHGlobal(sizeof(NativeOverlapped));
_nativeOverlapped->InternalLow = IntPtr.Zero;
_nativeOverlapped->InternalHigh = IntPtr.Zero;
}