public async Task IoManagerTestsAsync()
{
var file1 = Path.Combine(Common.GetWorkDir(), $"file1.dat");
var file2 = Path.Combine(Common.GetWorkDir(), $"file2.dat");
Random random = new Random();
List <string> lines = new List <string>();
for (int i = 0; i < 1000; i++)
{
string line = RandomString.AlphaNumeric(100);
lines.Add(line);
}
// Single thread file operations.
DigestableSafeMutexIoManager ioman1 = new DigestableSafeMutexIoManager(file1);
// Delete the file if Exist.
ioman1.DeleteMe();
Assert.False(ioman1.Exists());
Assert.False(File.Exists(ioman1.DigestFilePath));
// Write the data to the file.
await ioman1.WriteAllLinesAsync(lines);
Assert.True(ioman1.Exists());
// Check if the digest file is created.
Assert.True(File.Exists(ioman1.DigestFilePath));
public async Task InitializeAsync(string workFolderPath, Network network, HashChain hashChain)
{
using (BenchmarkLogger.Measure())
{
WorkFolderPath = Guard.NotNullOrEmptyOrWhitespace(nameof(workFolderPath), workFolderPath, trim: true);
Network = Guard.NotNull(nameof(network), network);
HashChain = Guard.NotNull(nameof(hashChain), hashChain);
var indexFilePath = Path.Combine(WorkFolderPath, "MatureIndex.dat");
MatureIndexFileManager = new DigestableSafeMutexIoManager(indexFilePath, digestRandomIndex: -1);
var immatureIndexFilePath = Path.Combine(WorkFolderPath, "ImmatureIndex.dat");
ImmatureIndexFileManager = new DigestableSafeMutexIoManager(immatureIndexFilePath, digestRandomIndex: -1);
StartingFilter = StartingFilters.GetStartingFilter(Network);
StartingHeight = StartingFilters.GetStartingHeight(Network);
ImmatureFilters = new List <FilterModel>(150);
IndexLock = new AsyncLock();
using (await IndexLock.LockAsync().ConfigureAwait(false))
using (await MatureIndexFileManager.Mutex.LockAsync().ConfigureAwait(false))
using (await ImmatureIndexFileManager.Mutex.LockAsync().ConfigureAwait(false))
{
IoHelpers.EnsureDirectoryExists(WorkFolderPath);
await TryEnsureBackwardsCompatibilityAsync().ConfigureAwait(false);
if (Network == Network.RegTest)
{
MatureIndexFileManager.DeleteMe(); // RegTest is not a global ledger, better to delete it.
ImmatureIndexFileManager.DeleteMe();
}
if (!MatureIndexFileManager.Exists())
{
await MatureIndexFileManager.WriteAllLinesAsync(new[] { StartingFilter.ToHeightlessLine() }).ConfigureAwait(false);
}
await InitializeFiltersAsync().ConfigureAwait(false);
}
}
}
private async Task DeleteIfDeprecatedAsync(DigestableSafeMutexIoManager ioManager)
{
string firstLine;
using (var content = ioManager.OpenText())
{
firstLine = await content.ReadLineAsync().ConfigureAwait(false);
}
try
{
FilterModel.FromLine(firstLine);
}
catch
{
Logger.LogWarning("Old Index file detected. Deleting it.");
MatureIndexFileManager.DeleteMe();
ImmatureIndexFileManager.DeleteMe();
Logger.LogWarning("Successfully deleted old Index file.");
}
}
public async Task InitializeAsync()
{
using (BenchmarkLogger.Measure())
{
var indexFilePath = Path.Combine(WorkFolderPath, "MatureIndex.dat");
MatureIndexFileManager = new DigestableSafeMutexIoManager(indexFilePath, digestRandomIndex: -1);
var immatureIndexFilePath = Path.Combine(WorkFolderPath, "ImmatureIndex.dat");
ImmatureIndexFileManager = new DigestableSafeMutexIoManager(immatureIndexFilePath, digestRandomIndex: -1);
StartingFilter = StartingFilters.GetStartingFilter(Network);
StartingHeight = StartingFilter.Header.Height;
ImmatureFilters = new List <FilterModel>(150);
IndexLock = new AsyncLock();
using (await IndexLock.LockAsync().ConfigureAwait(false))
using (await MatureIndexFileManager.Mutex.LockAsync().ConfigureAwait(false))
using (await ImmatureIndexFileManager.Mutex.LockAsync().ConfigureAwait(false))
{
IoHelpers.EnsureDirectoryExists(WorkFolderPath);
await EnsureBackwardsCompatibilityAsync().ConfigureAwait(false);
if (Network == Network.RegTest)
{
MatureIndexFileManager.DeleteMe(); // RegTest is not a global ledger, better to delete it.
ImmatureIndexFileManager.DeleteMe();
}
if (!MatureIndexFileManager.Exists())
{
await MatureIndexFileManager.WriteAllLinesAsync(new[] { StartingFilter.ToLine() }).ConfigureAwait(false);
}
await InitializeFiltersAsync().ConfigureAwait(false);
}
}
}
public async Task IoManagerTestsAsync()
{
var file1 = Path.Combine(Global.Instance.DataDir, EnvironmentHelpers.GetCallerFileName(), EnvironmentHelpers.GetMethodName(), $"file1.dat");
var file2 = Path.Combine(Global.Instance.DataDir, EnvironmentHelpers.GetCallerFileName(), EnvironmentHelpers.GetMethodName(), $"file2.dat");
Random random = new Random();
List <string> lines = new List <string>();
for (int i = 0; i < 1000; i++)
{
string line = new string(Enumerable.Repeat(Constants.Chars, 100)
.Select(s => s[random.Next(s.Length)]).ToArray());
lines.Add(line);
}
// Single thread file operations.
DigestableSafeMutexIoManager ioman1 = new DigestableSafeMutexIoManager(file1);
// Delete the file if Exist.
ioman1.DeleteMe();
Assert.False(ioman1.Exists());
Assert.False(File.Exists(ioman1.DigestFilePath));
// Write the data to the file.
await ioman1.WriteAllLinesAsync(lines);
Assert.True(ioman1.Exists());
// Check if the digest file is created.
Assert.True(File.Exists(ioman1.DigestFilePath));
public async Task IoTestsAsync()
{
var file = Path.Combine(Global.Instance.DataDir, nameof(IoTestsAsync), $"file.dat");
DigestableSafeMutexIoManager ioman = new DigestableSafeMutexIoManager(file);
ioman.DeleteMe();
await ioman.WriteAllLinesAsync(new string[0], dismissNullOrEmptyContent : false);
string RandomString()
{
StringBuilder builder = new StringBuilder();
var rnd = new Random();
char ch;
for (int i = 0; i < rnd.Next(10, 100); i++)
{
ch = Convert.ToChar(Convert.ToInt32(Math.Floor(26 * rnd.NextDouble() + 65)));
builder.Append(ch);
}
return(builder.ToString());
};
var list = new List <string>();
async Task WriteNextLineAsync()
{
var next = RandomString();
lock (list)
{
list.Add(next);
}
using (await ioman.Mutex.LockAsync())
{
var lines = (await ioman.ReadAllLinesAsync()).ToList();
lines.Add(next);
await ioman.WriteAllLinesAsync(lines);
}
};
var t1 = new Thread(() =>
{
for (var i = 0; i < 500; i++)
{
/* We have to block the Thread.
* If we use async/await pattern then Join() function at the end will indicate that the Thread is finished -
* which is not true bacause the WriteNextLineAsync() is not yet finished. The reason is that await will return execution
* the to the calling thread it is detected as the thread is done. t1 and t2 and t3 will still run in parallel!
*/
WriteNextLineAsync().Wait();
}
});
var t2 = new Thread(() =>
{
for (var i = 0; i < 500; i++)
{
WriteNextLineAsync().Wait();
}
});
var t3 = new Thread(() =>
{
for (var i = 0; i < 500; i++)
{
WriteNextLineAsync().Wait();
}
});
t1.Start();
t2.Start();
t3.Start();
await Task.Delay(100);
t1.Join();
t2.Join();
t3.Join();
Assert.False(t1.IsAlive);
Assert.False(t2.IsAlive);
Assert.False(t3.IsAlive);
var allLines = File.ReadAllLines(file);
Assert.NotEmpty(allLines);
/* Lines were added to the list and to the file parallel so the two data should be equal.
* If we "substract" them from each other we should get empty array.
*/
var diff = allLines.Except(list);
Assert.Empty(diff);
}
public async Task IoManagerTestsAsync()
{
var file1 = Path.Combine(Global.Instance.DataDir, nameof(IoManagerTestsAsync), $"file1.dat");
var file2 = Path.Combine(Global.Instance.DataDir, nameof(IoManagerTestsAsync), $"file2.dat");
Random random = new Random();
List <string> lines = new List <string>();
for (int i = 0; i < 1000; i++)
{
const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
string line = new string(Enumerable.Repeat(chars, 100)
.Select(s => s[random.Next(s.Length)]).ToArray());
lines.Add(line);
}
// Single thread file operations.
DigestableSafeMutexIoManager ioman1 = new DigestableSafeMutexIoManager(file1);
// Delete the file if Exist.
ioman1.DeleteMe();
Assert.False(ioman1.Exists());
Assert.False(File.Exists(ioman1.DigestFilePath));
// Write the data to the file.
await ioman1.WriteAllLinesAsync(lines);
Assert.True(ioman1.Exists());
// Check if the digest file is created.
Assert.True(File.Exists(ioman1.DigestFilePath));
// Read back the content and check.
bool IsStringArraysEqual(string[] lines1, string[] lines2)
{
if (lines1.Length != lines2.Length)
{
return(false);
}
for (int i = 0; i < lines1.Length; i++)
{
string line = lines2[i];
var readLine = lines1[i];
if (!line.Equals(readLine))
{
return(false);
}
}
return(true);
}
var readLines = await ioman1.ReadAllLinesAsync();
Assert.True(IsStringArraysEqual(readLines, lines.ToArray()));
// Check digest file, and write only differ logic.
// Write the same content, file should not be written.
var currentDate = File.GetLastWriteTimeUtc(ioman1.FilePath);
await Task.Delay(500);
await ioman1.WriteAllLinesAsync(lines);
var noChangeDate = File.GetLastWriteTimeUtc(ioman1.FilePath);
Assert.Equal(currentDate, noChangeDate);
// Write different content, file should be written.
currentDate = File.GetLastWriteTimeUtc(ioman1.FilePath);
await Task.Delay(500);
lines.Add("Lorem ipsum dolor sit amet, consectetur adipiscing elit.");
await ioman1.WriteAllLinesAsync(lines);
var newContentDate = File.GetLastWriteTimeUtc(ioman1.FilePath);
Assert.NotEqual(currentDate, newContentDate);
/* The next test is commented out because on mac and on linux File.Open does not lock the file
* it can be still written by the ioman1.WriteAllLinesAsync(). Tried with FileShare.None FileShare.Delete
* FileStream.Lock none of them are working or caused not supported on this platform exception.
* So there is no OP system way to garantee that the file won't be written during another write operation.
* For example git is using lock files to solve this problem. We are using system wide mutexes.
* For now there is no other way to do this. Some useful links :
* https://stackoverflow.com/questions/2751734/how-do-filesystems-handle-concurrent-read-write
* https://github.com/dotnet/corefx/issues/5964
*/
//using (File.OpenWrite(ioman1.FilePath))
//{
// // Should be OK because the same data is written.
// await ioman1.WriteAllLinesAsync(lines);
//}
//using (File.OpenWrite(ioman1.FilePath))
//{
// // Should fail because different data is written.
// await Assert.ThrowsAsync<IOException>(async () => await ioman1.WriteAllLinesAsync(lines));
//}
await ioman1.WriteAllLinesAsync(lines);
// Mutex tests.
// Acquire the Mutex with a background thread.
var myTask = Task.Run(async() =>
{
using (await ioman1.Mutex.LockAsync())
{
await Task.Delay(3000);
}
});
// Wait for the Task.Run to Acquire the Mutex.
await Task.Delay(100);
// Try to get the Mutex and save the time.
DateTime timeOfstart = DateTime.Now;
DateTime timeOfAcquired = default;
using (await ioman1.Mutex.LockAsync())
{
timeOfAcquired = DateTime.Now;
}
Assert.True(myTask.IsCompletedSuccessfully);
var elapsed = timeOfAcquired - timeOfstart;
Assert.InRange(elapsed, TimeSpan.FromMilliseconds(2000), TimeSpan.FromMilliseconds(4000));
// Simulate file write error and recovery logic.
// We have only *.new and *.old files.
File.Copy(ioman1.FilePath, ioman1.OldFilePath);
File.Move(ioman1.FilePath, ioman1.NewFilePath);
// At this point there is now OriginalFile.
var newFile = await ioman1.ReadAllLinesAsync();
Assert.True(IsStringArraysEqual(newFile, lines.ToArray()));
// Add one more line to have different data.
lines.Add("Lorem ipsum dolor sit amet, consectetur adipiscing elit.");
await ioman1.WriteAllLinesAsync(lines);
// Check recovery mechanism.
Assert.True(
File.Exists(ioman1.FilePath) &&
!File.Exists(ioman1.OldFilePath) &&
!File.Exists(ioman1.NewFilePath));
ioman1.DeleteMe();
Assert.False(ioman1.Exists());
// Check if directory is empty.
var fileCount = Directory.EnumerateFiles(Path.GetDirectoryName(ioman1.FilePath)).Count();
Assert.Equal(0, fileCount);
// Check Mutex usage on simultaneous file writes.
DigestableSafeMutexIoManager ioman2 = new DigestableSafeMutexIoManager(file2);
await Task.Run(async() =>
{
using (await ioman1.Mutex.LockAsync())
{
// Should not be a problem because they using different Mutexes.
using (await ioman2.Mutex.LockAsync())
{
await ioman1.WriteAllLinesAsync(lines);
await ioman2.WriteAllLinesAsync(lines);
ioman1.DeleteMe();
ioman2.DeleteMe();
}
}
});
// TryReplace test.
var dummyFilePath = $"{ioman1.FilePath}dummy";
var dummyContent = new string[]
{
"banana",
"peach"
};
await File.WriteAllLinesAsync(dummyFilePath, dummyContent);
await ioman1.WriteAllLinesAsync(lines);
ioman1.TryReplaceMeWith(dummyFilePath);
var fruits = await ioman1.ReadAllLinesAsync();
Assert.True(IsStringArraysEqual(dummyContent, fruits));
Assert.False(File.Exists(dummyFilePath));
ioman1.DeleteMe();
}
