public async Task IoManagerTestsAsync() { var file1 = Path.Combine(Common.GetWorkDir(), $"file1.dat"); var file2 = Path.Combine(Common.GetWorkDir(), $"file2.dat"); Random random = new Random(); List <string> lines = new List <string>(); for (int i = 0; i < 1000; i++) { string line = RandomString.AlphaNumeric(100); lines.Add(line); } // Single thread file operations. DigestableSafeMutexIoManager ioman1 = new DigestableSafeMutexIoManager(file1); // Delete the file if Exist. ioman1.DeleteMe(); Assert.False(ioman1.Exists()); Assert.False(File.Exists(ioman1.DigestFilePath)); // Write the data to the file. await ioman1.WriteAllLinesAsync(lines); Assert.True(ioman1.Exists()); // Check if the digest file is created. Assert.True(File.Exists(ioman1.DigestFilePath));
public async Task IoManagerTestsAsync() { var file1 = Path.Combine(Global.Instance.DataDir, EnvironmentHelpers.GetCallerFileName(), EnvironmentHelpers.GetMethodName(), $"file1.dat"); var file2 = Path.Combine(Global.Instance.DataDir, EnvironmentHelpers.GetCallerFileName(), EnvironmentHelpers.GetMethodName(), $"file2.dat"); Random random = new Random(); List <string> lines = new List <string>(); for (int i = 0; i < 1000; i++) { string line = new string(Enumerable.Repeat(Constants.Chars, 100) .Select(s => s[random.Next(s.Length)]).ToArray()); lines.Add(line); } // Single thread file operations. DigestableSafeMutexIoManager ioman1 = new DigestableSafeMutexIoManager(file1); // Delete the file if Exist. ioman1.DeleteMe(); Assert.False(ioman1.Exists()); Assert.False(File.Exists(ioman1.DigestFilePath)); // Write the data to the file. await ioman1.WriteAllLinesAsync(lines); Assert.True(ioman1.Exists()); // Check if the digest file is created. Assert.True(File.Exists(ioman1.DigestFilePath));
public async Task InitializeAsync(string workFolderPath, Network network, HashChain hashChain) { using (BenchmarkLogger.Measure()) { WorkFolderPath = Guard.NotNullOrEmptyOrWhitespace(nameof(workFolderPath), workFolderPath, trim: true); Network = Guard.NotNull(nameof(network), network); HashChain = Guard.NotNull(nameof(hashChain), hashChain); var indexFilePath = Path.Combine(WorkFolderPath, "MatureIndex.dat"); MatureIndexFileManager = new DigestableSafeMutexIoManager(indexFilePath, digestRandomIndex: -1); var immatureIndexFilePath = Path.Combine(WorkFolderPath, "ImmatureIndex.dat"); ImmatureIndexFileManager = new DigestableSafeMutexIoManager(immatureIndexFilePath, digestRandomIndex: -1); StartingFilter = StartingFilters.GetStartingFilter(Network); StartingHeight = StartingFilters.GetStartingHeight(Network); ImmatureFilters = new List <FilterModel>(150); IndexLock = new AsyncLock(); using (await IndexLock.LockAsync().ConfigureAwait(false)) using (await MatureIndexFileManager.Mutex.LockAsync().ConfigureAwait(false)) using (await ImmatureIndexFileManager.Mutex.LockAsync().ConfigureAwait(false)) { IoHelpers.EnsureDirectoryExists(WorkFolderPath); await TryEnsureBackwardsCompatibilityAsync().ConfigureAwait(false); if (Network == Network.RegTest) { MatureIndexFileManager.DeleteMe(); // RegTest is not a global ledger, better to delete it. ImmatureIndexFileManager.DeleteMe(); } if (!MatureIndexFileManager.Exists()) { await MatureIndexFileManager.WriteAllLinesAsync(new[] { StartingFilter.ToHeightlessLine() }).ConfigureAwait(false); } await InitializeFiltersAsync().ConfigureAwait(false); } } }
public async Task InitializeAsync() { using (BenchmarkLogger.Measure()) { var indexFilePath = Path.Combine(WorkFolderPath, "MatureIndex.dat"); MatureIndexFileManager = new DigestableSafeMutexIoManager(indexFilePath, digestRandomIndex: -1); var immatureIndexFilePath = Path.Combine(WorkFolderPath, "ImmatureIndex.dat"); ImmatureIndexFileManager = new DigestableSafeMutexIoManager(immatureIndexFilePath, digestRandomIndex: -1); StartingFilter = StartingFilters.GetStartingFilter(Network); StartingHeight = StartingFilter.Header.Height; ImmatureFilters = new List <FilterModel>(150); IndexLock = new AsyncLock(); using (await IndexLock.LockAsync().ConfigureAwait(false)) using (await MatureIndexFileManager.Mutex.LockAsync().ConfigureAwait(false)) using (await ImmatureIndexFileManager.Mutex.LockAsync().ConfigureAwait(false)) { IoHelpers.EnsureDirectoryExists(WorkFolderPath); await EnsureBackwardsCompatibilityAsync().ConfigureAwait(false); if (Network == Network.RegTest) { MatureIndexFileManager.DeleteMe(); // RegTest is not a global ledger, better to delete it. ImmatureIndexFileManager.DeleteMe(); } if (!MatureIndexFileManager.Exists()) { await MatureIndexFileManager.WriteAllLinesAsync(new[] { StartingFilter.ToLine() }).ConfigureAwait(false); } await InitializeFiltersAsync().ConfigureAwait(false); } } }
public async Task IoManagerTestsAsync() { var file1 = Path.Combine(Global.Instance.DataDir, nameof(IoManagerTestsAsync), $"file1.dat"); var file2 = Path.Combine(Global.Instance.DataDir, nameof(IoManagerTestsAsync), $"file2.dat"); Random random = new Random(); List <string> lines = new List <string>(); for (int i = 0; i < 1000; i++) { const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; string line = new string(Enumerable.Repeat(chars, 100) .Select(s => s[random.Next(s.Length)]).ToArray()); lines.Add(line); } // Single thread file operations. DigestableSafeMutexIoManager ioman1 = new DigestableSafeMutexIoManager(file1); // Delete the file if Exist. ioman1.DeleteMe(); Assert.False(ioman1.Exists()); Assert.False(File.Exists(ioman1.DigestFilePath)); // Write the data to the file. await ioman1.WriteAllLinesAsync(lines); Assert.True(ioman1.Exists()); // Check if the digest file is created. Assert.True(File.Exists(ioman1.DigestFilePath)); // Read back the content and check. bool IsStringArraysEqual(string[] lines1, string[] lines2) { if (lines1.Length != lines2.Length) { return(false); } for (int i = 0; i < lines1.Length; i++) { string line = lines2[i]; var readLine = lines1[i]; if (!line.Equals(readLine)) { return(false); } } return(true); } var readLines = await ioman1.ReadAllLinesAsync(); Assert.True(IsStringArraysEqual(readLines, lines.ToArray())); // Check digest file, and write only differ logic. // Write the same content, file should not be written. var currentDate = File.GetLastWriteTimeUtc(ioman1.FilePath); await Task.Delay(500); await ioman1.WriteAllLinesAsync(lines); var noChangeDate = File.GetLastWriteTimeUtc(ioman1.FilePath); Assert.Equal(currentDate, noChangeDate); // Write different content, file should be written. currentDate = File.GetLastWriteTimeUtc(ioman1.FilePath); await Task.Delay(500); lines.Add("Lorem ipsum dolor sit amet, consectetur adipiscing elit."); await ioman1.WriteAllLinesAsync(lines); var newContentDate = File.GetLastWriteTimeUtc(ioman1.FilePath); Assert.NotEqual(currentDate, newContentDate); /* The next test is commented out because on mac and on linux File.Open does not lock the file * it can be still written by the ioman1.WriteAllLinesAsync(). Tried with FileShare.None FileShare.Delete * FileStream.Lock none of them are working or caused not supported on this platform exception. * So there is no OP system way to garantee that the file won't be written during another write operation. * For example git is using lock files to solve this problem. We are using system wide mutexes. * For now there is no other way to do this. Some useful links : * https://stackoverflow.com/questions/2751734/how-do-filesystems-handle-concurrent-read-write * https://github.com/dotnet/corefx/issues/5964 */ //using (File.OpenWrite(ioman1.FilePath)) //{ // // Should be OK because the same data is written. // await ioman1.WriteAllLinesAsync(lines); //} //using (File.OpenWrite(ioman1.FilePath)) //{ // // Should fail because different data is written. // await Assert.ThrowsAsync<IOException>(async () => await ioman1.WriteAllLinesAsync(lines)); //} await ioman1.WriteAllLinesAsync(lines); // Mutex tests. // Acquire the Mutex with a background thread. var myTask = Task.Run(async() => { using (await ioman1.Mutex.LockAsync()) { await Task.Delay(3000); } }); // Wait for the Task.Run to Acquire the Mutex. await Task.Delay(100); // Try to get the Mutex and save the time. DateTime timeOfstart = DateTime.Now; DateTime timeOfAcquired = default; using (await ioman1.Mutex.LockAsync()) { timeOfAcquired = DateTime.Now; } Assert.True(myTask.IsCompletedSuccessfully); var elapsed = timeOfAcquired - timeOfstart; Assert.InRange(elapsed, TimeSpan.FromMilliseconds(2000), TimeSpan.FromMilliseconds(4000)); // Simulate file write error and recovery logic. // We have only *.new and *.old files. File.Copy(ioman1.FilePath, ioman1.OldFilePath); File.Move(ioman1.FilePath, ioman1.NewFilePath); // At this point there is now OriginalFile. var newFile = await ioman1.ReadAllLinesAsync(); Assert.True(IsStringArraysEqual(newFile, lines.ToArray())); // Add one more line to have different data. lines.Add("Lorem ipsum dolor sit amet, consectetur adipiscing elit."); await ioman1.WriteAllLinesAsync(lines); // Check recovery mechanism. Assert.True( File.Exists(ioman1.FilePath) && !File.Exists(ioman1.OldFilePath) && !File.Exists(ioman1.NewFilePath)); ioman1.DeleteMe(); Assert.False(ioman1.Exists()); // Check if directory is empty. var fileCount = Directory.EnumerateFiles(Path.GetDirectoryName(ioman1.FilePath)).Count(); Assert.Equal(0, fileCount); // Check Mutex usage on simultaneous file writes. DigestableSafeMutexIoManager ioman2 = new DigestableSafeMutexIoManager(file2); await Task.Run(async() => { using (await ioman1.Mutex.LockAsync()) { // Should not be a problem because they using different Mutexes. using (await ioman2.Mutex.LockAsync()) { await ioman1.WriteAllLinesAsync(lines); await ioman2.WriteAllLinesAsync(lines); ioman1.DeleteMe(); ioman2.DeleteMe(); } } }); // TryReplace test. var dummyFilePath = $"{ioman1.FilePath}dummy"; var dummyContent = new string[] { "banana", "peach" }; await File.WriteAllLinesAsync(dummyFilePath, dummyContent); await ioman1.WriteAllLinesAsync(lines); ioman1.TryReplaceMeWith(dummyFilePath); var fruits = await ioman1.ReadAllLinesAsync(); Assert.True(IsStringArraysEqual(dummyContent, fruits)); Assert.False(File.Exists(dummyFilePath)); ioman1.DeleteMe(); }