private void InitRollback()
{
byte[] btWork = new byte[8];
_fsRollbackHelper.Position = 0;
_fsRollbackHelper.Read(btWork, 0, 8);
eofRollback = btWork.To_Int64_BigEndian();
if (eofRollback == 0)
{
if (this._fsRollback.Length >= MaxRollbackFileSize)
{
this._fsRollback.Dispose();
this._configuration.FSFactory.Delete(this._fileName + ".rol");
//this._fsRollback = new FileStream(this._fileName + ".rol", FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None, _fileStreamBufferSize, FileOptions.WriteThrough);
this._fsRollback = this._configuration.FSFactory.CreateType1(this._fileName + ".rol", _fileStreamBufferSize);
//no sense to do anything with backup
}
return;
}
//!!!Check if data file is empty write first root 64 bytes, ??? Where it must stay after rollback restoration???
//Restoring rollback
RestoreInitRollback();
//Checking if we can recreate rollback file
if (this._fsRollback.Length >= MaxRollbackFileSize)
{
this._fsRollback.Dispose();
this._configuration.FSFactory.Delete(this._fileName + ".rol");
this._fsRollback = this._configuration.FSFactory.CreateType1(this._fileName + ".rol", _fileStreamBufferSize);
//File.Delete(this._fileName + ".rol");
//this._fsRollback = new FileStream(this._fileName + ".rol", FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None, _fileStreamBufferSize, FileOptions.WriteThrough);
//no sense to do anything with backup
}
eofRollback = 0;
_fsRollbackHelper.Position = 0;
_fsRollbackHelper.Write(eofRollback.To_8_bytes_array_BigEndian(), 0, 8);
NET_Flush(_fsRollbackHelper);
//try
//{
//}
//catch (Exception ex)
//{
// IsOperable = false;
// throw DBreezeException.Throw(DBreezeException.eDBreezeExceptions.RESTORE_ROLLBACK_DATA_FAILED, this._fileName, ex);
//}
}
private void When_opening_existing_local_file_in_Append_mode_it_must_succeed()
{
// Arrange
const string path = @"C:\some\file.txt";
IFileSystem fileSystem = new FakeFileSystemBuilder()
.IncludingTextFile(path, DefaultContents)
.Build();
// Act
using (IFileStream stream = fileSystem.File.Open(path, FileMode.Append))
{
byte[] buffer =
{
(byte)'X',
(byte)'Y',
(byte)'Z'
};
stream.Write(buffer, 0, buffer.Length);
}
// Assert
IFileInfo info = fileSystem.ConstructFileInfo(path);
info.Exists.Should().BeTrue();
info.Length.Should().Be(DefaultContents.Length + 3);
fileSystem.File.ReadAllText(path).Should().Be(DefaultContents + "XYZ");
}
private void When_increasing_file_size_using_Seek_followed_by_write_it_must_fail()
{
// Arrange
const string path = @"C:\file.txt";
IFileSystem fileSystem = new FakeFileSystemBuilder()
.IncludingVolume("C:", new FakeVolumeInfoBuilder()
.OfCapacity(8192)
.WithFreeSpace(512))
.IncludingBinaryFile(path, BufferFactory.Create(32))
.Build();
using (IFileStream stream = fileSystem.File.OpenWrite(path))
{
byte[] buffer = BufferFactory.Create(64);
stream.Write(buffer, 0, buffer.Length);
stream.Seek(1280, SeekOrigin.Begin);
// Act
// ReSharper disable once AccessToDisposedClosure
Action action = () => stream.WriteByte(0x33);
// Assert
action.Should().ThrowExactly <IOException>().WithMessage("There is not enough space on the disk.");
AssertFreeSpaceOnDrive(fileSystem, "C:", 448);
}
AssertFileSize(fileSystem, path, 64);
}
private void When_writing_large_buffer_to_file_it_must_succeed()
{
// Arrange
const string path = @"C:\file.txt";
IFileSystem fileSystem = new FakeFileSystemBuilder()
.Build();
const int size = 4096 * 16;
byte[] writeBuffer = CreateBuffer(size);
using (IFileStream stream = fileSystem.File.Create(path, 2048))
{
// Act
stream.Write(writeBuffer, 0, writeBuffer.Length);
// Assert
stream.Length.Should().Be(size);
stream.Position.Should().Be(size);
}
byte[] contents = fileSystem.File.ReadAllBytes(path);
contents.SequenceEqual(writeBuffer).Should().BeTrue();
}
private void InitFiles()
{
//Creates filestreams and rollbacks, restores rollback to the initial file, if necessary
try
{
//this._fsData = new FileStream(this._fileName, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None, _fileStreamBufferSize, FileOptions.WriteThrough);
//this._fsRollback = new FileStream(this._fileName + ".rol", FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None, _fileStreamBufferSize, FileOptions.WriteThrough);
//this._fsRollbackHelper = new FileStream(this._fileName + ".rhp", FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None, _fileStreamBufferSize, FileOptions.WriteThrough);
this._fsData = this._configuration.FSFactory.CreateType1(this._fileName, _fileStreamBufferSize);
this._fsRollback = this._configuration.FSFactory.CreateType1(this._fileName + ".rol", _fileStreamBufferSize);
this._fsRollbackHelper = this._configuration.FSFactory.CreateType1(this._fileName + ".rhp", _fileStreamBufferSize);
//!!!!We dont have this value in root yet, could have and economize tail of the file in case if rollback occured
if (this._fsData.Length == 0)
{
//Writing initial root data
_fsData.Position = 0;
_fsData.Write(new byte[this._trieSettings.ROOT_SIZE], 0, this._trieSettings.ROOT_SIZE);
if (_backupIsActive)
{
this._configuration.Backup.WriteBackupElement(ulFileName, 0, 0, new byte[this._trieSettings.ROOT_SIZE]);
}
//no flush here
}
eofData = this._fsData.Length;
fsLength = this._fsData.Length;
//Check is .rhp is empty add 0 pointer
if (this._fsRollbackHelper.Length == 0)
{
//no sense to write here
//_fsRollbackHelper.Position = 0;
//_fsRollbackHelper.Write(eofRollback.To_8_bytes_array_BigEndian(), 0, 8);
//NET_Flush(_fsRollbackHelper);
}
else
{
InitRollback();
}
_storageFixTime = DateTime.UtcNow;
}
catch (Exception ex)
{
IsOperable = false;
throw DBreezeException.Throw(DBreezeException.eDBreezeExceptions.DB_IS_NOT_OPERABLE, "FSR INIT FAILED: " + this._fileName, ex);
}
}
public static void WriteAllBytes([NotNull] this IFile file, [NotNull] string path, [NotNull] byte[] bytes)
{
Guard.NotNull(file, nameof(file));
Guard.NotNull(path, nameof(path));
Guard.NotNull(bytes, nameof(bytes));
using (IFileStream stream = file.OpenWrite(path))
{
stream.Write(bytes, 0, bytes.Length);
}
}
/// <summary>
/// Must be called from lock_fs
/// </summary>
void FlushSequentialBuffer()
{
if (_seqBuf.EOF == 0)
{
return;
}
long pos = _fsData.Length;
_fsData.Position = pos;
_fsData.Write(_seqBuf.RawBuffer, 0, _seqBuf.EOF);
if (_backupIsActive)
{
byte[] btWork = new byte[_seqBuf.EOF];
Buffer.BlockCopy(_seqBuf.RawBuffer, 0, btWork, 0, btWork.Length);
this._configuration.Backup.WriteBackupElement(ulFileName, 0, pos, btWork);
}
_seqBuf.Clear(true);
}
/// <summary>
/// Represents incremental backup of protocol 1
/// </summary>
/// <param name="fileNumber"></param>
/// <param name="type">0 - table file, 1 - rollback file, 2 - rollbackhelper, 3 - recreate table file, 5 - removing complete table </param>
/// <param name="pos"></param>
/// <param name="data"></param>
internal void WriteBackupElement(ulong fileNumber, byte type, long pos, byte[] data)
{
lock (lock_ibp_fs)
{
this.GetFileStream();
uint size = 0;
byte[] toSave = null;
switch (type)
{
case 0:
case 1:
//8(fileNumber)+1(type)+8(position)+data.Length
size = Convert.ToUInt32(8 + 1 + 8 + data.Length);
toSave = size.To_4_bytes_array_BigEndian().ConcatMany(fileNumber.To_8_bytes_array_BigEndian(),
new byte[] { type },
pos.To_8_bytes_array_BigEndian(),
data);
break;
case 2:
//Now we save new information into rollback
size = Convert.ToUInt32(8 + 1 + 8 + data.Length);
toSave = size.To_4_bytes_array_BigEndian().ConcatMany(fileNumber.To_8_bytes_array_BigEndian(),
new byte[] { 2 },
((long)0).To_8_bytes_array_BigEndian(),
data);
break;
case 3: //recreate table file
case 5: //removing complete table
//8(fileNumber)+1(type)
size = 9;
toSave = size.To_4_bytes_array_BigEndian()
.ConcatMany(
fileNumber.To_8_bytes_array_BigEndian(),
new byte[] { type }
);
break;
}
wasWritten = true;
fs.Write(toSave, 0, toSave.Length);
}
//Console.WriteLine(String.Format("{0}> FN: {1} - {2}; at {3} q {4}", writeTime.ToString("dd.MM.yyyy HH:mm:ss"), fileNumber, type.ToString(), pos.ToString(),
// data.Length.ToString()));
}
/// <summary>
/// RollbackFileWrite
/// </summary>
/// <param name="position"></param>
/// <param name="withFlush"></param>
/// <param name="data"></param>
/// <returns></returns>
public byte[] RollbackHelperFileWrite(long position, bool withFlush, byte[] data)
{
lock (lock_fs)
{
_fsRollbackHelper.Position = position;
_fsRollbackHelper.Write(data, 0, data.Length);
if (withFlush)
{
FSR.NET_Flush(_fsRollbackHelper);
}
byte[] protocol = new byte[] { ProtocolVersion } //Protocol version
.Concat(BitConverter.GetBytes(_fsRollbackHelper.Length));
return(protocol);
}
}
private void When_writing_empty_buffer_to_file_it_must_succeed()
{
// Arrange
const string path = @"C:\file.txt";
IFileSystem fileSystem = new FakeFileSystemBuilder()
.Build();
using (IFileStream stream = fileSystem.File.Create(path))
{
// Act
stream.Write(new byte[] { 0xAA }, 0, 0);
// Assert
stream.Length.Should().Be(0);
stream.Position.Should().Be(0);
}
}
private void When_writing_to_readonly_stream_it_must_fail()
{
// Arrange
const string path = @"C:\file.txt";
IFileSystem fileSystem = new FakeFileSystemBuilder()
.IncludingEmptyFile(path)
.Build();
using (IFileStream stream = fileSystem.File.Open(path, FileMode.Open, FileAccess.Read))
{
// Act
// ReSharper disable once AccessToDisposedClosure
Action action = () => stream.Write(new byte[] { 0xFF }, 0, 1);
// Assert
action.ShouldThrow <NotSupportedException>().WithMessage("Stream does not support writing.");
}
}
private void When_writing_to_new_file_with_DeleteOnClose_it_must_succeed()
{
// Arrange
IFileSystem fileSystem = new FakeFileSystemBuilder()
.IncludingVolume("C:", new FakeVolumeInfoBuilder()
.OfCapacity(8192)
.WithFreeSpace(4096))
.Build();
// Act
using (IFileStream stream = fileSystem.File.Create(@"C:\file.txt", options: FileOptions.DeleteOnClose))
{
byte[] buffer = BufferFactory.Create(1024);
stream.Write(buffer, 0, buffer.Length);
}
// Assert
IDriveInfo driveInfo = fileSystem.ConstructDriveInfo("C:");
driveInfo.AvailableFreeSpace.Should().Be(4096);
}
private void When_opening_existing_local_file_in_Append_mode_it_must_succeed()
{
// Arrange
const string path = @"C:\some\file.txt";
DateTime creationTimeUtc = 9.September(2016);
var clock = new SystemClock
{
UtcNow = () => creationTimeUtc
};
IFileSystem fileSystem = new FakeFileSystemBuilder(clock)
.IncludingTextFile(path, DefaultContents)
.Build();
DateTime lastWriteTimeUtc = 12.September(2016);
clock.UtcNow = () => lastWriteTimeUtc;
// Act
using (IFileStream stream = fileSystem.File.Open(path, FileMode.Append))
{
byte[] buffer = { (byte)'X', (byte)'Y', (byte)'Z' };
stream.Write(buffer, 0, buffer.Length);
}
// Assert
IFileInfo info = fileSystem.ConstructFileInfo(path);
info.Exists.Should().BeTrue();
info.Length.Should().Be(DefaultContents.Length + 3);
info.CreationTimeUtc.Should().Be(creationTimeUtc);
info.LastAccessTimeUtc.Should().Be(lastWriteTimeUtc);
info.LastWriteTimeUtc.Should().Be(lastWriteTimeUtc);
fileSystem.File.ReadAllText(path).Should().Be(DefaultContents + "XYZ");
}
/// <summary>
/// 下载文件
/// </summary>
/// <param name="fileStream">文件流</param>
/// <param name="remoteFileName">远程文件名</param>
/// <returns></returns>
public async Task DownloadFile(IFileStream fileStream, string remoteFileName)
{
//LogHelper.DebugWriteLog("begin download frist Data fileName:" + Path.GetFileName(localFileName));
long position = fileStream.Length;
reset:
var responsed = await this.AwaitOpenDownloadData(remoteFileName, position);//首数据包，带文件状态信息及文件分块
var status = 0;
//返回null表示已断开连接
if (responsed.IsNull())
{
var positionNull = await this.AwaitResetDownloadFile(fileStream);//等待重新连接
if (positionNull.HasValue)
{
position = positionNull.Value;
goto reset;
}
status = 0;
}
else if (responsed.Status == 1) //成功打开文件
{
await Task.Run(() => fileStream.Write(responsed.Data, 0, responsed.Data.Length));
status = 1;
}
else
{
status = responsed.Status;//文件访问失败
}
this.OnFileTransferProgressEventHandler?.Invoke(this, FileTransferFlag.Begin, remoteFileName, position, responsed.FileSize);
while (status == 1)
{
if (TransferTaskFlage == TransferTaskFlage.Abort)
{
//停止传输
this.RemoteTaskStop();
break;
}
if (fileStream.Length >= responsed.FileSize)
{
break;//文件传输完成
}
var data = await this.AwaitDownloadDataPack().ConfigureAwait(true);
//LogHelper.DebugWriteLog("download data:" + (data == null ? "null" : data.Data.Length.ToString()));
if (this.WhetherClose)
{
break;//传输中途关闭应用
}
if (data.IsNull())
{
var positionNull = await this.AwaitResetDownloadFile(fileStream).ConfigureAwait(true);
if (!positionNull.HasValue)
{
break;//session断开期间应用被关闭
}
position = positionNull.Value;
goto reset;
}
await Task.Run(() => fileStream.Write(data.Data, 0, data.Data.Length));
this.OnFileTransferProgressEventHandler?.Invoke(this, FileTransferFlag.Transfering, remoteFileName, fileStream.Length, responsed.FileSize);
}
this.OnFileTransferProgressEventHandler?.Invoke(this, FileTransferFlag.End, remoteFileName, fileStream.Length, responsed.FileSize);
fileStream.Close();
}
/// <summary>
/// Is called only from lock_fs and must be finished by calling NET_Flush
/// </summary>
void FlushRandomBuffer()
{
if (_randBuf.Count() == 0)
{
return;
}
//First we write all data into rollback file and helper, calling flush on rollback
//then updating data of data file but dont call update
//clearing random buffer
//Creating rollback header
byte[] offset = null;
byte[] btRoll = null;
bool flushRollback = false;
//first loop for saving rollback data
foreach (var de in _randBuf.OrderBy(r => r.Key))
{
offset = ((ulong)de.Key).To_8_bytes_array_BigEndian().Substring(8 - DefaultPointerLen, DefaultPointerLen);
if (_rollbackCache.ContainsKey(de.Key))
{
continue;
}
//Reading from dataFile values which must be rolled back
btRoll = new byte[de.Value.Length];
_fsData.Position = de.Key;
_fsData.Read(btRoll, 0, btRoll.Length);
//Console.WriteLine("2;{0};{1}", de.Key, ((btRoll == null) ? -1 : btRoll.Length));
//Forming protocol for rollback
btRoll = new byte[] { 1 }
.ConcatMany(
offset,
((uint)btRoll.Length).To_4_bytes_array_BigEndian(),
btRoll
);
//Writing rollback
_fsRollback.Position = eofRollback;
_fsRollback.Write(btRoll, 0, btRoll.Length);
if (_backupIsActive)
{
this._configuration.Backup.WriteBackupElement(ulFileName, 1, eofRollback, btRoll);
}
_rollbackCache.Add(de.Key, new r {
o = eofRollback + 1 + offset.Length + 4, l = de.Value.Length
});
//increasing eof rollback file
eofRollback += btRoll.Length;
flushRollback = true;
}
if (flushRollback)
{
//Flushing rollback
NET_Flush(_fsRollback);
//Writing into helper
_fsRollbackHelper.Position = 0;
_fsRollbackHelper.Write(eofRollback.To_8_bytes_array_BigEndian(), 0, 8);
//Flushing rollback helper
NET_Flush(_fsRollbackHelper);
if (_backupIsActive)
{
this._configuration.Backup.WriteBackupElement(ulFileName, 2, 0, eofRollback.To_8_bytes_array_BigEndian());
this._configuration.Backup.Flush();
}
}
//second loop for saving data
foreach (var de in _randBuf.OrderBy(r => r.Key)) //sorting can mean nothing here, only takes extra time
{
_fsData.Position = de.Key;
_fsData.Write(de.Value, 0, de.Value.Length);
if (de.Key + de.Value.Length > fsLength)
{
fsLength = de.Key + de.Value.Length;
}
if (_backupIsActive)
{
this._configuration.Backup.WriteBackupElement(ulFileName, 0, de.Key, de.Value);
}
}
//No flush of data file, it will be done on Flush()
_randBuf.Clear();
usedBufferSize = 0;
}
private void InitFiles()
{
//Creates filestreams and rollbacks, restores rollback to the initial file, if necessary
try
{
//this._fsData = new FileStream(this._fileName, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None, _fileStreamBufferSize, FileOptions.WriteThrough);
//this._fsRollback = new FileStream(this._fileName + ".rol", FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None, _fileStreamBufferSize, FileOptions.WriteThrough);
//this._fsRollbackHelper = new FileStream(this._fileName + ".rhp", FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None, _fileStreamBufferSize, FileOptions.WriteThrough);
this._fsData = this._configuration.FSFactory.CreateType1(this._fileName, _fileStreamBufferSize);
this._fsRollback = this._configuration.FSFactory.CreateType1(this._fileName + ".rol", _fileStreamBufferSize);
this._fsRollbackHelper = this._configuration.FSFactory.CreateType1(this._fileName + ".rhp", _fileStreamBufferSize);
//!!!!We dont have this value in root yet, could have and economize tail of the file in case if rollback occured
if (this._fsData.Length == 0)
{
//Writing initial root data
_fsData.Position = 0;
_fsData.Write(new byte[this._trieSettings.ROOT_SIZE], 0, this._trieSettings.ROOT_SIZE);
if (_backupIsActive)
{
this._configuration.Backup.WriteBackupElement(ulFileName, 0, 0, new byte[this._trieSettings.ROOT_SIZE]);
}
//no flush here
}
eofData = this._fsData.Length;
//Check is .rhp is empty add 0 pointer
if (this._fsRollbackHelper.Length == 0)
{
//no sense to write here
//_fsRollbackHelper.Position = 0;
//_fsRollbackHelper.Write(eofRollback.To_8_bytes_array_BigEndian(), 0, 8);
//NET_Flush(_fsRollbackHelper);
}
else
{
InitRollback();
}
_storageFixTime = DateTime.UtcNow;
}
catch (Exception ex)
{
IsOperable = false;
throw DBreezeException.Throw(DBreezeException.eDBreezeExceptions.DB_IS_NOT_OPERABLE, "FSR INIT FAILED: " + this._fileName, ex);
}
}
private static void WriteBytesToFile([NotNull] IFileStream stream, [NotNull] byte[] buffer)
{
stream.Write(buffer, 0, buffer.Length);
}
private void DoPackage(byte[] pack)
{
ulong fileNumber = pack.Substring(0, 8).To_UInt64_BigEndian();
byte type = pack.Substring(8, 1)[0];
string filename = BackupFNP.ParseFilenameBack(fileNumber);
long offset = 0;
byte[] data = null;
//Console.WriteLine("t: {0}", type);
//types description
// 0 - table file, 1 - rollback file, 2 - rollbackhelper, 3 - recreate table file (only table file), 4 - recreate rollback file (only rollback file), 5 - removing complete table
IFileStream lfs = null;
bool contains = false;
string tfn = String.Empty;
switch (type)
{
case 0:
//Write into table file
lfs = this.GetFileStream(filename);
//if (lfs == null)
//{
// Console.WriteLine("Backup lfs = null");
// return;
//}
offset = pack.Substring(9, 8).To_Int64_BigEndian();
data = pack.Substring(17);
lfs.Position = offset;
lfs.Write(data, 0, data.Length);
lfs.Flush();
break;
case 1:
//write into rollback file
lfs = this.GetFileStream(filename + ".rol");
if (lfs == null)
{
System.Diagnostics.Debug.WriteLine("Backup lfs = null");
return;
}
offset = pack.Substring(9, 8).To_Int64_BigEndian();
data = pack.Substring(17);
lfs.Position = offset;
lfs.Write(data, 0, data.Length);
lfs.Flush();
break;
case 2:
//write into rollbackhelper
lfs = this.GetFileStream(filename + ".rhp");
if (lfs == null)
{
System.Diagnostics.Debug.WriteLine("Backup lfs = null");
return;
}
offset = pack.Substring(9, 8).To_Int64_BigEndian();
data = pack.Substring(17);
lfs.Position = offset;
lfs.Write(data, 0, data.Length);
lfs.Flush();
break;
case 3:
//3 - recreate table file (only table file)
contains = ds.ContainsKey(filename);
if (contains)
{
ds[filename].Dispose();
}
tfn = Path.Combine(this.DataBaseFolder, filename);
this.configuration.FSFactory.Delete(tfn);
if (!contains)
{
ds.Add(filename, null);
}
ds[filename] = this.configuration.FSFactory.CreateType2(tfn); // new FileStream(tfn, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None);
break;
case 4:
//4 - recreate rollback file (only rollback file)
contains = ds.ContainsKey(filename + ".rol");
if (contains)
{
ds[filename + ".rol"].Dispose();
}
tfn = Path.Combine(this.DataBaseFolder, filename + ".rol");
this.configuration.FSFactory.Delete(tfn);
if (!contains)
{
ds.Add(filename + ".rol", null);
}
ds[filename + ".rol"] = this.configuration.FSFactory.CreateType2(tfn); //new FileStream(tfn, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None);
break;
case 5:
//5 - removing complete table
if (ds.ContainsKey(filename))
{
ds[filename].Dispose();
}
ds.Remove(filename);
if (ds.ContainsKey(filename + ".rol"))
{
ds[filename + ".rol"].Dispose();
}
ds.Remove(filename + ".rol");
if (ds.ContainsKey(filename + ".rhp"))
{
ds[filename + ".rhp"].Dispose();
}
ds.Remove(filename + ".rhp");
break;
}
}
