private void When_reading_large_buffer_from_file_it_must_succeed()
{
// Arrange
const string path = @"C:\file.txt";
const int size = 4096 * 16;
byte[] contents = CreateBuffer(size);
IFileSystem fileSystem = new FakeFileSystemBuilder()
.IncludingBinaryFile(path, contents)
.Build();
using (IFileStream stream = fileSystem.File.Open(path, FileMode.Open, FileAccess.Read))
{
var buffer = new byte[size * 2];
// Act
int numBytesRead = stream.Read(buffer, 0, buffer.Length);
// Assert
stream.Length.Should().Be(size);
stream.Position.Should().Be(size);
numBytesRead.Should().Be(size);
buffer.Take(size).SequenceEqual(contents).Should().BeTrue();
}
}
private void InitRollback()
{
byte[] btWork = new byte[8];
_fsRollbackHelper.Position = 0;
_fsRollbackHelper.Read(btWork, 0, 8);
eofRollback = btWork.To_Int64_BigEndian();
if (eofRollback == 0)
{
if (this._fsRollback.Length >= MaxRollbackFileSize)
{
this._fsRollback.Dispose();
this._configuration.FSFactory.Delete(this._fileName + ".rol");
//this._fsRollback = new FileStream(this._fileName + ".rol", FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None, _fileStreamBufferSize, FileOptions.WriteThrough);
this._fsRollback = this._configuration.FSFactory.CreateType1(this._fileName + ".rol", _fileStreamBufferSize);
//no sense to do anything with backup
}
return;
}
//!!!Check if data file is empty write first root 64 bytes, ??? Where it must stay after rollback restoration???
//Restoring rollback
RestoreInitRollback();
//Checking if we can recreate rollback file
if (this._fsRollback.Length >= MaxRollbackFileSize)
{
this._fsRollback.Dispose();
this._configuration.FSFactory.Delete(this._fileName + ".rol");
this._fsRollback = this._configuration.FSFactory.CreateType1(this._fileName + ".rol", _fileStreamBufferSize);
//File.Delete(this._fileName + ".rol");
//this._fsRollback = new FileStream(this._fileName + ".rol", FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None, _fileStreamBufferSize, FileOptions.WriteThrough);
//no sense to do anything with backup
}
eofRollback = 0;
_fsRollbackHelper.Position = 0;
_fsRollbackHelper.Write(eofRollback.To_8_bytes_array_BigEndian(), 0, 8);
NET_Flush(_fsRollbackHelper);
//try
//{
//}
//catch (Exception ex)
//{
// IsOperable = false;
// throw DBreezeException.Throw(DBreezeException.eDBreezeExceptions.RESTORE_ROLLBACK_DATA_FAILED, this._fileName, ex);
//}
}
public static byte[] ReadAllBytes([NotNull] this IFile file, [NotNull] string path)
{
Guard.NotNull(file, nameof(file));
Guard.NotNull(path, nameof(path));
using (IFileStream stream = OpenRead(file, path))
{
var buffer = new byte[stream.Length];
stream.Read(buffer, 0, buffer.Length);
return(buffer);
}
}
/// <summary>
/// RollbackHelperFileRead
/// </summary>
/// <param name="position"></param>
/// <returns></returns>
public byte[] RollbackHelperFileRead(long position, int count)
{
lock (lock_fs)
{
byte[] bt = new byte[count];
_fsRollbackHelper.Position = position;
_fsRollbackHelper.Read(bt, 0, bt.Length);
byte[] protocol = new byte[] { ProtocolVersion } //Protocol version
.Concat(bt);
return(protocol);
}
}
// Will be used for unit testing only - since Moq cannot handle Span<byte>
public string FOR_UNIT_TEST_SLOW_FUNCTION_ReadLineCharByCharTillCRLF()
{
if (_stream == null)
{
return(String.Empty);
}
string result = String.Empty;
byte[] buf = new byte[1];
int index = 0;
var pos = _stream.Position;
for (; ;)
{
if (_stream.Read(buf) != 1)
{
return(String.Empty);
}
var c = buf[0];
++index;
if (c == '\n')
{
break;
}
}
buf = new byte[index];
_stream.Seek(pos, SeekOrigin.Begin);
if (_stream.Read(buf) != index)
{
return(String.Empty);
}
string complete = System.Text.Encoding.Default.GetString(buf);
return(complete);
}
private byte[] ReadFileStream(IFileStream fileStream)
{
int lenght = fileStream.Read(_fileBuffer, 0, _fileBuffer.Length);
if (lenght == _fileBuffer.Length)
{
return(_fileBuffer);
}
else
{
byte[] buf = new byte[lenght];//当数据小于缓冲区
Array.Copy(_fileBuffer, buf, lenght);
return(buf);
}
}
bool SeekNextLineFeedInNegativeDirectionAndPositionStreamOnIt(IFileStream stream, int steps) //, bool skipNearbyCRLF = true)
{
if (_buffer == null || (_buffer.Length != steps))
{
_buffer = new byte[steps];
}
Span <byte> buffer = _buffer.AsSpan <byte>();
var initial = stream.Position;
for (; ;)
{
var current = stream.Position;
if (current == 0)
{
break;
}
int toRead = steps;
if (toRead > current)
{
toRead = (int)current;
buffer = buffer.Slice(0, toRead);
}
SetPositionRelative(stream, -toRead);
var currendMidPos0 = stream.Position;
int size = stream.Read(buffer);
var currentMidPos = stream.Position;
if (size != toRead)
{
// That shouldn't happen ???
break;
}
int index = buffer.LastIndexOf((byte)'\n');
if (index >= 0)
{
var posBefore = stream.Position;
var newPos = toRead - index;
SetPositionRelative(stream, -newPos);
var pos = stream.Position;
return(true);
}
SetPositionRelative(stream, -toRead); // Continue with next characters
}
SetPosition(stream, initial);
return(false);
}
public string SeekLastLineFromCurrentAndPositionOnStartOfItAndReturnReadLine(IFileStream stream)
{
if (!SeekLastLineFromCurrentAndPositionOnStartOfIt(stream))
{
return(null);
}
var current = stream.Position;
string result = String.Empty;
for (; ;)
{
int read = stream.Read(_buffer);
var xxxremove_me_directly = System.Text.Encoding.Default.GetString(_buffer);
Span <byte> buffer = _buffer.AsSpan <byte>();
var index = buffer.IndexOf((byte)'\n');
if (index != -1)
{
// We don't want to have a '\r' at the end of our log line
if (index > 0 && buffer[index - 1] == '\r')
{
--index;
}
if (index > 0)
{
result += System.Text.Encoding.Default.GetString(_buffer, 0, index);
}
break;
}
if ((read == buffer.Length) && (_buffer[buffer.Length - 1] == '\r'))
{
// Perhaps we haven't found a \n but it could be a \r at the end - if so don't copy \r
result += System.Text.Encoding.Default.GetString(_buffer, 0, buffer.Length - 2);
}
else
{
result += System.Text.Encoding.Default.GetString(_buffer);
}
}
SetPosition(stream, current); // Reset so we will read the next line backwards on the next call
if (current == 0 && result == String.Empty)
{
return(null); // We cannot differentiate between String.Empty nothing found and String.Empty = empty log
}
// (though by definition right now a log is not empty) but to prevent errors just return null == nothing found
return(result);
}
private void When_reading_from_closed_stream_it_must_fail()
{
// Arrange
const string path = @"C:\file.txt";
IFileSystem fileSystem = new FakeFileSystemBuilder()
.IncludingEmptyFile(path)
.Build();
using (IFileStream stream = fileSystem.File.Open(path, FileMode.Open, FileAccess.Read))
{
stream.Dispose();
var buffer = new byte[50];
// Act
// ReSharper disable once AccessToDisposedClosure
Action action = () => stream.Read(buffer, 0, buffer.Length);
// Assert
action.ShouldThrow <ObjectDisposedException>().WithMessage("Cannot access a closed file.");
}
}
/// <summary>
///
/// </summary>
/// <param name="useCache"></param>
/// <param name="offset"></param>
/// <param name="count"></param>
/// <returns></returns>
public byte[] Table_Read(bool useCache, long offset, int count)
{
byte[] res = null;
lock (lock_fs)
{
if (!useCache)
{
//WRITER
//We read exactly what is already saved, without rollback.
//all data must be either in file or in buffer
//we must form resulting byte[]
//Buffer
//Finding first element which is smaller or equal then offset
List <long> bufKeys = new List <long>();
if (_randBuf.Count() > 0)
{
var qkvp = _randBuf.OrderByDescending(r => r.Key).Where(r => r.Key < offset).Take(1).Where(r => (r.Key + r.Value.Length - 1) >= offset);
if (qkvp.Count() > 0)
{
bufKeys.Add(qkvp.FirstOrDefault().Key);
}
foreach (var kvp in _randBuf.OrderBy(r => r.Key).Where(r => r.Key >= offset && r.Key < (offset + count)))
{
bufKeys.Add(kvp.Key);
}
}
//reading full byte[] from original file and putting on top keys
//We use full length of the file
//if (offset + count > _fsData.Length + _seqBuf.EOF)
// res = new byte[_fsData.Length + _seqBuf.EOF - offset];
if (offset + count > fsLength + _seqBuf.EOF)
{
res = new byte[fsLength + _seqBuf.EOF - offset];
}
else
{
res = new byte[count];
}
byte[] btWork = null;
//if (offset < _fsData.Length)
if (offset < fsLength)
{
//Starting reading from file
_fsData.Position = offset;
if (offset + res.Length <= _fsData.Length)
{
//must be taken completely from file
_fsData.Read(res, 0, res.Length);
//Console.WriteLine("3;{0};{1}", offset, ((res == null) ? -1 : res.Length));
}
else
{
//partly from file, partly from sequential cache
//int v1 = Convert.ToInt32(_fsData.Length - offset);
int v1 = Convert.ToInt32(fsLength - offset);
_fsData.Read(res, 0, v1);
//Console.WriteLine("4;{0};{1}", offset, ((res == null) ? -1 : res.Length));
Buffer.BlockCopy(_seqBuf.RawBuffer, 0, res, v1, res.Length - v1);
}
}
else
{
//!!! threat if seqBuf is empty, should not happen thou
//completely taken from seqbuf
//Buffer.BlockCopy(_seqBuf.RawBuffer, Convert.ToInt32(offset - _fsData.Length), res, 0, res.Length);
Buffer.BlockCopy(_seqBuf.RawBuffer, Convert.ToInt32(offset - fsLength), res, 0, res.Length);
}
//putting on top
foreach (var bk in bufKeys)
{
if (offset + res.Length <= bk)
{
continue;
}
btWork = _randBuf[bk];
bool cut = false;
int start = 0;
int stop = btWork.Length;
if (bk < offset)
{
cut = true;
start = Convert.ToInt32(offset - bk);
}
if ((offset + count) < (bk + btWork.Length))
{
cut = true;
stop = Convert.ToInt32(offset + count - bk);
}
if (cut)
{
byte[] tmp = new byte[stop - start];
Buffer.BlockCopy(btWork, start, tmp, 0, tmp.Length);
btWork = tmp;
}
Buffer.BlockCopy(btWork, 0, res, (start > 0) ? 0 : Convert.ToInt32(bk - offset), btWork.Length);
}
}
else
{
//READER
//we must use rollback file.
//We can read only up to commited file lengh eofData
List <long> bufKeys = new List <long>();
if (_rollbackCache.Count() > 0)
{
var qkvp = _rollbackCache.OrderByDescending(r => r.Key).Where(r => r.Key < offset).Take(1).Where(r => (r.Key + r.Value.l - 1) >= offset);
if (qkvp.Count() > 0)
{
bufKeys.Add(qkvp.FirstOrDefault().Key);
}
foreach (var kvp in _rollbackCache.OrderBy(r => r.Key).Where(r => r.Key >= offset && r.Key < (offset + count)))
{
bufKeys.Add(kvp.Key);
}
}
//reading full byte[] from original file and putting on top keys
/*
* Transaction with minimum 2 tables. T2 is inserted, reference to T2 KVP is taken, then this reference is saved into T1.
* Commit().
* Commit calls TransactionalCommit for every table sequentially. First it meets table T1, then T2.
* In both tables TransactionalCommit procedures are successfull.
* then Commit procedure for each table calls TransactionalCommitIsFinished (this proc will clear rollback refs and moves eofData for every table).
* First encounters T1 and only then T2.
* ....Somewhere here (between calling T1 and T2 TransactionalCommitIsFinished) starts a parallel thread.
* After T1 TransactionalCommitIsFinished our parallel thread P1 reads data from T1,
* and gets SelectDirect reference to T2 KVP. Then tries to read from not yet TransactionalCommitIsFinished T2.
* and for T2 happens: eofData < offset
*
* To avoid such specific case we use for calculation this._fsData.Length instead of eofData in case if (eofData < offset && TransactionalCommitIsStarted)
* 19.07.2013 10:25
*/
//WAS
//if (offset + count > eofData)
// res = new byte[eofData - offset];
//else
// res = new byte[count];
///////
//NOW
if (offset + count > eofData)
{
if (eofData < offset && TransactionalCommitIsStarted) //NOT FINISHED multi-table COMMIT. SelectDirect case
{
//Probably not finished transaction and SelectDirect case. We return value,
//because at this momont all transaction table have successfully gone through TransactionalCommit() procedure.
//if (offset + count > this._fsData.Length)
if (offset + count > fsLength)
{
//res = new byte[this._fsData.Length - offset];
res = new byte[fsLength - offset];
}
else
{
res = new byte[count];
}
}
else
{
res = new byte[eofData - offset];
}
}
else
{
res = new byte[count];
}
///////
_fsData.Position = offset;
_fsData.Read(res, 0, res.Length);
//Console.WriteLine("1;{0};{1}", offset, ((res == null) ? -1 : res.Length));
byte[] btWork = null;
r rb = null;
//putting on top
foreach (var bk in bufKeys)
{
if (offset + res.Length <= bk)
{
continue;
}
rb = _rollbackCache[bk];
//reading from rollback
btWork = new byte[rb.l];
_fsRollback.Position = rb.o;
_fsRollback.Read(btWork, 0, btWork.Length);
bool cut = false;
int start = 0;
int stop = btWork.Length;
if (bk < offset)
{
cut = true;
start = Convert.ToInt32(offset - bk);
}
if ((offset + count) < (bk + btWork.Length))
{
cut = true;
stop = Convert.ToInt32(offset + count - bk);
}
if (cut)
{
byte[] tmp = new byte[stop - start];
Buffer.BlockCopy(btWork, start, tmp, 0, tmp.Length);
btWork = tmp;
}
Buffer.BlockCopy(btWork, 0, res, (start > 0) ? 0 : Convert.ToInt32(bk - offset), btWork.Length);
}
}
}
return(res);
}
/// <summary>
/// Is called only from lock_fs and must be finished by calling NET_Flush
/// </summary>
void FlushRandomBuffer()
{
if (_randBuf.Count() == 0)
{
return;
}
//First we write all data into rollback file and helper, calling flush on rollback
//then updating data of data file but dont call update
//clearing random buffer
//Creating rollback header
byte[] offset = null;
byte[] btRoll = null;
bool flushRollback = false;
//first loop for saving rollback data
foreach (var de in _randBuf.OrderBy(r => r.Key))
{
offset = ((ulong)de.Key).To_8_bytes_array_BigEndian().Substring(8 - DefaultPointerLen, DefaultPointerLen);
if (_rollbackCache.ContainsKey(de.Key))
{
continue;
}
//Reading from dataFile values which must be rolled back
btRoll = new byte[de.Value.Length];
_fsData.Position = de.Key;
_fsData.Read(btRoll, 0, btRoll.Length);
//Console.WriteLine("2;{0};{1}", de.Key, ((btRoll == null) ? -1 : btRoll.Length));
//Forming protocol for rollback
btRoll = new byte[] { 1 }
.ConcatMany(
offset,
((uint)btRoll.Length).To_4_bytes_array_BigEndian(),
btRoll
);
//Writing rollback
_fsRollback.Position = eofRollback;
_fsRollback.Write(btRoll, 0, btRoll.Length);
if (_backupIsActive)
{
this._configuration.Backup.WriteBackupElement(ulFileName, 1, eofRollback, btRoll);
}
_rollbackCache.Add(de.Key, new r {
o = eofRollback + 1 + offset.Length + 4, l = de.Value.Length
});
//increasing eof rollback file
eofRollback += btRoll.Length;
flushRollback = true;
}
if (flushRollback)
{
//Flushing rollback
NET_Flush(_fsRollback);
//Writing into helper
_fsRollbackHelper.Position = 0;
_fsRollbackHelper.Write(eofRollback.To_8_bytes_array_BigEndian(), 0, 8);
//Flushing rollback helper
NET_Flush(_fsRollbackHelper);
if (_backupIsActive)
{
this._configuration.Backup.WriteBackupElement(ulFileName, 2, 0, eofRollback.To_8_bytes_array_BigEndian());
this._configuration.Backup.Flush();
}
}
//second loop for saving data
foreach (var de in _randBuf.OrderBy(r => r.Key)) //sorting can mean nothing here, only takes extra time
{
_fsData.Position = de.Key;
_fsData.Write(de.Value, 0, de.Value.Length);
if (de.Key + de.Value.Length > fsLength)
{
fsLength = de.Key + de.Value.Length;
}
if (_backupIsActive)
{
this._configuration.Backup.WriteBackupElement(ulFileName, 0, de.Key, de.Value);
}
}
//No flush of data file, it will be done on Flush()
_randBuf.Clear();
usedBufferSize = 0;
}