public void TryCopyingANonEmptyFileComparingTheContentsOfBothCopiesAfterwards()
{
VirtualFileSystem fileSystem = VirtualFileSystemFactory.CreateDefaultFileSystem();
byte[] interestingData = ByteBufferFactory.BuildSomeGuidsIntoByteArray(10000);
var greatGatsby = fileSystem.CreateFile(@"\TheGreatGatsbyEncoded.bin");
using (var stream = fileSystem.OpenFileForWriting(greatGatsby.FullPath))
{
stream.Write(interestingData, 0, interestingData.Length);
}
var folderInfo = fileSystem.CreateFolder(@"\AifdjgodfjgodfjglidflgjalglkrelgkrelllerlnlrklgnLKGLKREL");
fileSystem.CopyFile(greatGatsby.FullPath,
fileSystem.PathBuilder.CombinePaths(folderInfo.FullPath, "HiThereSport.txt"));
var interestingDataAsReadFromFile = new byte[interestingData.Length];
using (var stream = fileSystem.OpenFileForReading(fileSystem.PathBuilder.CombinePaths(folderInfo.FullPath, "HiThereSport.txt")))
{
stream.Read(interestingDataAsReadFromFile, 0, stream.Length);
}
CollectionAssert.AreEqual(interestingData, interestingDataAsReadFromFile);
}
public void MakeSureDeletingAFileFreesSpaceForSure() // был такой баг.
{
var collaborators = TestCollaboratorsFactory.CreateAllCollaborators(50000, false);
VirtualFileSystem fileSystem = collaborators.VirtualFileSystem;
int freeSpaceBeforeAnything = fileSystem.FreeSpaceInBytes;
byte[] interestingData = ByteBufferFactory.BuildSomeGuidsIntoByteArray(10000);
using (var dataStream = fileSystem.CreateAndOpenFileForWriting(@"\hey"))
{
for (int i = 0; i < 50; i++)
{
dataStream.Write(interestingData, 0, interestingData.Length);
}
}
fileSystem.DeleteFile(@"\hey");
Assert.AreEqual(freeSpaceBeforeAnything, fileSystem.FreeSpaceInBytes);
fileSystem = TestCollaboratorsFactory.CreateFileSystemFromExistingStream(collaborators.Stream);
Assert.AreEqual(freeSpaceBeforeAnything, fileSystem.FreeSpaceInBytes);
}
private void ReceiveAsync(IAsyncResult res)
{
if (State == SocketState.Disconnected)
{
return;
}
byte[] buf = null;
IPEndPoint remoteIpEndPoint = new IPEndPoint(IPAddress.Any, 8000);
if (_client == null)
{
return;
}
buf = _client.EndReceive(res, ref remoteIpEndPoint);
if (buf == null || buf.Length < UdpPeer.MtuHeaderLength || buf.Length > _mtuSize)
{
throw new Exception(string.Format("The length of received data[{0}] is out of range between {1} and {2}", buf.Length, UdpPeer.MtuHeaderLength, _mtuSize));
}
var buffer = ByteBufferFactory.NewBuffer(buf);
_broker.OnReceive(buffer);
if (_client == null)
{
return;
}
_client.BeginReceive(ReceiveAsync, null);
}
private async Task <IByteBuffer> ReceiveBufferAsync()
{
try
{
var messageSizeBytes = new byte[4];
await _stream.ReadBytesAsync(messageSizeBytes, 0, 4, _backgroundTaskCancellationToken).ConfigureAwait(false);
var messageSize = BitConverter.ToInt32(messageSizeBytes, 0);
var inputBufferChunkSource = new InputBufferChunkSource(BsonChunkPool.Default);
var buffer = ByteBufferFactory.Create(inputBufferChunkSource, messageSize);
buffer.Length = messageSize;
buffer.SetBytes(0, messageSizeBytes, 0, 4);
await _stream.ReadBytesAsync(buffer, 4, messageSize - 4, _backgroundTaskCancellationToken).ConfigureAwait(false);
_lastUsedAtUtc = DateTime.UtcNow;
buffer.MakeReadOnly();
return(buffer);
}
catch (Exception ex)
{
var wrappedException = WrapException(ex, "receiving a message from the server");
ConnectionFailed(wrappedException);
throw wrappedException;
}
}
private async Task <bool> ReceiveBackgroundTask(CancellationToken cancellationToken)
{
try
{
var messageSizeBytes = new byte[4];
await _stream.FillBufferAsync(messageSizeBytes, 0, 4, cancellationToken).ConfigureAwait(false);
var messageSize = BitConverter.ToInt32(messageSizeBytes, 0);
var buffer = ByteBufferFactory.Create(BsonChunkPool.Default, messageSize);
buffer.WriteBytes(0, messageSizeBytes, 0, 4);
await _stream.FillBufferAsync(buffer, 4, messageSize - 4, cancellationToken).ConfigureAwait(false);
_lastUsedAtUtc = DateTime.UtcNow;
var responseToBytes = new byte[4];
buffer.ReadBytes(8, responseToBytes, 0, 4);
var responseTo = BitConverter.ToInt32(responseToBytes, 0);
_inboundDropbox.Post(responseTo, buffer);
return(true);
}
catch (Exception ex)
{
ConnectionFailed(ex);
return(false);
}
}
public void Disconnect(DisconnectReason reason, bool sendingRequired)
{
if (_udpSocket.State == SocketState.Disconnecting || _udpSocket.State == SocketState.Disconnected)
{
return;
}
Log(LogLevel.Debug, "Disconnect Reson[{0}]", reason.ToString());
if (sendingRequired)
{
_udpSocket.Disconnecting();
var buffer = ByteBufferFactory.NewBuffer();
buffer.WriteInt((int)reason);
EnqueueOutgoingCommand(new OutgoingCommand(CommandType.Disconnect, (byte)(ChannelCount - 1), buffer.ToArray()));
FlushSendQueues();
}
_udpSocket.Dispose();
if (_listener != null)
{
switch (reason)
{
case DisconnectReason.Timeout:
case DisconnectReason.ServerUserLimit:
case DisconnectReason.ServerLogic:
case DisconnectReason.QueueFull:
case DisconnectReason.InvalidConnection:
case DisconnectReason.InvalidDataFormat:
case DisconnectReason.ClientDisconnect:
case DisconnectReason.ApplicationStop:
_listener.OnStatusChanged(StatusCode.Disconnected, string.Format("Disconnected by {0}", reason));
break;
case DisconnectReason.ConnectionFailed:
_listener.OnStatusChanged(StatusCode.FailedToConnect, string.Format("Failed to connect to {0}:{1}", RemoteEndPoint.Address, RemoteEndPoint.Port));
break;
default:
_listener.OnStatusChanged(StatusCode.Disconnected, "Disconnected by unknown error");
break;
}
_listener.OnClose();
}
if (_sendAckBackgroundThread != null)
{
_sendAckBackgroundThread.Abort();
_sendAckBackgroundThread = null;
}
LogMessageRecevied = null;
_isAckThreadRunning = false;
}
public IndexPopulationJob(MultipleIndexPopulator multiPopulator, IndexingService.Monitor monitor, bool verifyBeforeFlipping)
{
this._multiPopulator = multiPopulator;
this._monitor = monitor;
this._verifyBeforeFlipping = verifyBeforeFlipping;
this._memoryAllocationTracker = new ThreadSafePeakMemoryAllocationTracker(GlobalMemoryTracker.INSTANCE);
this._bufferFactory = new ByteBufferFactory(() => new UnsafeDirectByteBufferAllocator(_memoryAllocationTracker), parseBlockSize());
}
public void Create_should_return_expected_result(
[Values(1, 63, 64, 65, 128)]
int minimumCapacity)
{
var chunkSource = new BsonChunkPool(1, 64);
var result = ByteBufferFactory.Create(chunkSource, minimumCapacity);
result.Capacity.Should().BeGreaterOrEqualTo(minimumCapacity);
}
public void Create_should_return_SingleChunkBuffer_when_a_single_chunk_is_sufficient(
[Values(1, 63, 64)]
int minimumCapacity)
{
var chunkSource = new BsonChunkPool(1, 64);
var result = ByteBufferFactory.Create(chunkSource, minimumCapacity);
result.Should().BeOfType <SingleChunkBuffer>();
}
public void Create_should_return_MultiChunkBuffer_when_multiple_chunks_are_required(
[Values(65, 128)]
int minimumCapacity)
{
var chunkSource = new BsonChunkPool(1, 64);
var result = ByteBufferFactory.Create(chunkSource, minimumCapacity);
result.Should().BeOfType <MultiChunkBuffer>();
}
public void Create_should_throw_when_minimumCapacity_is_invalid(
[Values(-1, 0)]
int minimumCapacity)
{
var mockChunkSource = new Mock <IBsonChunkSource>();
Action action = () => ByteBufferFactory.Create(mockChunkSource.Object, minimumCapacity);
action.ShouldThrow <ArgumentOutOfRangeException>().And.ParamName.Should().Be("minimumCapacity");
}
internal OutgoingCommand(CommandType commandType, byte channel, bool encryted, byte[] payload)
{
ReliableSequenceNumber = UdpChannel.InitialSequenceNumber;
UnreliableSequenceNumber = UdpChannel.InitialSequenceNumber;
CommandType = commandType;
Channel = channel;
Payload = payload != null && payload.Length > 0 ? ByteBufferFactory.NewBuffer(payload) : ByteBufferFactory.NewBuffer();
switch (CommandType)
{
case CommandType.Acknowledge:
_headerLength = (int)Lengths.AcknowledgeHeader;
Flags = CommandFlags.Unreliable;
break;
case CommandType.Disconnect:
_headerLength = (int)Lengths.DisconnectHeader;
Flags = CommandFlags.Reliable;
break;
case CommandType.Ping:
_headerLength = (int)Lengths.PingHeader;
Flags = CommandFlags.Reliable;
break;
case CommandType.Reliable:
_headerLength = (int)Lengths.ReliableHeader;
Flags = encryted ? CommandFlags.Reliable | CommandFlags.Encrypted : CommandFlags.Reliable;
break;
case CommandType.Unreliable:
_headerLength = (int)Lengths.UnreliableHeader;
Flags = encryted ? CommandFlags.Unreliable | CommandFlags.Encrypted : CommandFlags.Unreliable;
break;
case CommandType.Fragmented:
_headerLength = (int)Lengths.FragmentHeader;
Flags = encryted ? CommandFlags.Reliable | CommandFlags.Encrypted : CommandFlags.Unreliable;
break;
case CommandType.SNTP:
_headerLength = (int)Lengths.SntpHeader;
Flags = CommandFlags.Reliable;
break;
default:
throw new ArgumentException(string.Format("Unknown CommandType: {0}", commandType));
}
Size = Payload.Count > 0 ? (short)(_headerLength + Payload.Count) : _headerLength;
}
public void TestGetMultipleChunkSlice()
{
var chunkSize = BsonChunkPool.Default.ChunkSize;
var length = chunkSize * 3;
using (var buffer = ByteBufferFactory.Create(BsonChunkPool.Default, length))
{
buffer.MakeReadOnly();
var slice = buffer.GetSlice(chunkSize, chunkSize + 1);
Assert.IsInstanceOf <MultiChunkBuffer>(slice);
}
}
public void TestGetSingleChunkSlice()
{
var chunkSize = BsonChunkPool.Default.ChunkSize;
var capacity = chunkSize * 3;
using (var buffer = ByteBufferFactory.Create(BsonChunkPool.Default, capacity))
{
buffer.Length = capacity;
buffer.MakeReadOnly();
var slice = buffer.GetSlice(chunkSize, 1);
Assert.IsInstanceOf <SingleChunkBuffer>(slice);
}
}
public static OutgoingCommand CreateAck(this IncomingCommand command)
{
if (false == command.IsReliable)
{
return(null);
}
var payload = ByteBufferFactory.NewBuffer();
payload.WriteLong(command.ReliableSequenceNumber);
payload.WriteLong(command.ServerSentTime);
return(new OutgoingCommand(CommandType.Acknowledge, command.Channel, payload.ToArray()));
}
public void MakeSureReadDoesNotThrowIfReadingBeyondTheStream()
{
DataStreamReadableWritable dataStream = TestCollaboratorsFactory.CreateEmptyDataStream();
byte[] bytes = ByteBufferFactory.CreateByteBufferWithAllBytesSet(55000, 123);
dataStream.Write(bytes, 0, bytes.Length);
dataStream.SetPosition(55000 - 35);
var largeArray = new byte[2000];
Assert.AreEqual(35, dataStream.Read(largeArray, 0, 2000));
}
public void MakeSureShrinkingFreesDiskSpace()
{
var testCollaborators = TestCollaboratorsFactory.CreateCollaboratorsForTestingDataStreams();
DataStreamReadableWritable dataStream =
testCollaborators.VirtualFileSystem.OpenFileForWriting(testCollaborators.FileInfo.FullPath);
int valueCloseToDiskSize = (int)testCollaborators.Disk.DiskSizeInBytes - 25 * testCollaborators.Disk.BlockSizeInBytes;
dataStream.SetLength(valueCloseToDiskSize);
var fileSystem = testCollaborators.VirtualFileSystem;
FileInfo newFileNode = fileSystem.CreateFile(@"\testFile2");
DataStreamReadableWritable newFileNodeStream = fileSystem.OpenFileForWriting(newFileNode.FullPath);
ExceptionAssert.MakeSureExceptionIsRaisedBy <InsufficientSpaceException>(
delegate
{
newFileNodeStream.SetLength(valueCloseToDiskSize);
});
dataStream.SetLength(0);
newFileNodeStream.SetLength(valueCloseToDiskSize);
dataStream.SetPosition(0);
const int answeringToAllQuestionsByte = 42;
byte[] bytesToWrite = ByteBufferFactory.CreateByteBufferWithAllBytesSet(valueCloseToDiskSize, answeringToAllQuestionsByte);
newFileNodeStream.Write(bytesToWrite, 0, bytesToWrite.Length);
newFileNodeStream.SetPosition(0);
var buffer = new byte[valueCloseToDiskSize];
newFileNodeStream.Read(buffer, 0, buffer.Length);
MakeSureEachByteInArrayIsRight(buffer, answeringToAllQuestionsByte);
}
public EventMessage(short code, DataObject parameters, INetworkPeer peer, bool encrypt)
{
var buffer = ByteBufferFactory.NewBuffer();
// Write a message header (2 bytes).
buffer.WriteByte(DataSerializer.Version);
buffer.WriteByte((byte)MessageType.EventMessage);
buffer.WriteShort(code);
buffer.WriteDataObject(parameters);
if (encrypt)
{
byte[] bytes = buffer.ToArray();
_bytes = peer.Cipher.Encrypt(bytes);
}
else
{
_bytes = buffer.ToArray();
}
}
/// <summary>
/// Fetch a payload. If a incoming command is invalid, return null.
/// </summary>
/// <returns>A payload in a incoming command.</returns>
private IByteBuffer FetchPayload(IncomingCommand command)
{
if (false == command.IsPayloadAvailable())
{
return(null);
}
// The length of InitialResponse is 2. Thus payload length should be bigger then 2.
if (command.GetPayload().Length < 2)
{
Log(LogLevel.Error, "Incoming UDP data is too short; Length [{0}] ", command.GetPayload().Length);
return(null);
}
if (command.IsEncrypted)
{
try
{
command.SetPayload(_cipher.Decrypt(command.GetPayload()));
}
catch (Exception)
{
Log(LogLevel.Error, "Invalid encrypted data");
return(null);
}
}
var payload = ByteBufferFactory.NewBuffer(command.GetPayload());
byte serializerVersion = payload.ReadByte();
if (serializerVersion != DataSerializer.Version)
{
Log(LogLevel.Error, "Unknown serializer Version");
return(null);
}
return(payload);
}
protected void ReceiveFromLoop()
{
while (State != SocketState.Disconnected)
{
try
{
IPEndPoint remoteEndPoint = new IPEndPoint(IPAddress.Any, 0);
byte[] buf = _client.Receive(ref remoteEndPoint);
if (buf == null || buf.Length < UdpPeer.MtuHeaderLength || buf.Length > _mtuSize)
{
throw new Exception(string.Format("The length of received data[{0}] is out of range between {1} and {2}", buf.Length, UdpPeer.MtuHeaderLength, _mtuSize));
}
var buffer = ByteBufferFactory.NewBuffer(buf);
_broker.OnReceive(buffer);
}
catch (SocketException ex)
{
if (State != SocketState.Disconnecting && State != SocketState.Disconnected)
{
_broker.Log(LogLevel.Error, ex.ToString());
}
break;
}
catch (Exception ex)
{
if (State != SocketState.Disconnecting && State != SocketState.Disconnected)
{
_broker.Log(LogLevel.Error, ex.ToString());
}
break;
}
}
}
private byte[] Serialize()
{
if (_buffer != null)
{
return(_buffer.ToArray());
}
_buffer = ByteBufferFactory.NewBuffer();
_buffer.WriteByte((byte)CommandType);
_buffer.WriteByte((byte)Channel);
_buffer.WriteByte((byte)Flags);
_buffer.WriteShort(Size);
_buffer.WriteLong(ReliableSequenceNumber);
switch (CommandType)
{
case CommandType.Unreliable:
_buffer.WriteLong(UnreliableSequenceNumber);
break;
case CommandType.Fragmented:
_buffer.WriteLong(StartSequenceNumber);
_buffer.WriteShort(FragmentCount);
_buffer.WriteShort(FragmentNumber);
_buffer.WriteLong(TotalLength);
_buffer.WriteLong(FragmentOffset);
break;
}
if (Payload.Count > 0)
{
_buffer.WriteBytes(Payload);
}
return(_buffer.ToArray());
}
internal MongoReplyMessage <TDocument> ReceiveMessage <TDocument>(
BsonBinaryReaderSettings readerSettings,
IBsonSerializer serializer,
IBsonSerializationOptions serializationOptions)
{
if (_state == MongoConnectionState.Closed)
{
throw new InvalidOperationException("Connection is closed.");
}
lock (_connectionLock)
{
try
{
_lastUsedAt = DateTime.UtcNow;
var networkStream = GetNetworkStream();
var readTimeout = (int)_serverInstance.Settings.SocketTimeout.TotalMilliseconds;
if (readTimeout != 0)
{
networkStream.ReadTimeout = readTimeout;
}
using (var byteBuffer = ByteBufferFactory.LoadFrom(networkStream))
using (var bsonBuffer = new BsonBuffer(byteBuffer, true))
{
byteBuffer.MakeReadOnly();
var reply = new MongoReplyMessage <TDocument>(readerSettings, serializer, serializationOptions);
reply.ReadFrom(bsonBuffer);
return(reply);
}
}
catch (Exception ex)
{
HandleException(ex);
throw;
}
}
}
public byte[] CreateConnectCommand(
short channelCount,
short mtu,
int protoVerison,
int disconnectionTimeout,
bool isCrcEnabled,
BigInteger publicKey, long sendingTime)
{
var buffer = ByteBufferFactory.NewBuffer();
buffer.WriteByte((byte)CommandType.Connect);
buffer.WriteInt(protoVerison); //4
buffer.WriteLong(sendingTime); //8
buffer.WriteShort(channelCount); //2
buffer.WriteShort(mtu); //2
buffer.WriteInt(disconnectionTimeout); //4
buffer.WriteShort((short)(isCrcEnabled == true ? 1 : 0)); //2
var publicKeyBytes = publicKey.ToByteArray();
buffer.WriteByte((byte)publicKeyBytes.Length);
buffer.WriteBytes(publicKeyBytes); //4
buffer.WriteLong(0); //8 crc space
var bytes = buffer.ToArray();
if (isCrcEnabled)
{
long crc = bytes.CalculateCrc(bytes.Length);
int offset = bytes.Length - (int)Lengths.Crc;
ByteWrite.SetLong(bytes, ref offset, crc);
}
SentCount++;
return(bytes);
}
public void TryWritingABigFileReadingItAfterSystemReload()
{
var testCollaborators = TestCollaboratorsFactory.CreateAllCollaborators(50000, false);
byte[] interestingBytes = ByteBufferFactory.BuildSomeGuidsIntoByteArray(5000);
int streamLength;
const int numberOfGuidPortionsToReadAndWrite = 500;
using (DataStreamReadableWritable stream = testCollaborators.VirtualFileSystem.CreateAndOpenFileForWriting(@"\hey"))
{
for (int i = 0; i < numberOfGuidPortionsToReadAndWrite; i++)
{
stream.Write(interestingBytes, 0, interestingBytes.Length);
}
streamLength = stream.Length;
}
VirtualFileSystem fileSystemReloaded =
TestCollaboratorsFactory.CreateFileSystemFromExistingStream(testCollaborators.Stream);
using (DataStreamReadable stream = fileSystemReloaded.OpenFileForReading(@"\hey"))
{
Assert.AreEqual(streamLength, stream.Length);
var bytesRead = new byte[interestingBytes.Length];
for (int i = 0; i < numberOfGuidPortionsToReadAndWrite; i++)
{
int numberOfBytesRead = stream.Read(bytesRead, 0, interestingBytes.Length);
CollectionAssert.AreEqual(interestingBytes, bytesRead);
}
}
}
public override IndexPopulator GetPopulator(StoreIndexDescriptor descriptor, IndexSamplingConfig samplingConfig, ByteBufferFactory bufferFactory)
{
IndexPopulator populator = Delegate.getPopulator(descriptor, samplingConfig, bufferFactory);
return(new ControlledIndexPopulator(populator, Latch));
}
public void Create_should_throw_when_chunkSource_is_null()
{
Action action = () => ByteBufferFactory.Create(null, 0);
action.ShouldThrow <ArgumentNullException>().And.ParamName.Should().Be("chunkSource");
}
public override IndexPopulator GetPopulator(StoreIndexDescriptor descriptor, IndexSamplingConfig samplingConfig, ByteBufferFactory bufferFactory)
{
return(_indexProvider.getPopulator(descriptor, samplingConfig, bufferFactory));
}
public override IndexPopulator getPopulator(StoreIndexDescriptor descriptor, IndexSamplingConfig samplingConfig, ByteBufferFactory bufferFactory)
{
return(new IndexPopulatorAnonymousInnerClass(this));
}
public override IndexPopulator GetPopulator(StoreIndexDescriptor descriptor, IndexSamplingConfig samplingConfig, ByteBufferFactory bufferFactory)
{
PartitionedIndexStorage indexStorage = GetIndexStorage(descriptor.Id);
FulltextIndexDescriptor fulltextIndexDescriptor = readOrInitialiseDescriptor(descriptor, _defaultAnalyzerName, _tokenHolders.propertyKeyTokens(), indexStorage.IndexFolder, _fileSystem);
DatabaseIndex <FulltextIndexReader> fulltextIndex = FulltextIndexBuilder.Create(fulltextIndexDescriptor, _config, _tokenHolders.propertyKeyTokens()).withFileSystem(_fileSystem).withOperationalMode(_operationalMode).withIndexStorage(indexStorage).withPopulatingMode(true).build();
if (fulltextIndex.ReadOnly)
{
throw new System.NotSupportedException("Can't create populator for read only index");
}
_log.debug("Creating populator for fulltext schema index: %s", descriptor);
return(new FulltextIndexPopulator(fulltextIndexDescriptor, fulltextIndex, () => FulltextIndexSettings.saveFulltextIndexSettings(fulltextIndexDescriptor, indexStorage.IndexFolder, _fileSystem)));
}
public ThreadLocalByteBuffer(ByteBufferFactory outerInstance)
{
this._outerInstance = outerInstance;
}
