public void WriteUpdatesLengthAndPosition()
{
const int expectedLength = 100;
var memoryManager = this.GetMemoryManager();
var stream = new RecyclableMemoryStream(memoryManager, string.Empty, expectedLength);
var buffer = this.GetRandomBuffer(expectedLength);
stream.Write(buffer, 0, buffer.Length);
Assert.That(stream.Length, Is.EqualTo(expectedLength));
Assert.That(stream.Position, Is.EqualTo(expectedLength));
}
private MemoryStream EncryptPackageAgile(byte[] package, ExcelEncryption encryption)
{
var xml = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>\r\n";
xml += "<encryption xmlns=\"http://schemas.microsoft.com/office/2006/encryption\" xmlns:p=\"http://schemas.microsoft.com/office/2006/keyEncryptor/password\" xmlns:c=\"http://schemas.microsoft.com/office/2006/keyEncryptor/certificate\">";
xml += "<keyData saltSize=\"16\" blockSize=\"16\" keyBits=\"256\" hashSize=\"64\" cipherAlgorithm=\"AES\" cipherChaining=\"ChainingModeCBC\" hashAlgorithm=\"SHA512\" saltValue=\"\"/>";
xml += "<dataIntegrity encryptedHmacKey=\"\" encryptedHmacValue=\"\"/>";
xml += "<keyEncryptors>";
xml += "<keyEncryptor uri=\"http://schemas.microsoft.com/office/2006/keyEncryptor/password\">";
xml += "<p:encryptedKey spinCount=\"100000\" saltSize=\"16\" blockSize=\"16\" keyBits=\"256\" hashSize=\"64\" cipherAlgorithm=\"AES\" cipherChaining=\"ChainingModeCBC\" hashAlgorithm=\"SHA512\" saltValue=\"\" encryptedVerifierHashInput=\"\" encryptedVerifierHashValue=\"\" encryptedKeyValue=\"\" />";
xml += "</keyEncryptor></keyEncryptors></encryption>";
var encryptionInfo = new EncryptionInfoAgile();
encryptionInfo.ReadFromXml(xml);
var encr = encryptionInfo.KeyEncryptors[0];
var rnd = RandomNumberGenerator.Create();
var s = new byte[16];
rnd.GetBytes(s);
encryptionInfo.KeyData.SaltValue = s;
rnd.GetBytes(s);
encr.SaltValue = s;
encr.KeyValue = new byte[encr.KeyBits / 8];
rnd.GetBytes(encr.KeyValue);
//Get the password key.
var hashProvider = GetHashProvider(encryptionInfo.KeyEncryptors[0]);
var baseHash = GetPasswordHash(hashProvider, encr.SaltValue, encryption.Password, encr.SpinCount, encr.HashSize);
var hashFinal = GetFinalHash(hashProvider, BlockKey_KeyValue, baseHash);
hashFinal = FixHashSize(hashFinal, encr.KeyBits / 8);
var encrData = EncryptDataAgile(package, encryptionInfo, hashProvider);
/**** Data Integrity ****/
var saltHMAC = new byte[64];
rnd.GetBytes(saltHMAC);
SetHMAC(encryptionInfo, hashProvider, saltHMAC, encrData);
/**** Verifier ****/
encr.VerifierHashInput = new byte[16];
rnd.GetBytes(encr.VerifierHashInput);
encr.VerifierHash = hashProvider.ComputeHash(encr.VerifierHashInput);
var VerifierInputKey = GetFinalHash(hashProvider, BlockKey_HashInput, baseHash);
var VerifierHashKey = GetFinalHash(hashProvider, BlockKey_HashValue, baseHash);
var KeyValueKey = GetFinalHash(hashProvider, BlockKey_KeyValue, baseHash);
var ms = RecyclableMemoryStream.GetStream();
EncryptAgileFromKey(encr, VerifierInputKey, encr.VerifierHashInput, 0, encr.VerifierHashInput.Length, encr.SaltValue, ms);
encr.EncryptedVerifierHashInput = ms.ToArray();
ms = RecyclableMemoryStream.GetStream();
EncryptAgileFromKey(encr, VerifierHashKey, encr.VerifierHash, 0, encr.VerifierHash.Length, encr.SaltValue, ms);
encr.EncryptedVerifierHash = ms.ToArray();
ms = RecyclableMemoryStream.GetStream();
EncryptAgileFromKey(encr, KeyValueKey, encr.KeyValue, 0, encr.KeyValue.Length, encr.SaltValue, ms);
encr.EncryptedKeyValue = ms.ToArray();
xml = encryptionInfo.Xml.OuterXml;
var byXml = Encoding.UTF8.GetBytes(xml);
ms = RecyclableMemoryStream.GetStream();
ms.Write(BitConverter.GetBytes((ushort)4), 0, 2); //Major Version
ms.Write(BitConverter.GetBytes((ushort)4), 0, 2); //Minor Version
ms.Write(BitConverter.GetBytes((uint)0x40), 0, 4); //Reserved
ms.Write(byXml, 0, byXml.Length);
var doc = new CompoundDocument();
//Add the dataspace streams
CreateDataSpaces(doc);
//EncryptionInfo...
doc.Storage.DataStreams.Add("EncryptionInfo", ms.ToArray());
//...and the encrypted package
doc.Storage.DataStreams.Add("EncryptedPackage", encrData);
ms = RecyclableMemoryStream.GetStream();
doc.Save(ms);
//ms.Write(e,0,e.Length);
return(ms);
}
public void ActualCapacityAtLeastRequestedCapacityAndMultipleOfBlockSize()
{
var memoryManager = this.GetMemoryManager();
var requestedSize = memoryManager.BlockSize + 1;
var stream = new RecyclableMemoryStream(memoryManager, string.Empty, requestedSize);
Assert.That(stream.Capacity, Is.GreaterThanOrEqualTo(requestedSize));
Assert.That(stream.Capacity % memoryManager.BlockSize == 0, Is.True,
"stream capacity is not a multiple of the block size");
}
public void AllocationStackIsRecorded()
{
var memMgr = this.GetMemoryManager();
memMgr.GenerateCallStacks = true;
var stream = new RecyclableMemoryStream(memMgr);
Assert.That(stream.AllocationStack, Is.StringContaining("RecyclableMemoryStream..ctor"));
stream.Dispose();
memMgr.GenerateCallStacks = false;
var stream2 = new RecyclableMemoryStream(memMgr);
Assert.That(stream2.AllocationStack, Is.Null);
stream2.Dispose();
}
public void StreamHasTagAndGuid()
{
const string expectedTag = "Nunit Test";
var stream = new RecyclableMemoryStream(this.GetMemoryManager(), expectedTag);
Assert.That(stream.Id, Is.Not.EqualTo(Guid.Empty));
Assert.That(stream.Tag, Is.EqualTo(expectedTag));
}
public void StreamHasDefaultCapacity()
{
var memoryManager = this.GetMemoryManager();
var stream = new RecyclableMemoryStream(memoryManager);
Assert.That(stream.Capacity, Is.EqualTo(memoryManager.BlockSize));
}
public void Pooling_NewStreamIncrementsInUseBytes()
{
var memMgr = this.GetMemoryManager();
Assert.That(memMgr.SmallPoolInUseSize, Is.EqualTo(0));
var stream = new RecyclableMemoryStream(memMgr);
Assert.That(stream.Capacity, Is.EqualTo(memMgr.BlockSize));
Assert.That(memMgr.SmallPoolInUseSize, Is.EqualTo(memMgr.BlockSize));
Assert.That(memMgr.SmallPoolFreeSize, Is.EqualTo(0));
}
public void GetBufferAdjustsLargePoolFreeSize()
{
var stream = this.GetDefaultStream();
var memMgr = stream.MemoryManager;
var bufferLength = stream.MemoryManager.BlockSize * 4;
var buffer = this.GetRandomBuffer(bufferLength);
stream.Write(buffer, 0, buffer.Length);
var newBuffer = stream.GetBuffer();
stream.Dispose();
Assert.That(memMgr.LargePoolFreeSize, Is.EqualTo(newBuffer.Length));
var newStream = new RecyclableMemoryStream(memMgr);
newStream.Write(buffer, 0, buffer.Length);
var newBuffer2 = newStream.GetBuffer();
Assert.That(newBuffer2.Length, Is.EqualTo(newBuffer.Length));
Assert.That(memMgr.LargePoolFreeSize, Is.EqualTo(0));
}
public void ToStringReturnsHelpfulDebugInfo()
{
var tag = "Nunit test";
var stream = new RecyclableMemoryStream(this.GetMemoryManager(), tag);
var buffer = this.GetRandomBuffer(1000);
stream.Write(buffer, 0, buffer.Length);
var debugInfo = stream.ToString();
Assert.That(debugInfo, Contains.Substring(stream.Id.ToString()));
Assert.That(debugInfo, Contains.Substring(tag));
Assert.That(debugInfo, Contains.Substring(buffer.Length.ToString("N0")));
}
public void ToStringWithNullTagIsOk()
{
var stream = new RecyclableMemoryStream(this.GetMemoryManager(), null);
var buffer = this.GetRandomBuffer(1000);
stream.Write(buffer, 0, buffer.Length);
var debugInfo = stream.ToString();
Assert.That(debugInfo, Contains.Substring(stream.Id.ToString()));
Assert.That(debugInfo, Contains.Substring(buffer.Length.ToString("N0")));
}
public async Task <UpsertOutcome> UpsertAsync(ResourceWrapper resource, WeakETag weakETag, bool allowCreate, bool keepHistory, CancellationToken cancellationToken)
{
int?eTag = weakETag == null
? (int?)null
: (int.TryParse(weakETag.VersionId, out var parsedETag) ? parsedETag : -1); // Set the etag to a sentinel value to enable expected failure paths when updating with both existing and nonexistent resources.
var resourceMetadata = new ResourceMetadata(
resource.CompartmentIndices,
resource.SearchIndices?.ToLookup(e => _searchParameterTypeMap.GetSearchValueType(e)),
resource.LastModifiedClaims);
using (SqlConnectionWrapper sqlConnectionWrapper = await _sqlConnectionWrapperFactory.ObtainSqlConnectionWrapperAsync(cancellationToken, true))
using (SqlCommandWrapper sqlCommandWrapper = sqlConnectionWrapper.CreateSqlCommand())
using (var stream = new RecyclableMemoryStream(_memoryStreamManager))
{
CompressedRawResourceConverter.WriteCompressedRawResource(stream, resource.RawResource.Data);
stream.Seek(0, 0);
PopulateUpsertResourceCommand(sqlCommandWrapper, resource, resourceMetadata, allowCreate, keepHistory, eTag, stream);
try
{
var newVersion = (int?)await sqlCommandWrapper.ExecuteScalarAsync(cancellationToken);
if (newVersion == null)
{
// indicates a redundant delete
return(null);
}
resource.Version = newVersion.ToString();
SaveOutcomeType saveOutcomeType;
if (newVersion == 1)
{
saveOutcomeType = SaveOutcomeType.Created;
}
else
{
saveOutcomeType = SaveOutcomeType.Updated;
resource.RawResource.IsMetaSet = false;
}
return(new UpsertOutcome(resource, saveOutcomeType));
}
catch (SqlException e)
{
switch (e.Number)
{
case SqlErrorCodes.PreconditionFailed:
throw new PreconditionFailedException(string.Format(Core.Resources.ResourceVersionConflict, weakETag?.VersionId));
case SqlErrorCodes.NotFound:
if (weakETag != null)
{
throw new ResourceNotFoundException(string.Format(Core.Resources.ResourceNotFoundByIdAndVersion, resource.ResourceTypeName, resource.ResourceId, weakETag.VersionId));
}
goto default;
case SqlErrorCodes.MethodNotAllowed:
throw new MethodNotAllowedException(Core.Resources.ResourceCreationNotAllowed);
default:
_logger.LogError(e, "Error from SQL database on upsert");
throw;
}
}
}
}
private void PopulateUpsertResourceCommand(SqlCommandWrapper sqlCommandWrapper, ResourceWrapper resource, ResourceMetadata resourceMetadata, bool allowCreate, bool keepHistory, int?eTag, RecyclableMemoryStream stream)
{
long baseResourceSurrogateId = ResourceSurrogateIdHelper.LastUpdatedToResourceSurrogateId(resource.LastModified.UtcDateTime);
short resourceTypeId = _model.GetResourceTypeId(resource.ResourceTypeName);
if (_schemaInformation.Current >= SchemaVersionConstants.SupportForReferencesWithMissingTypeVersion)
{
VLatest.UpsertResource.PopulateCommand(
sqlCommandWrapper,
baseResourceSurrogateId: baseResourceSurrogateId,
resourceTypeId: resourceTypeId,
resourceId: resource.ResourceId,
eTag: eTag,
allowCreate: allowCreate,
isDeleted: resource.IsDeleted,
keepHistory: keepHistory,
requestMethod: resource.Request.Method,
rawResource: stream,
tableValuedParameters: _upsertResourceTvpGeneratorVLatest.Generate(resourceMetadata));
}
else
{
V6.UpsertResource.PopulateCommand(
sqlCommandWrapper,
baseResourceSurrogateId: baseResourceSurrogateId,
resourceTypeId: resourceTypeId,
resourceId: resource.ResourceId,
eTag: eTag,
allowCreate: allowCreate,
isDeleted: resource.IsDeleted,
keepHistory: keepHistory,
requestMethod: resource.Request.Method,
rawResource: stream,
tableValuedParameters: _upsertResourceTvpGeneratorV6.Generate(resourceMetadata));
}
}
internal static void LoadXmlSafe(XmlDocument xmlDoc, string xml, Encoding encoding)
{
var stream = RecyclableMemoryStream.GetStream(encoding.GetBytes(xml));
LoadXmlSafe(xmlDoc, stream);
}
public async Task <UpsertOutcome> UpsertAsync(ResourceWrapper resource, WeakETag weakETag, bool allowCreate, bool keepHistory, CancellationToken cancellationToken)
{
await _model.EnsureInitialized();
int etag = 0;
if (weakETag != null && !int.TryParse(weakETag.VersionId, out etag))
{
throw new ResourceConflictException(weakETag);
}
var resourceMetadata = new ResourceMetadata(
resource.CompartmentIndices,
resource.SearchIndices?.ToLookup(e => _searchParameterTypeMap.GetSearchValueType(e)),
resource.LastModifiedClaims);
using (var connection = new SqlConnection(_configuration.ConnectionString))
{
await connection.OpenAsync(cancellationToken);
using (var command = connection.CreateCommand())
using (var stream = new RecyclableMemoryStream(_memoryStreamManager))
using (var gzipStream = new GZipStream(stream, CompressionMode.Compress))
using (var writer = new StreamWriter(gzipStream, ResourceEncoding))
{
writer.Write(resource.RawResource.Data);
writer.Flush();
stream.Seek(0, 0);
V1.UpsertResource.PopulateCommand(
command,
baseResourceSurrogateId: ResourceSurrogateIdHelper.LastUpdatedToResourceSurrogateId(resource.LastModified.UtcDateTime),
resourceTypeId: _model.GetResourceTypeId(resource.ResourceTypeName),
resourceId: resource.ResourceId,
eTag: weakETag == null ? null : (int?)etag,
allowCreate: allowCreate,
isDeleted: resource.IsDeleted,
keepHistory: keepHistory,
requestMethod: resource.Request.Method,
rawResource: stream,
tableValuedParameters: _upsertResourceTvpGenerator.Generate(resourceMetadata));
try
{
var newVersion = (int?)await command.ExecuteScalarAsync(cancellationToken);
if (newVersion == null)
{
// indicates a redundant delete
return(null);
}
resource.Version = newVersion.ToString();
return(new UpsertOutcome(resource, newVersion == 1 ? SaveOutcomeType.Created : SaveOutcomeType.Updated));
}
catch (SqlException e)
{
switch (e.Number)
{
case SqlErrorCodes.NotFound:
throw new MethodNotAllowedException(Core.Resources.ResourceCreationNotAllowed);
case SqlErrorCodes.PreconditionFailed:
throw new ResourceConflictException(weakETag);
default:
_logger.LogError(e, "Error from SQL database on upsert");
throw;
}
}
}
}
}
private void RunWriteQueueAction()
{
//Dequeue all items until threshold is passed
long totalLength = 0;
RecyclableMemoryStream stream = null;
while (totalLength < CoalescingThreshold)
{
OperationState state;
if (!_writeQueue.TryDequeue(out state))
{
//No more items in the write queue
break;
}
short streamId;
if (!_freeOperations.TryPop(out streamId))
{
//Queue it up for later.
_writeQueue.Enqueue(state);
//When receiving the next complete message, we can process it.
Logger.Info("Enqueued, no streamIds available. If this message is recurrent consider configuring more connections per host or lower the pressure");
break;
}
Logger.Verbose("Sending #{0} for {1} to {2}", streamId, state.Request.GetType().Name, Address);
if (_isCanceled)
{
state.InvokeCallback(new SocketException((int)SocketError.NotConnected));
break;
}
_pendingOperations.AddOrUpdate(streamId, state, (k, oldValue) => state);
int frameLength;
try
{
//lazy initialize the stream
stream = stream ?? (RecyclableMemoryStream)Configuration.BufferPool.GetStream(StreamWriteTag);
frameLength = state.Request.WriteFrame(streamId, stream, _serializer);
if (state.TimeoutMillis > 0)
{
var requestTimeout = Configuration.Timer.NewTimeout(OnTimeout, streamId, state.TimeoutMillis);
state.SetTimeout(requestTimeout);
}
}
catch (Exception ex)
{
//There was an error while serializing or begin sending
Logger.Error(ex);
//The request was not written, clear it from pending operations
RemoveFromPending(streamId);
//Callback with the Exception
state.InvokeCallback(ex);
break;
}
//We will not use the request any more, stop reference it.
state.Request = null;
totalLength += frameLength;
}
if (totalLength == 0L)
{
// Nothing to write, set the queue as not running
Interlocked.CompareExchange(ref _writeState, WriteStateInit, WriteStateRunning);
// Until now, we were preventing other threads to running the queue.
// Check if we can now write:
// a read could have finished (freeing streamIds) or new request could have been added to the queue
if (!_freeOperations.IsEmpty && !_writeQueue.IsEmpty)
{
//The write queue is not empty
//An item was added to the queue but we were running: try to launch a new queue
RunWriteQueue();
}
if (stream != null)
{
//The stream instance could be created if there was an exception while generating the frame
stream.Dispose();
}
return;
}
//Write and close the stream when flushed
// ReSharper disable once PossibleNullReferenceException : if totalLength > 0 the stream is initialized
_tcpSocket.Write(stream, () => stream.Dispose());
}
