public void WriteUpdatesLengthAndPosition() { const int expectedLength = 100; var memoryManager = this.GetMemoryManager(); var stream = new RecyclableMemoryStream(memoryManager, string.Empty, expectedLength); var buffer = this.GetRandomBuffer(expectedLength); stream.Write(buffer, 0, buffer.Length); Assert.That(stream.Length, Is.EqualTo(expectedLength)); Assert.That(stream.Position, Is.EqualTo(expectedLength)); }
private MemoryStream EncryptPackageAgile(byte[] package, ExcelEncryption encryption) { var xml = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>\r\n"; xml += "<encryption xmlns=\"http://schemas.microsoft.com/office/2006/encryption\" xmlns:p=\"http://schemas.microsoft.com/office/2006/keyEncryptor/password\" xmlns:c=\"http://schemas.microsoft.com/office/2006/keyEncryptor/certificate\">"; xml += "<keyData saltSize=\"16\" blockSize=\"16\" keyBits=\"256\" hashSize=\"64\" cipherAlgorithm=\"AES\" cipherChaining=\"ChainingModeCBC\" hashAlgorithm=\"SHA512\" saltValue=\"\"/>"; xml += "<dataIntegrity encryptedHmacKey=\"\" encryptedHmacValue=\"\"/>"; xml += "<keyEncryptors>"; xml += "<keyEncryptor uri=\"http://schemas.microsoft.com/office/2006/keyEncryptor/password\">"; xml += "<p:encryptedKey spinCount=\"100000\" saltSize=\"16\" blockSize=\"16\" keyBits=\"256\" hashSize=\"64\" cipherAlgorithm=\"AES\" cipherChaining=\"ChainingModeCBC\" hashAlgorithm=\"SHA512\" saltValue=\"\" encryptedVerifierHashInput=\"\" encryptedVerifierHashValue=\"\" encryptedKeyValue=\"\" />"; xml += "</keyEncryptor></keyEncryptors></encryption>"; var encryptionInfo = new EncryptionInfoAgile(); encryptionInfo.ReadFromXml(xml); var encr = encryptionInfo.KeyEncryptors[0]; var rnd = RandomNumberGenerator.Create(); var s = new byte[16]; rnd.GetBytes(s); encryptionInfo.KeyData.SaltValue = s; rnd.GetBytes(s); encr.SaltValue = s; encr.KeyValue = new byte[encr.KeyBits / 8]; rnd.GetBytes(encr.KeyValue); //Get the password key. var hashProvider = GetHashProvider(encryptionInfo.KeyEncryptors[0]); var baseHash = GetPasswordHash(hashProvider, encr.SaltValue, encryption.Password, encr.SpinCount, encr.HashSize); var hashFinal = GetFinalHash(hashProvider, BlockKey_KeyValue, baseHash); hashFinal = FixHashSize(hashFinal, encr.KeyBits / 8); var encrData = EncryptDataAgile(package, encryptionInfo, hashProvider); /**** Data Integrity ****/ var saltHMAC = new byte[64]; rnd.GetBytes(saltHMAC); SetHMAC(encryptionInfo, hashProvider, saltHMAC, encrData); /**** Verifier ****/ encr.VerifierHashInput = new byte[16]; rnd.GetBytes(encr.VerifierHashInput); encr.VerifierHash = hashProvider.ComputeHash(encr.VerifierHashInput); var VerifierInputKey = GetFinalHash(hashProvider, BlockKey_HashInput, baseHash); var VerifierHashKey = GetFinalHash(hashProvider, BlockKey_HashValue, baseHash); var KeyValueKey = GetFinalHash(hashProvider, BlockKey_KeyValue, baseHash); var ms = RecyclableMemoryStream.GetStream(); EncryptAgileFromKey(encr, VerifierInputKey, encr.VerifierHashInput, 0, encr.VerifierHashInput.Length, encr.SaltValue, ms); encr.EncryptedVerifierHashInput = ms.ToArray(); ms = RecyclableMemoryStream.GetStream(); EncryptAgileFromKey(encr, VerifierHashKey, encr.VerifierHash, 0, encr.VerifierHash.Length, encr.SaltValue, ms); encr.EncryptedVerifierHash = ms.ToArray(); ms = RecyclableMemoryStream.GetStream(); EncryptAgileFromKey(encr, KeyValueKey, encr.KeyValue, 0, encr.KeyValue.Length, encr.SaltValue, ms); encr.EncryptedKeyValue = ms.ToArray(); xml = encryptionInfo.Xml.OuterXml; var byXml = Encoding.UTF8.GetBytes(xml); ms = RecyclableMemoryStream.GetStream(); ms.Write(BitConverter.GetBytes((ushort)4), 0, 2); //Major Version ms.Write(BitConverter.GetBytes((ushort)4), 0, 2); //Minor Version ms.Write(BitConverter.GetBytes((uint)0x40), 0, 4); //Reserved ms.Write(byXml, 0, byXml.Length); var doc = new CompoundDocument(); //Add the dataspace streams CreateDataSpaces(doc); //EncryptionInfo... doc.Storage.DataStreams.Add("EncryptionInfo", ms.ToArray()); //...and the encrypted package doc.Storage.DataStreams.Add("EncryptedPackage", encrData); ms = RecyclableMemoryStream.GetStream(); doc.Save(ms); //ms.Write(e,0,e.Length); return(ms); }
public void ActualCapacityAtLeastRequestedCapacityAndMultipleOfBlockSize() { var memoryManager = this.GetMemoryManager(); var requestedSize = memoryManager.BlockSize + 1; var stream = new RecyclableMemoryStream(memoryManager, string.Empty, requestedSize); Assert.That(stream.Capacity, Is.GreaterThanOrEqualTo(requestedSize)); Assert.That(stream.Capacity % memoryManager.BlockSize == 0, Is.True, "stream capacity is not a multiple of the block size"); }
public void AllocationStackIsRecorded() { var memMgr = this.GetMemoryManager(); memMgr.GenerateCallStacks = true; var stream = new RecyclableMemoryStream(memMgr); Assert.That(stream.AllocationStack, Is.StringContaining("RecyclableMemoryStream..ctor")); stream.Dispose(); memMgr.GenerateCallStacks = false; var stream2 = new RecyclableMemoryStream(memMgr); Assert.That(stream2.AllocationStack, Is.Null); stream2.Dispose(); }
public void StreamHasTagAndGuid() { const string expectedTag = "Nunit Test"; var stream = new RecyclableMemoryStream(this.GetMemoryManager(), expectedTag); Assert.That(stream.Id, Is.Not.EqualTo(Guid.Empty)); Assert.That(stream.Tag, Is.EqualTo(expectedTag)); }
public void StreamHasDefaultCapacity() { var memoryManager = this.GetMemoryManager(); var stream = new RecyclableMemoryStream(memoryManager); Assert.That(stream.Capacity, Is.EqualTo(memoryManager.BlockSize)); }
public void Pooling_NewStreamIncrementsInUseBytes() { var memMgr = this.GetMemoryManager(); Assert.That(memMgr.SmallPoolInUseSize, Is.EqualTo(0)); var stream = new RecyclableMemoryStream(memMgr); Assert.That(stream.Capacity, Is.EqualTo(memMgr.BlockSize)); Assert.That(memMgr.SmallPoolInUseSize, Is.EqualTo(memMgr.BlockSize)); Assert.That(memMgr.SmallPoolFreeSize, Is.EqualTo(0)); }
public void GetBufferAdjustsLargePoolFreeSize() { var stream = this.GetDefaultStream(); var memMgr = stream.MemoryManager; var bufferLength = stream.MemoryManager.BlockSize * 4; var buffer = this.GetRandomBuffer(bufferLength); stream.Write(buffer, 0, buffer.Length); var newBuffer = stream.GetBuffer(); stream.Dispose(); Assert.That(memMgr.LargePoolFreeSize, Is.EqualTo(newBuffer.Length)); var newStream = new RecyclableMemoryStream(memMgr); newStream.Write(buffer, 0, buffer.Length); var newBuffer2 = newStream.GetBuffer(); Assert.That(newBuffer2.Length, Is.EqualTo(newBuffer.Length)); Assert.That(memMgr.LargePoolFreeSize, Is.EqualTo(0)); }
public void ToStringReturnsHelpfulDebugInfo() { var tag = "Nunit test"; var stream = new RecyclableMemoryStream(this.GetMemoryManager(), tag); var buffer = this.GetRandomBuffer(1000); stream.Write(buffer, 0, buffer.Length); var debugInfo = stream.ToString(); Assert.That(debugInfo, Contains.Substring(stream.Id.ToString())); Assert.That(debugInfo, Contains.Substring(tag)); Assert.That(debugInfo, Contains.Substring(buffer.Length.ToString("N0"))); }
public void ToStringWithNullTagIsOk() { var stream = new RecyclableMemoryStream(this.GetMemoryManager(), null); var buffer = this.GetRandomBuffer(1000); stream.Write(buffer, 0, buffer.Length); var debugInfo = stream.ToString(); Assert.That(debugInfo, Contains.Substring(stream.Id.ToString())); Assert.That(debugInfo, Contains.Substring(buffer.Length.ToString("N0"))); }
public async Task <UpsertOutcome> UpsertAsync(ResourceWrapper resource, WeakETag weakETag, bool allowCreate, bool keepHistory, CancellationToken cancellationToken) { int?eTag = weakETag == null ? (int?)null : (int.TryParse(weakETag.VersionId, out var parsedETag) ? parsedETag : -1); // Set the etag to a sentinel value to enable expected failure paths when updating with both existing and nonexistent resources. var resourceMetadata = new ResourceMetadata( resource.CompartmentIndices, resource.SearchIndices?.ToLookup(e => _searchParameterTypeMap.GetSearchValueType(e)), resource.LastModifiedClaims); using (SqlConnectionWrapper sqlConnectionWrapper = await _sqlConnectionWrapperFactory.ObtainSqlConnectionWrapperAsync(cancellationToken, true)) using (SqlCommandWrapper sqlCommandWrapper = sqlConnectionWrapper.CreateSqlCommand()) using (var stream = new RecyclableMemoryStream(_memoryStreamManager)) { CompressedRawResourceConverter.WriteCompressedRawResource(stream, resource.RawResource.Data); stream.Seek(0, 0); PopulateUpsertResourceCommand(sqlCommandWrapper, resource, resourceMetadata, allowCreate, keepHistory, eTag, stream); try { var newVersion = (int?)await sqlCommandWrapper.ExecuteScalarAsync(cancellationToken); if (newVersion == null) { // indicates a redundant delete return(null); } resource.Version = newVersion.ToString(); SaveOutcomeType saveOutcomeType; if (newVersion == 1) { saveOutcomeType = SaveOutcomeType.Created; } else { saveOutcomeType = SaveOutcomeType.Updated; resource.RawResource.IsMetaSet = false; } return(new UpsertOutcome(resource, saveOutcomeType)); } catch (SqlException e) { switch (e.Number) { case SqlErrorCodes.PreconditionFailed: throw new PreconditionFailedException(string.Format(Core.Resources.ResourceVersionConflict, weakETag?.VersionId)); case SqlErrorCodes.NotFound: if (weakETag != null) { throw new ResourceNotFoundException(string.Format(Core.Resources.ResourceNotFoundByIdAndVersion, resource.ResourceTypeName, resource.ResourceId, weakETag.VersionId)); } goto default; case SqlErrorCodes.MethodNotAllowed: throw new MethodNotAllowedException(Core.Resources.ResourceCreationNotAllowed); default: _logger.LogError(e, "Error from SQL database on upsert"); throw; } } } }
private void PopulateUpsertResourceCommand(SqlCommandWrapper sqlCommandWrapper, ResourceWrapper resource, ResourceMetadata resourceMetadata, bool allowCreate, bool keepHistory, int?eTag, RecyclableMemoryStream stream) { long baseResourceSurrogateId = ResourceSurrogateIdHelper.LastUpdatedToResourceSurrogateId(resource.LastModified.UtcDateTime); short resourceTypeId = _model.GetResourceTypeId(resource.ResourceTypeName); if (_schemaInformation.Current >= SchemaVersionConstants.SupportForReferencesWithMissingTypeVersion) { VLatest.UpsertResource.PopulateCommand( sqlCommandWrapper, baseResourceSurrogateId: baseResourceSurrogateId, resourceTypeId: resourceTypeId, resourceId: resource.ResourceId, eTag: eTag, allowCreate: allowCreate, isDeleted: resource.IsDeleted, keepHistory: keepHistory, requestMethod: resource.Request.Method, rawResource: stream, tableValuedParameters: _upsertResourceTvpGeneratorVLatest.Generate(resourceMetadata)); } else { V6.UpsertResource.PopulateCommand( sqlCommandWrapper, baseResourceSurrogateId: baseResourceSurrogateId, resourceTypeId: resourceTypeId, resourceId: resource.ResourceId, eTag: eTag, allowCreate: allowCreate, isDeleted: resource.IsDeleted, keepHistory: keepHistory, requestMethod: resource.Request.Method, rawResource: stream, tableValuedParameters: _upsertResourceTvpGeneratorV6.Generate(resourceMetadata)); } }
internal static void LoadXmlSafe(XmlDocument xmlDoc, string xml, Encoding encoding) { var stream = RecyclableMemoryStream.GetStream(encoding.GetBytes(xml)); LoadXmlSafe(xmlDoc, stream); }
public async Task <UpsertOutcome> UpsertAsync(ResourceWrapper resource, WeakETag weakETag, bool allowCreate, bool keepHistory, CancellationToken cancellationToken) { await _model.EnsureInitialized(); int etag = 0; if (weakETag != null && !int.TryParse(weakETag.VersionId, out etag)) { throw new ResourceConflictException(weakETag); } var resourceMetadata = new ResourceMetadata( resource.CompartmentIndices, resource.SearchIndices?.ToLookup(e => _searchParameterTypeMap.GetSearchValueType(e)), resource.LastModifiedClaims); using (var connection = new SqlConnection(_configuration.ConnectionString)) { await connection.OpenAsync(cancellationToken); using (var command = connection.CreateCommand()) using (var stream = new RecyclableMemoryStream(_memoryStreamManager)) using (var gzipStream = new GZipStream(stream, CompressionMode.Compress)) using (var writer = new StreamWriter(gzipStream, ResourceEncoding)) { writer.Write(resource.RawResource.Data); writer.Flush(); stream.Seek(0, 0); V1.UpsertResource.PopulateCommand( command, baseResourceSurrogateId: ResourceSurrogateIdHelper.LastUpdatedToResourceSurrogateId(resource.LastModified.UtcDateTime), resourceTypeId: _model.GetResourceTypeId(resource.ResourceTypeName), resourceId: resource.ResourceId, eTag: weakETag == null ? null : (int?)etag, allowCreate: allowCreate, isDeleted: resource.IsDeleted, keepHistory: keepHistory, requestMethod: resource.Request.Method, rawResource: stream, tableValuedParameters: _upsertResourceTvpGenerator.Generate(resourceMetadata)); try { var newVersion = (int?)await command.ExecuteScalarAsync(cancellationToken); if (newVersion == null) { // indicates a redundant delete return(null); } resource.Version = newVersion.ToString(); return(new UpsertOutcome(resource, newVersion == 1 ? SaveOutcomeType.Created : SaveOutcomeType.Updated)); } catch (SqlException e) { switch (e.Number) { case SqlErrorCodes.NotFound: throw new MethodNotAllowedException(Core.Resources.ResourceCreationNotAllowed); case SqlErrorCodes.PreconditionFailed: throw new ResourceConflictException(weakETag); default: _logger.LogError(e, "Error from SQL database on upsert"); throw; } } } } }
private void RunWriteQueueAction() { //Dequeue all items until threshold is passed long totalLength = 0; RecyclableMemoryStream stream = null; while (totalLength < CoalescingThreshold) { OperationState state; if (!_writeQueue.TryDequeue(out state)) { //No more items in the write queue break; } short streamId; if (!_freeOperations.TryPop(out streamId)) { //Queue it up for later. _writeQueue.Enqueue(state); //When receiving the next complete message, we can process it. Logger.Info("Enqueued, no streamIds available. If this message is recurrent consider configuring more connections per host or lower the pressure"); break; } Logger.Verbose("Sending #{0} for {1} to {2}", streamId, state.Request.GetType().Name, Address); if (_isCanceled) { state.InvokeCallback(new SocketException((int)SocketError.NotConnected)); break; } _pendingOperations.AddOrUpdate(streamId, state, (k, oldValue) => state); int frameLength; try { //lazy initialize the stream stream = stream ?? (RecyclableMemoryStream)Configuration.BufferPool.GetStream(StreamWriteTag); frameLength = state.Request.WriteFrame(streamId, stream, _serializer); if (state.TimeoutMillis > 0) { var requestTimeout = Configuration.Timer.NewTimeout(OnTimeout, streamId, state.TimeoutMillis); state.SetTimeout(requestTimeout); } } catch (Exception ex) { //There was an error while serializing or begin sending Logger.Error(ex); //The request was not written, clear it from pending operations RemoveFromPending(streamId); //Callback with the Exception state.InvokeCallback(ex); break; } //We will not use the request any more, stop reference it. state.Request = null; totalLength += frameLength; } if (totalLength == 0L) { // Nothing to write, set the queue as not running Interlocked.CompareExchange(ref _writeState, WriteStateInit, WriteStateRunning); // Until now, we were preventing other threads to running the queue. // Check if we can now write: // a read could have finished (freeing streamIds) or new request could have been added to the queue if (!_freeOperations.IsEmpty && !_writeQueue.IsEmpty) { //The write queue is not empty //An item was added to the queue but we were running: try to launch a new queue RunWriteQueue(); } if (stream != null) { //The stream instance could be created if there was an exception while generating the frame stream.Dispose(); } return; } //Write and close the stream when flushed // ReSharper disable once PossibleNullReferenceException : if totalLength > 0 the stream is initialized _tcpSocket.Write(stream, () => stream.Dispose()); }