/// <summary> /// Gets the deserialized header from the blob. Uses /// </summary> /// <param name="blob">The Blob.</param> /// <param name="assumedValidHeaderDefinition">will output the header definition that is valid, null if there is not currently one</param> /// <returns>A populated StreamBlobHeader.</returns> private StreamBlobHeader GetHeaderWithRetry(WrappedPageBlob blob, out HeaderDefinitionMetadata validHeaderDefinition) { HeaderDefinitionMetadata assumedValidHeaderDefinition = null; StreamBlobHeader header = null; Exception lastException = null; // first, if the primary header metadata key does not exist then we default if (!blob.Metadata.ContainsKey(_primaryHeaderDefinitionKey)) { assumedValidHeaderDefinition = new HeaderDefinitionMetadata(); header = new StreamBlobHeader(); } // do the fallback logic to try and find a valid header if (header == null) { for (int i = 0; i != 3; ++i) { assumedValidHeaderDefinition = GetHeaderDefinitionMetadata(blob, i); header = SafeGetHeader(blob, assumedValidHeaderDefinition, out lastException); if (header != null) { break; } } } // It is possible we will still have no header here and still be in an ok state. This is a case where the aggregates first // commit set some metadata (specifically) the _primaryHeaderDefinitionKey, but then failed to write the header. This case // will have a _primaryHeaderDefinitionKey but no secondary or terciary. In addition it will have a key of first write succeeded // set to false. If it does not have any key at all, it is a "legacy" one and will get the key set upon a future successful write. // legacy ones with issue will continue to fail and require manual intervention currently if (header == null) { string firstWriteCompleted; if (blob.Metadata.TryGetValue(_firstWriteCompletedKey, out firstWriteCompleted)) { if (firstWriteCompleted == "f") { header = new StreamBlobHeader(); assumedValidHeaderDefinition = new HeaderDefinitionMetadata(); } } } if (header != null) { validHeaderDefinition = assumedValidHeaderDefinition; return(header); } throw lastException ?? new Exception("No header could be created");; }
/// <summary> /// Commits the header information which essentially commits any transactions that occured /// related to that header. /// </summary> /// <param name="newCommit">the new commit to write</param> /// <param name="headerStartOffsetBytes">where in the blob the new commit will be written</param> /// <param name="blob">blob header applies to</param> /// <param name="header">the new header data</param> /// <returns></returns> private void CommitNewMessage(WrappedPageBlob blob, byte[] newCommit, StreamBlobHeader header, int offsetBytes) { newCommit = newCommit ?? new byte[0]; var serializedHeader = _serializer.Serialize(header); var writeStartLocationAligned = GetPageAlignedSize(offsetBytes); var amountToWriteAligned = GetPageAlignedSize(serializedHeader.Length + newCommit.Length); var totalSpaceNeeded = writeStartLocationAligned + amountToWriteAligned; var totalBlobLength = blob.Properties.Length; if (totalBlobLength < totalSpaceNeeded) { blob.Resize(totalSpaceNeeded); totalBlobLength = blob.Properties.Length; } // set the header definition to make it all official var headerDefinitionMetadata = new HeaderDefinitionMetadata(); headerDefinitionMetadata.HeaderSizeInBytes = serializedHeader.Length; headerDefinitionMetadata.HeaderStartLocationOffsetBytes = writeStartLocationAligned + newCommit.Length; blob.Metadata[_isEventStreamAggregateKey] = "yes"; blob.Metadata[_hasUndispatchedCommitsKey] = header.PageBlobCommitDefinitions.Any((x) => !x.IsDispatched).ToString(); if (blob.Metadata.ContainsKey(_primaryHeaderDefinitionKey)) { blob.Metadata[_fallbackHeaderDefinitionKey] = blob.Metadata[_primaryHeaderDefinitionKey]; } blob.Metadata[_primaryHeaderDefinitionKey] = Convert.ToBase64String(headerDefinitionMetadata.GetRaw()); blob.SetMetadata(); using (var ms = new MemoryStream(amountToWriteAligned)) { ms.Write(newCommit, 0, newCommit.Length); ms.Write(serializedHeader, 0, serializedHeader.Length); var remainder = (ms.Position % _blobPageSize); var fillSpace = amountToWriteAligned - newCommit.Length - serializedHeader.Length; if (fillSpace != 0) { ms.Position += fillSpace - 1; ms.WriteByte(0); } ms.Position = 0; blob.Write(ms, writeStartLocationAligned); } }
/// <summary> /// Safely receive the header. /// </summary> /// <param name="blob">blob</param> /// <param name="headerDefinition">header definition</param> /// <param name="exception">exception that occured</param> /// <returns>the header, otherwise null if it could not be fetched</returns> private StreamBlobHeader SafeGetHeader(WrappedPageBlob blob, HeaderDefinitionMetadata headerDefinition, out Exception exception) { StreamBlobHeader header = null; exception = null; try { var downloadedData = blob.DownloadBytes(headerDefinition.HeaderStartLocationOffsetBytes, headerDefinition.HeaderStartLocationOffsetBytes + headerDefinition.HeaderSizeInBytes); using (var ms = new MemoryStream(downloadedData, false)) { header = _serializer.Deserialize <StreamBlobHeader>(ms.ToArray()); } } catch (Exception ex) { exception = ex; } return(header); }
/// <summary> /// Safely receive the header. /// </summary> /// <param name="blob">blob</param> /// <param name="headerDefinition">header definition</param> /// <param name="exception">exception that occured</param> /// <returns>the header, otherwise null if it could not be fetched</returns> private StreamBlobHeader SafeGetHeader(WrappedPageBlob blob, HeaderDefinitionMetadata headerDefinition, out Exception exception) { StreamBlobHeader header = null; exception = null; try { if (headerDefinition == null || headerDefinition.HeaderSizeInBytes == 0) { throw new InvalidHeaderDataException(string.Format("Attempted to download a header, but the size specified is zero. This aggregate with id [{0}] may be corrupt.", blob.Name)); } var downloadedData = blob.DownloadBytes(headerDefinition.HeaderStartLocationOffsetBytes, headerDefinition.HeaderStartLocationOffsetBytes + headerDefinition.HeaderSizeInBytes); using (var ms = new MemoryStream(downloadedData, false)) { header = _serializer.Deserialize <StreamBlobHeader>(ms.ToArray()); } } catch (Exception ex) { exception = ex; } return(header); }
/// <summary> /// Commits the header information which essentially commits any transactions that occurred /// related to that header. /// </summary> /// <param name="newCommit">the new commit to write</param> /// <param name="blob">blob header applies to</param> /// <param name="updatedHeader">the new header to be serialized out</param> /// <param name="currentGoodHeaderDefinition">the definition for the current header, before this change is committed</param> /// <param name="nonAlignedBytesUsedAlready">non aligned offset of index where last commit data is stored (not inclusive of header)</param> /// <returns></returns> private void CommitNewMessage(WrappedPageBlob blob, byte[] newCommit, StreamBlobHeader updatedHeader, HeaderDefinitionMetadata currentGoodHeaderDefinition, int nonAlignedBytesUsedAlready) { newCommit = newCommit ?? new byte[0]; var serializedHeader = _serializer.Serialize(updatedHeader); var writeStartLocationAligned = GetPageAlignedSize(nonAlignedBytesUsedAlready); var amountToWriteAligned = GetPageAlignedSize(serializedHeader.Length + newCommit.Length); var totalSpaceNeeded = writeStartLocationAligned + amountToWriteAligned; var newHeaderStartLocationNonAligned = writeStartLocationAligned + newCommit.Length; var totalBlobLength = blob.Properties.Length; if (totalBlobLength < totalSpaceNeeded) { blob.Resize(totalSpaceNeeded); totalBlobLength = blob.Properties.Length; } // set the header definition to make it all official bool isFirstWrite = currentGoodHeaderDefinition.HeaderSizeInBytes == 0; var headerDefinitionMetadata = new HeaderDefinitionMetadata(); headerDefinitionMetadata.HeaderSizeInBytes = serializedHeader.Length; headerDefinitionMetadata.HeaderStartLocationOffsetBytes = writeStartLocationAligned + newCommit.Length; blob.Metadata[_isEventStreamAggregateKey] = "yes"; blob.Metadata[_hasUndispatchedCommitsKey] = updatedHeader.PageBlobCommitDefinitions.Any((x) => !x.IsDispatched).ToString(); if (!isFirstWrite) { blob.Metadata[_secondaryHeaderDefinitionKey] = Convert.ToBase64String(currentGoodHeaderDefinition.GetRaw()); // this is a thirt layer backup in the case we have a issue in the middle of this upcoming write operation. var tempHeaderDefinition = currentGoodHeaderDefinition.Clone(); tempHeaderDefinition.HeaderStartLocationOffsetBytes = newHeaderStartLocationNonAligned; blob.Metadata[_terciaryHeaderDefintionKey] = Convert.ToBase64String(tempHeaderDefinition.GetRaw()); blob.Metadata[_firstWriteCompletedKey] = "t"; } else { blob.Metadata[_firstWriteCompletedKey] = "f"; } blob.Metadata[_primaryHeaderDefinitionKey] = Convert.ToBase64String(headerDefinitionMetadata.GetRaw()); blob.SetMetadata(); using (var ms = CreateAndFillStreamAligned(amountToWriteAligned, newCommit, serializedHeader)) { blob.Write(ms, writeStartLocationAligned, newHeaderStartLocationNonAligned, currentGoodHeaderDefinition); } // we pay the cost of an extra call for our first ever write (this is effectively creation of the aggregate. // we do this because we actually host our header in the blob, but the reference to that header in our metadata. // we set the metadata with the potential states prior to actually writing the new header. If this was the first // ever write and we set the metadata, but then fail to write the header, we can get in a state where the aggregate // becomes unusable because it believes there should be a header according to the metadata. // For that reason we must record when our first write completes if (isFirstWrite) { blob.Metadata[_firstWriteCompletedKey] = "t"; blob.SetMetadata(); } }