/// <summary> /// Safely receive the header. /// </summary> /// <param name="blob">blob</param> /// <param name="headerDefinition">header definition</param> /// <param name="exception">exception that occured</param> /// <returns>the header, otherwise null if it could not be fetched</returns> private StreamBlobHeader SafeGetHeader(WrappedPageBlob blob, HeaderDefinitionMetadata headerDefinition, out Exception exception) { StreamBlobHeader header = null; exception = null; try { if (headerDefinition == null || headerDefinition.HeaderSizeInBytes == 0) { throw new InvalidHeaderDataException( $"Attempted to download a header, but the size specified is zero. This aggregate with id [{blob.Name}] may be corrupt."); } var downloadedData = blob.DownloadBytes(headerDefinition.HeaderStartLocationOffsetBytes, headerDefinition.HeaderStartLocationOffsetBytes + headerDefinition.HeaderSizeInBytes); using (var ms = new MemoryStream(downloadedData, false)) { header = _serializer.Deserialize <StreamBlobHeader>(ms.ToArray()); } } catch (Exception ex) { exception = ex; } return(header); }
/// <summary> /// Gets the deserialized header from the blob. Uses /// </summary> /// <param name="blob">The Blob.</param> /// <param name="assumedValidHeaderDefinition">will output the header definition that is valid, null if there is not currently one</param> /// <returns>A populated StreamBlobHeader.</returns> private StreamBlobHeader GetHeaderWithRetry(WrappedPageBlob blob, out HeaderDefinitionMetadata validHeaderDefinition) { HeaderDefinitionMetadata assumedValidHeaderDefinition = null; StreamBlobHeader header = null; Exception lastException = null; // first, if the primary header metadata key does not exist then we default if (!blob.Metadata.ContainsKey(PrimaryHeaderDefinitionKey)) { assumedValidHeaderDefinition = new HeaderDefinitionMetadata(); header = new StreamBlobHeader(); } // do the fallback logic to try and find a valid header if (header == null) { for (var i = 0; i != 3; ++i) { assumedValidHeaderDefinition = GetHeaderDefinitionMetadata(blob, i); header = SafeGetHeader(blob, assumedValidHeaderDefinition, out lastException); if (header != null) { break; } } } // It is possible we will still have no header here and still be in an ok state. This is a case where the aggregates first // commit set some metadata (specifically) the PrimaryHeaderDefinitionKey, but then failed to write the header. This case // will have a PrimaryHeaderDefinitionKey but no secondary or terciary. In addition it will have a key of first write succeeded // set to false. If it does not have any key at all, it is a "legacy" one and will get the key set upon a future successful write. // legacy ones with issue will continue to fail and require manual intervention currently if (header == null) { string firstWriteCompleted; if (blob.Metadata.TryGetValue(FirstWriteCompletedKey, out firstWriteCompleted)) { if (firstWriteCompleted == "f") { header = new StreamBlobHeader(); assumedValidHeaderDefinition = new HeaderDefinitionMetadata(); } } } if (header != null) { validHeaderDefinition = assumedValidHeaderDefinition; return(header); } throw lastException ?? new Exception("No header could be created");; }
/// <summary> /// Commits the header information which essentially commits any transactions that occurred /// related to that header. /// </summary> /// <param name="newCommit">the new commit to write</param> /// <param name="blob">blob header applies to</param> /// <param name="updatedHeader">the new header to be serialized out</param> /// <param name="currentGoodHeaderDefinition">the definition for the current header, before this change is committed</param> /// <param name="nonAlignedBytesUsedAlready">non aligned offset of index where last commit data is stored (not inclusive of header)</param> /// <returns></returns> private void CommitNewMessage(WrappedPageBlob blob, byte[] newCommit, StreamBlobHeader updatedHeader, HeaderDefinitionMetadata currentGoodHeaderDefinition, int nonAlignedBytesUsedAlready) { newCommit = newCommit ?? new byte[0]; var serializedHeader = _serializer.Serialize(updatedHeader); var writeStartLocationAligned = GetPageAlignedSize(nonAlignedBytesUsedAlready); var amountToWriteAligned = GetPageAlignedSize(serializedHeader.Length + newCommit.Length); var totalSpaceNeeded = writeStartLocationAligned + amountToWriteAligned; var newHeaderStartLocationNonAligned = writeStartLocationAligned + newCommit.Length; var totalBlobLength = blob.Properties.Length; if (totalBlobLength < totalSpaceNeeded) { blob.Resize(totalSpaceNeeded); totalBlobLength = blob.Properties.Length; } // set the header definition to make it all official var isFirstWrite = currentGoodHeaderDefinition.HeaderSizeInBytes == 0; var headerDefinitionMetadata = new HeaderDefinitionMetadata(); headerDefinitionMetadata.HeaderSizeInBytes = serializedHeader.Length; headerDefinitionMetadata.HeaderStartLocationOffsetBytes = writeStartLocationAligned + newCommit.Length; blob.Metadata[IsEventStreamAggregateKey] = "yes"; blob.Metadata[HasUndispatchedCommitsKey] = updatedHeader.PageBlobCommitDefinitions.Any((x) => !x.IsDispatched).ToString(); if (!isFirstWrite) { blob.Metadata[SecondaryHeaderDefinitionKey] = Convert.ToBase64String(currentGoodHeaderDefinition.GetRaw()); // this is a thirt layer backup in the case we have a issue in the middle of this upcoming write operation. var tempHeaderDefinition = currentGoodHeaderDefinition.Clone(); tempHeaderDefinition.HeaderStartLocationOffsetBytes = newHeaderStartLocationNonAligned; blob.Metadata[TertiaryHeaderDefintionKey] = Convert.ToBase64String(tempHeaderDefinition.GetRaw()); blob.Metadata[FirstWriteCompletedKey] = "t"; } else { blob.Metadata[FirstWriteCompletedKey] = "f"; } blob.Metadata[PrimaryHeaderDefinitionKey] = Convert.ToBase64String(headerDefinitionMetadata.GetRaw()); blob.SetMetadata(); using (var ms = CreateAndFillStreamAligned(amountToWriteAligned, newCommit, serializedHeader)) { blob.Write(ms, writeStartLocationAligned, newHeaderStartLocationNonAligned, currentGoodHeaderDefinition); } // we pay the cost of an extra call for our first ever write (this is effectively creation of the aggregate. // we do this because we actually host our header in the blob, but the reference to that header in our metadata. // we set the metadata with the potential states prior to actually writing the new header. If this was the first // ever write and we set the metadata, but then fail to write the header, we can get in a state where the aggregate // becomes unusable because it believes there should be a header according to the metadata. // For that reason we must record when our first write completes if (isFirstWrite) { blob.Metadata[FirstWriteCompletedKey] = "t"; blob.SetMetadata(); } }