/// <summary>
/// Gets the deserialized header from the blob. Uses
/// </summary>
/// <param name="blob">The Blob.</param>
/// <param name="assumedValidHeaderDefinition">will output the header definition that is valid, null if there is not currently one</param>
/// <returns>A populated StreamBlobHeader.</returns>
private StreamBlobHeader GetHeaderWithRetry(WrappedPageBlob blob, out HeaderDefinitionMetadata validHeaderDefinition)
{
HeaderDefinitionMetadata assumedValidHeaderDefinition = null;
StreamBlobHeader header = null;
Exception lastException = null;
// first, if the primary header metadata key does not exist then we default
if (!blob.Metadata.ContainsKey(_primaryHeaderDefinitionKey))
{
assumedValidHeaderDefinition = new HeaderDefinitionMetadata();
header = new StreamBlobHeader();
}
// do the fallback logic to try and find a valid header
if (header == null)
{
for (int i = 0; i != 3; ++i)
{
assumedValidHeaderDefinition = GetHeaderDefinitionMetadata(blob, i);
header = SafeGetHeader(blob, assumedValidHeaderDefinition, out lastException);
if (header != null)
{
break;
}
}
}
// It is possible we will still have no header here and still be in an ok state. This is a case where the aggregates first
// commit set some metadata (specifically) the _primaryHeaderDefinitionKey, but then failed to write the header. This case
// will have a _primaryHeaderDefinitionKey but no secondary or terciary. In addition it will have a key of first write succeeded
// set to false. If it does not have any key at all, it is a "legacy" one and will get the key set upon a future successful write.
// legacy ones with issue will continue to fail and require manual intervention currently
if (header == null)
{
string firstWriteCompleted;
if (blob.Metadata.TryGetValue(_firstWriteCompletedKey, out firstWriteCompleted))
{
if (firstWriteCompleted == "f")
{
header = new StreamBlobHeader();
assumedValidHeaderDefinition = new HeaderDefinitionMetadata();
}
}
}
if (header != null)
{
validHeaderDefinition = assumedValidHeaderDefinition;
return(header);
}
throw lastException ?? new Exception("No header could be created");;
}
/// <summary>
/// Commits the header information which essentially commits any transactions that occured
/// related to that header.
/// </summary>
/// <param name="newCommit">the new commit to write</param>
/// <param name="headerStartOffsetBytes">where in the blob the new commit will be written</param>
/// <param name="blob">blob header applies to</param>
/// <param name="header">the new header data</param>
/// <returns></returns>
private void CommitNewMessage(WrappedPageBlob blob, byte[] newCommit, StreamBlobHeader header, int offsetBytes)
{
newCommit = newCommit ?? new byte[0];
var serializedHeader = _serializer.Serialize(header);
var writeStartLocationAligned = GetPageAlignedSize(offsetBytes);
var amountToWriteAligned = GetPageAlignedSize(serializedHeader.Length + newCommit.Length);
var totalSpaceNeeded = writeStartLocationAligned + amountToWriteAligned;
var totalBlobLength = blob.Properties.Length;
if (totalBlobLength < totalSpaceNeeded)
{
blob.Resize(totalSpaceNeeded);
totalBlobLength = blob.Properties.Length;
}
// set the header definition to make it all official
var headerDefinitionMetadata = new HeaderDefinitionMetadata();
headerDefinitionMetadata.HeaderSizeInBytes = serializedHeader.Length;
headerDefinitionMetadata.HeaderStartLocationOffsetBytes = writeStartLocationAligned + newCommit.Length;
blob.Metadata[_isEventStreamAggregateKey] = "yes";
blob.Metadata[_hasUndispatchedCommitsKey] = header.PageBlobCommitDefinitions.Any((x) => !x.IsDispatched).ToString();
if (blob.Metadata.ContainsKey(_primaryHeaderDefinitionKey))
{
blob.Metadata[_fallbackHeaderDefinitionKey] = blob.Metadata[_primaryHeaderDefinitionKey];
}
blob.Metadata[_primaryHeaderDefinitionKey] = Convert.ToBase64String(headerDefinitionMetadata.GetRaw());
blob.SetMetadata();
using (var ms = new MemoryStream(amountToWriteAligned))
{
ms.Write(newCommit, 0, newCommit.Length);
ms.Write(serializedHeader, 0, serializedHeader.Length);
var remainder = (ms.Position % _blobPageSize);
var fillSpace = amountToWriteAligned - newCommit.Length - serializedHeader.Length;
if (fillSpace != 0)
{
ms.Position += fillSpace - 1;
ms.WriteByte(0);
}
ms.Position = 0;
blob.Write(ms, writeStartLocationAligned);
}
}
/// <summary>
/// Safely receive the header.
/// </summary>
/// <param name="blob">blob</param>
/// <param name="headerDefinition">header definition</param>
/// <param name="exception">exception that occured</param>
/// <returns>the header, otherwise null if it could not be fetched</returns>
private StreamBlobHeader SafeGetHeader(WrappedPageBlob blob, HeaderDefinitionMetadata headerDefinition, out Exception exception)
{
StreamBlobHeader header = null;
exception = null;
try
{
var downloadedData = blob.DownloadBytes(headerDefinition.HeaderStartLocationOffsetBytes,
headerDefinition.HeaderStartLocationOffsetBytes + headerDefinition.HeaderSizeInBytes);
using (var ms = new MemoryStream(downloadedData, false))
{ header = _serializer.Deserialize <StreamBlobHeader>(ms.ToArray()); }
}
catch (Exception ex)
{ exception = ex; }
return(header);
}
/// <summary>
/// Safely receive the header.
/// </summary>
/// <param name="blob">blob</param>
/// <param name="headerDefinition">header definition</param>
/// <param name="exception">exception that occured</param>
/// <returns>the header, otherwise null if it could not be fetched</returns>
private StreamBlobHeader SafeGetHeader(WrappedPageBlob blob, HeaderDefinitionMetadata headerDefinition, out Exception exception)
{
StreamBlobHeader header = null;
exception = null;
try
{
if (headerDefinition == null || headerDefinition.HeaderSizeInBytes == 0)
{
throw new InvalidHeaderDataException(string.Format("Attempted to download a header, but the size specified is zero. This aggregate with id [{0}] may be corrupt.", blob.Name));
}
var downloadedData = blob.DownloadBytes(headerDefinition.HeaderStartLocationOffsetBytes,
headerDefinition.HeaderStartLocationOffsetBytes + headerDefinition.HeaderSizeInBytes);
using (var ms = new MemoryStream(downloadedData, false))
{ header = _serializer.Deserialize <StreamBlobHeader>(ms.ToArray()); }
}
catch (Exception ex)
{ exception = ex; }
return(header);
}
/// <summary>
/// Commits the header information which essentially commits any transactions that occurred
/// related to that header.
/// </summary>
/// <param name="newCommit">the new commit to write</param>
/// <param name="blob">blob header applies to</param>
/// <param name="updatedHeader">the new header to be serialized out</param>
/// <param name="currentGoodHeaderDefinition">the definition for the current header, before this change is committed</param>
/// <param name="nonAlignedBytesUsedAlready">non aligned offset of index where last commit data is stored (not inclusive of header)</param>
/// <returns></returns>
private void CommitNewMessage(WrappedPageBlob blob, byte[] newCommit,
StreamBlobHeader updatedHeader, HeaderDefinitionMetadata currentGoodHeaderDefinition,
int nonAlignedBytesUsedAlready)
{
newCommit = newCommit ?? new byte[0];
var serializedHeader = _serializer.Serialize(updatedHeader);
var writeStartLocationAligned = GetPageAlignedSize(nonAlignedBytesUsedAlready);
var amountToWriteAligned = GetPageAlignedSize(serializedHeader.Length + newCommit.Length);
var totalSpaceNeeded = writeStartLocationAligned + amountToWriteAligned;
var newHeaderStartLocationNonAligned = writeStartLocationAligned + newCommit.Length;
var totalBlobLength = blob.Properties.Length;
if (totalBlobLength < totalSpaceNeeded)
{
blob.Resize(totalSpaceNeeded);
totalBlobLength = blob.Properties.Length;
}
// set the header definition to make it all official
bool isFirstWrite = currentGoodHeaderDefinition.HeaderSizeInBytes == 0;
var headerDefinitionMetadata = new HeaderDefinitionMetadata();
headerDefinitionMetadata.HeaderSizeInBytes = serializedHeader.Length;
headerDefinitionMetadata.HeaderStartLocationOffsetBytes = writeStartLocationAligned + newCommit.Length;
blob.Metadata[_isEventStreamAggregateKey] = "yes";
blob.Metadata[_hasUndispatchedCommitsKey] = updatedHeader.PageBlobCommitDefinitions.Any((x) => !x.IsDispatched).ToString();
if (!isFirstWrite)
{
blob.Metadata[_secondaryHeaderDefinitionKey] = Convert.ToBase64String(currentGoodHeaderDefinition.GetRaw());
// this is a thirt layer backup in the case we have a issue in the middle of this upcoming write operation.
var tempHeaderDefinition = currentGoodHeaderDefinition.Clone();
tempHeaderDefinition.HeaderStartLocationOffsetBytes = newHeaderStartLocationNonAligned;
blob.Metadata[_terciaryHeaderDefintionKey] = Convert.ToBase64String(tempHeaderDefinition.GetRaw());
blob.Metadata[_firstWriteCompletedKey] = "t";
}
else
{
blob.Metadata[_firstWriteCompletedKey] = "f";
}
blob.Metadata[_primaryHeaderDefinitionKey] = Convert.ToBase64String(headerDefinitionMetadata.GetRaw());
blob.SetMetadata();
using (var ms = CreateAndFillStreamAligned(amountToWriteAligned, newCommit, serializedHeader))
{ blob.Write(ms, writeStartLocationAligned, newHeaderStartLocationNonAligned, currentGoodHeaderDefinition); }
// we pay the cost of an extra call for our first ever write (this is effectively creation of the aggregate.
// we do this because we actually host our header in the blob, but the reference to that header in our metadata.
// we set the metadata with the potential states prior to actually writing the new header. If this was the first
// ever write and we set the metadata, but then fail to write the header, we can get in a state where the aggregate
// becomes unusable because it believes there should be a header according to the metadata.
// For that reason we must record when our first write completes
if (isFirstWrite)
{
blob.Metadata[_firstWriteCompletedKey] = "t";
blob.SetMetadata();
}
}