/// <summary>
/// Ordered fetch by checkpoint
/// </summary>
/// <param name="checkpointToken"></param>
/// <returns>this method will get very slow as the number of aggregates increase</returns>
public IEnumerable <ICommit> GetFrom(string checkpointToken = null)
{
var containers = _blobClient.ListContainers(_eventSourcePrefix);
var allCommitDefinitions = new List <Tuple <WrappedPageBlob, PageBlobCommitDefinition> >();
foreach (var container in containers)
{
var blobs = WrappedPageBlob.GetAllMatchinPrefix(container, GetContainerName());
// this could be a tremendous amount of data. Depending on system used
// this may not be performant enough and may require some sort of index be built.
foreach (var pageBlob in blobs)
{
HeaderDefinitionMetadata headerDefinitionMetadata = null;
var header = GetHeaderWithRetry(pageBlob, out headerDefinitionMetadata);
foreach (var definition in header.PageBlobCommitDefinitions)
{
allCommitDefinitions.Add(new Tuple <WrappedPageBlob, PageBlobCommitDefinition>(pageBlob, definition));
}
}
}
// now sort the definitions so we can return out sorted
var orderedCommitDefinitions = allCommitDefinitions.OrderBy((x) => x.Item2.Checkpoint);
foreach (var orderedCommitDefinition in orderedCommitDefinitions)
{
yield return(CreateCommitFromDefinition(orderedCommitDefinition.Item1, orderedCommitDefinition.Item2));
}
}
/// <summary>
/// Gets the list of commits from a given blobEntry, starting from a given date
/// until the end date.
/// </summary>
/// <param name="bucketId">The blobEntry id to pull commits from.</param>
/// <param name="start">The starting date for commits.</param>
/// <param name="end">The ending date for commits.</param>
/// <returns>The list of commits from the given blobEntry and greater than or equal to the start date and less than or equal to the end date.</returns>
public IEnumerable <ICommit> GetFromTo(string bucketId, DateTime start, DateTime end)
{
var pageBlobs = WrappedPageBlob.GetAllMatchinPrefix(_primaryContainer, GetContainerName() + "/" + bucketId);
// this could be a tremendous amount of data. Depending on system used
// this may not be performant enough and may require some sort of index be built.
var allCommitDefinitions = new List <Tuple <WrappedPageBlob, PageBlobCommitDefinition> >();
foreach (var pageBlob in pageBlobs)
{
HeaderDefinitionMetadata headerDefinitionMetadata = null;
var header = GetHeaderWithRetry(pageBlob, out headerDefinitionMetadata);
foreach (var definition in header.PageBlobCommitDefinitions)
{
if (definition.CommitStampUtc >= start && definition.CommitStampUtc <= end)
{
allCommitDefinitions.Add(new Tuple <WrappedPageBlob, PageBlobCommitDefinition>(pageBlob, definition));
}
}
}
// now sort the definitions so we can return out sorted
var orderedCommitDefinitions = allCommitDefinitions.OrderBy((x) => x.Item2.CommitStampUtc);
foreach (var orderedCommitDefinition in orderedCommitDefinitions)
{
yield return(CreateCommitFromDefinition(orderedCommitDefinition.Item1, orderedCommitDefinition.Item2));
}
}
/// <summary>
/// Gets all undispatched commits across all buckets.
/// </summary>
/// <returns>A list of all undispatched commits.</returns>
public IEnumerable <ICommit> GetUndispatchedCommits()
{
Logger.Info("Getting undispatched commits. This is only done during initialization. This may take a while...");
var allCommitDefinitions = new List <Tuple <WrappedPageBlob, PageBlobCommitDefinition> >();
// this container is fetched lazily. so actually filtering down at this level will improve our performance,
// assuming the options dictate a date range that filters down our set.
var pageBlobs = WrappedPageBlob.GetAllMatchinPrefix(_primaryContainer, null);
Logger.Info("Checking [{0}] blobs for undispatched commits... this may take a while", pageBlobs.Count());
// this could be a tremendous amount of data. Depending on system used
// this may not be performant enough and may require some sort of index be built.
foreach (var pageBlob in pageBlobs)
{
var temp = pageBlob;
if (temp.Metadata.ContainsKey(_isEventStreamAggregateKey))
{
// we only care about guys who may be dirty
bool isDirty = false;
string isDirtyString;
if (temp.Metadata.TryGetValue(_hasUndispatchedCommitsKey, out isDirtyString))
{
isDirty = Boolean.Parse(isDirtyString);
}
if (isDirty)
{
Logger.Info("undispatched commit possibly found with aggregate [{0}]", temp.Name);
// Because fetching the header for a specific blob is a two phase operation it may take a couple tries if we are working with the
// blob. This is just a quality of life improvement for the user of the store so loading of the store does not hit frequent optimistic
// concurrency hits that cause the store to have to re-initialize.
var maxTries = 0;
while (true)
{
try
{
HeaderDefinitionMetadata headerDefinitionMetadata = null;
var header = GetHeaderWithRetry(temp, out headerDefinitionMetadata);
bool wasActuallyDirty = false;
if (header.UndispatchedCommitCount > 0)
{
foreach (var definition in header.PageBlobCommitDefinitions)
{
if (!definition.IsDispatched)
{
Logger.Warn("Found undispatched commit for stream [{0}] revision [{1}]", temp.Name, definition.Revision);
wasActuallyDirty = true;
allCommitDefinitions.Add(new Tuple <WrappedPageBlob, PageBlobCommitDefinition>(temp, definition));
}
}
}
if (!wasActuallyDirty)
{
temp.Metadata[_hasUndispatchedCommitsKey] = false.ToString();
temp.SetMetadata();
}
break;
}
catch (ConcurrencyException)
{
if (maxTries++ > 20)
{
Logger.Error("Reached max tries for getting undispatched commits and keep receiving concurrency exception. throwing out.");
throw;
}
else
{
Logger.Info("Concurrency issue detected while processing undispatched commits. going to retry to load container");
try
{ temp = WrappedPageBlob.GetAllMatchinPrefix(_primaryContainer, pageBlob.Name).Single(); }
catch (Exception ex)
{ Logger.Warn("Attempted to reload during concurrency and failed... will retry. [{0}]", ex.Message); }
}
}
catch (CryptographicException ex)
{
Logger.Fatal("Received a CryptographicException while processing aggregate with id [{0}]. The header is possibly be corrupt. Error is [{1}]",
pageBlob.Name, ex.ToString());
break;
}
catch (InvalidHeaderDataException ex)
{
Logger.Fatal("Received a InvalidHeaderDataException while processing aggregate with id [{0}]. The header is possibly be corrupt. Error is [{1}]",
pageBlob.Name, ex.ToString());
break;
}
}
}
}
}
// now sort the definitions so we can return out sorted
Logger.Info("Found [{0}] undispatched commits", allCommitDefinitions.Count);
var orderedCommitDefinitions = allCommitDefinitions.OrderBy((x) => x.Item2.Checkpoint);
foreach (var orderedCommitDefinition in orderedCommitDefinitions)
{
yield return(CreateCommitFromDefinition(orderedCommitDefinition.Item1, orderedCommitDefinition.Item2));
}
}
/// <summary>
/// Gets all undispatched commits across all buckets.
/// </summary>
/// <returns>A list of all undispatched commits.</returns>
public IEnumerable <ICommit> GetUndispatchedCommits()
{
// this is most likely extremely inefficient as the size of our store grows to 100's of millions of streams (possibly even just 1000's)
var containers = _blobClient.ListContainers();
var allCommitDefinitions = new List <Tuple <WrappedPageBlob, PageBlobCommitDefinition> >();
foreach (var container in containers)
{
DateTime sinceDateUtc = DateTime.MinValue;
try
{
// only need to subtract if something different than TimeSpan.MaxValue is specified
if (!_options.MaxTimeSpanForUndispatched.Equals(TimeSpan.MaxValue))
{
sinceDateUtc = DateTime.UtcNow.Subtract(_options.MaxTimeSpanForUndispatched);
}
}
catch (ArgumentOutOfRangeException)
{ Logger.Info("Date Time was out of range. falling back to the smallest date/time possible"); }
// this container is fetched lazily. so actually filtering down at this level will improve our performance,
// assuming the options dictate a date range that filters down our set.
var pageBlobs = WrappedPageBlob.GetAllMatchinPrefix(container, null);
pageBlobs = pageBlobs.OrderByDescending((x) => x.Properties.LastModified).Where((x) => x.Properties.LastModified > sinceDateUtc);
// this could be a tremendous amount of data. Depending on system used
// this may not be performant enough and may require some sort of index be built.
foreach (var pageBlob in pageBlobs)
{
if (pageBlob.Metadata.ContainsKey(_isEventStreamAggregateKey))
{
// we only care about guys who may be dirty
bool isDirty = true;
string isDirtyString;
if (pageBlob.Metadata.TryGetValue(_hasUndispatchedCommitsKey, out isDirtyString))
{
isDirty = Boolean.Parse(isDirtyString);
}
if (isDirty)
{
// Because fetching the header for a specific blob is a two phase operation it may take a couple tries if we are working with the
// blob. This is just a quality of life improvement for the user of the store so loading of the store does not hit frequent optimistic
// concurrency hits that cause the store to have to re-initialize.
var maxTries = 0;
while (true)
{
try
{
var header = GetHeaderWithRetry(pageBlob, null);
if (header.UndispatchedCommitCount > 0)
{
foreach (var definition in header.PageBlobCommitDefinitions)
{
if (!definition.IsDispatched)
{
allCommitDefinitions.Add(new Tuple <WrappedPageBlob, PageBlobCommitDefinition>(pageBlob, definition));
}
}
}
break;
}
catch (ConcurrencyException)
{
if (maxTries++ > 20)
{
Logger.Error("Reached max tries for getting undispatched commits and keep receiving concurrency exception. throwing out.");
throw;
}
else
{
Logger.Info("Concurrency issue detected while processing undispatched commits. going to retry to load container");
}
}
}
}
}
}
}
// now sort the definitions so we can return out sorted
var orderedCommitDefinitions = allCommitDefinitions.OrderBy((x) => x.Item2.CommitStampUtc);
foreach (var orderedCommitDefinition in orderedCommitDefinitions)
{
yield return(CreateCommitFromDefinition(orderedCommitDefinition.Item1, orderedCommitDefinition.Item2));
}
}