/// <summary> /// Gets the list of commits from a given blobEntry, starting from a given date /// until the end date. /// </summary> /// <param name="bucketId">The blobEntry id to pull commits from.</param> /// <param name="start">The starting date for commits.</param> /// <param name="end">The ending date for commits.</param> /// <returns>The list of commits from the given blobEntry and greater than or equal to the start date and less than or equal to the end date.</returns> public IEnumerable <ICommit> GetFromTo(string bucketId, DateTime start, DateTime end) { var pageBlobs = WrappedPageBlob.GetAllMatchinPrefix(_primaryContainer, GetContainerName() + "/" + bucketId); // this could be a tremendous amount of data. Depending on system used // this may not be performant enough and may require some sort of index be built. var allCommitDefinitions = new List <Tuple <WrappedPageBlob, PageBlobCommitDefinition> >(); foreach (var pageBlob in pageBlobs) { HeaderDefinitionMetadata headerDefinitionMetadata = null; var header = GetHeaderWithRetry(pageBlob, out headerDefinitionMetadata); foreach (var definition in header.PageBlobCommitDefinitions) { if (definition.CommitStampUtc >= start && definition.CommitStampUtc <= end) { allCommitDefinitions.Add( new Tuple <WrappedPageBlob, PageBlobCommitDefinition>( pageBlob, definition)); } } } // now sort the definitions so we can return out sorted var orderedCommitDefinitions = allCommitDefinitions.OrderBy((x) => x.Item2.CommitStampUtc); foreach (var orderedCommitDefinition in orderedCommitDefinitions) { yield return(CreateCommitFromDefinition(orderedCommitDefinition.Item1, orderedCommitDefinition.Item2)); } }
/// <summary> /// Ordered fetch by checkpoint /// </summary> /// <param name="checkpointToken"></param> /// <returns>this method will get very slow as the number of aggregates increase</returns> public IEnumerable <ICommit> GetFrom(string checkpointToken = null) { var containers = _blobClient.ListContainers(EventSourcePrefix); var allCommitDefinitions = new List <Tuple <WrappedPageBlob, PageBlobCommitDefinition> >(); foreach (var container in containers) { var blobs = WrappedPageBlob.GetAllMatchinPrefix(container, GetContainerName()); // this could be a tremendous amount of data. Depending on system used // this may not be performant enough and may require some sort of index be built. foreach (var pageBlob in blobs) { HeaderDefinitionMetadata headerDefinitionMetadata = null; var header = GetHeaderWithRetry(pageBlob, out headerDefinitionMetadata); foreach (var definition in header.PageBlobCommitDefinitions) { allCommitDefinitions.Add( new Tuple <WrappedPageBlob, PageBlobCommitDefinition>( pageBlob, definition)); } } } // now sort the definitions so we can return out sorted var orderedCommitDefinitions = allCommitDefinitions.OrderBy((x) => x.Item2.Checkpoint); foreach (var orderedCommitDefinition in orderedCommitDefinitions) { yield return(CreateCommitFromDefinition(orderedCommitDefinition.Item1, orderedCommitDefinition.Item2)); } }
/// <summary> /// Gets all undispatched commits across all buckets. /// </summary> /// <returns>A list of all undispatched commits.</returns> public IEnumerable <ICommit> GetUndispatchedCommits() { Logger.Info("Getting undispatched commits. This is only done during initialization. This may take a while..."); var allCommitDefinitions = new List <Tuple <WrappedPageBlob, PageBlobCommitDefinition> >(); // this container is fetched lazily. so actually filtering down at this level will improve our performance, // assuming the options dictate a date range that filters down our set. var pageBlobs = WrappedPageBlob.GetAllMatchinPrefix(_primaryContainer, null); Logger.Info("Checking [{0}] blobs for undispatched commits... this may take a while", pageBlobs.Count()); // this could be a tremendous amount of data. Depending on system used // this may not be performant enough and may require some sort of index be built. foreach (var pageBlob in pageBlobs) { var temp = pageBlob; if (temp.Metadata.ContainsKey(IsEventStreamAggregateKey)) { // we only care about guys who may be dirty var isDirty = false; string isDirtyString; if (temp.Metadata.TryGetValue(HasUndispatchedCommitsKey, out isDirtyString)) { isDirty = bool.Parse(isDirtyString); } if (isDirty) { Logger.Info("undispatched commit possibly found with aggregate [{0}]", temp.Name); // Because fetching the header for a specific blob is a two phase operation it may take a couple tries if we are working with the // blob. This is just a quality of life improvement for the user of the store so loading of the store does not hit frequent optimistic // concurrency hits that cause the store to have to re-initialize. var maxTries = 0; while (true) { try { HeaderDefinitionMetadata headerDefinitionMetadata = null; var header = GetHeaderWithRetry(temp, out headerDefinitionMetadata); var wasActuallyDirty = false; if (header.UndispatchedCommitCount > 0) { foreach (var definition in header.PageBlobCommitDefinitions) { if (!definition.IsDispatched) { Logger.Warn("Found undispatched commit for stream [{0}] revision [{1}]", temp.Name, definition.Revision); wasActuallyDirty = true; allCommitDefinitions.Add(new Tuple <WrappedPageBlob, PageBlobCommitDefinition>(temp, definition)); } } } if (!wasActuallyDirty) { temp.Metadata[HasUndispatchedCommitsKey] = false.ToString(); temp.SetMetadata(); } break; } catch (ConcurrencyException) { if (maxTries++ > 20) { Logger.Error("Reached max tries for getting undispatched commits and keep receiving concurrency exception. throwing out."); throw; } else { Logger.Info("Concurrency issue detected while processing undispatched commits. going to retry to load container"); try { temp = WrappedPageBlob .GetAllMatchinPrefix(_primaryContainer, pageBlob.Name) .Single(); } catch (Exception ex) { Logger.Warn("Attempted to reload during concurrency and failed... will retry. [{0}]", ex.Message); } } } catch (CryptographicException ex) { Logger.Fatal("Received a CryptographicException while processing aggregate with id [{0}]. The header is possibly be corrupt. Error is [{1}]", pageBlob.Name, ex.ToString()); break; } catch (InvalidHeaderDataException ex) { Logger.Fatal("Received a InvalidHeaderDataException while processing aggregate with id [{0}]. The header is possibly be corrupt. Error is [{1}]", pageBlob.Name, ex.ToString()); break; } } } } } // now sort the definitions so we can return out sorted Logger.Info("Found [{0}] undispatched commits", allCommitDefinitions.Count); var orderedCommitDefinitions = allCommitDefinitions.OrderBy((x) => x.Item2.Checkpoint); foreach (var orderedCommitDefinition in orderedCommitDefinitions) { yield return(CreateCommitFromDefinition(orderedCommitDefinition.Item1, orderedCommitDefinition.Item2)); } }