public Task <TAggregate> GetById <TAggregate>(IDomainId id, int version) where TAggregate : class, IAggregate
{
var aggregate = ConstructAggregate <TAggregate>();
Events.ForEach(aggregate.ApplyEvent);
return(Task.FromResult(aggregate));
}
/// <inheritdoc />
public Task <SymbolBlobIdentifier> UploadFileAsync(IDomainId domainId, Uri blobStoreUri, string requestId, string filename, BlobIdentifier blobIdentifier, CancellationToken cancellationToken)
{
return(RetryAsync(
nameof(ISymbolServiceClient.UploadFileAsync),
(client, ct) => client.UploadFileAsync(domainId, blobStoreUri, requestId, filename, blobIdentifier, ct),
cancellationToken));
}
/// <inheritdoc />
public Task<DropItem> CreateAsync(IDomainId domainId, string dropName, bool isAppendOnly, DateTime? expirationDate, bool chunkDedup, CancellationToken cancellationToken)
{
return RetryAsync(
nameof(IDropServiceClient.CreateAsync),
(client, ct) => client.CreateAsync(domainId, dropName, isAppendOnly, expirationDate, chunkDedup, cancellationToken),
cancellationToken);
}
/// <inheritdoc />
public Task <Request> CreateRequestAsync(IDomainId domainId, string requestName, bool isChunked, CancellationToken cancellationToken)
{
return(RetryAsync(
nameof(ISymbolServiceClient.CreateRequestAsync),
(client, ct) => client.CreateRequestAsync(domainId, requestName, isChunked, ct),
cancellationToken));
}
/// <summary>
/// Creates a symbol request.
/// </summary>
public async Task <Request> CreateAsync(CancellationToken token)
{
if (!m_config.DomainId.HasValue)
{
m_logger.Verbose("DomainId is not specified. Creating symbol publishing request using DefaultDomainId.");
}
IDomainId domainId = m_config.DomainId.HasValue
? new ByteDomainId(m_config.DomainId.Value)
: WellKnownDomainIds.DefaultDomainId;
Request result;
using (m_counters.StartStopwatch(SymbolClientCounter.CreateDuration))
{
result = await m_symbolClient.CreateRequestAsync(domainId, RequestName, m_config.EnableChunkDedup, token);
}
m_requestId = result.Id;
m_domainId = result.DomainId;
// info about a request in a human-readable form
var requestDetails = $"Symbol request has been created:{Environment.NewLine}"
+ $"ID: {result.Id}{Environment.NewLine}"
+ $"Name: {result.Name}{Environment.NewLine}"
+ $"Content list: '{result.Url}/DebugEntries'";
// Send the message to the main log.
Analysis.IgnoreResult(await m_apiClient.LogMessage(requestDetails));
m_logger.Verbose(requestDetails);
return(result);
}
/// <inheritdoc />
public Task <DropItem> CreateAsync(IDomainId domainId, string dropName, bool isAppendOnly, DateTime?expirationDate, bool chunkDedup, bool enableSymbolicLinkPreservation, bool enableExecutionBitPreservation, CancellationToken cancellationToken)
{
return(RetryAsync(
nameof(IDropServiceClient.CreateAsync),
(client, ct) => client.CreateAsync(domainId, dropName, isAppendOnly, expirationDate, chunkDedup, enableSymbolicLinkPreservation, enableExecutionBitPreservation, cancellationToken),
cancellationToken));
}
/// <summary>
/// Initializes a new instance of the <see cref="BackingContentStoreHttpClientFactory"/> class.
/// </summary>
/// <param name="backingStoreBaseUri">The base VSTS Uri to the dedup service.</param>
/// <param name="vssCredentialsFactory">A provider for credentials connecting to the factory.</param>
/// <param name="httpSendTimeout">How long to wait for a response after sending an http request before timing out.</param>
/// <param name="domain">Domain ID to use against the backing store.</param>
/// <param name="useAad">Whether or not to use production AAD for authentication.</param>
public BackingContentStoreHttpClientFactory(Uri backingStoreBaseUri, VssCredentialsFactory vssCredentialsFactory, TimeSpan httpSendTimeout, IDomainId domain, bool useAad = true)
{
_backingStoreBaseUri = backingStoreBaseUri;
_vssCredentialsFactory = vssCredentialsFactory;
_httpSendTimeout = httpSendTimeout;
_useAad = useAad;
_domain = domain;
}
protected async Task <TAggregate> Get <TAggregate>(IDomainId id) where TAggregate : AggregateBase
{
var aggregate = await Repository.GetById <TAggregate>(id.Value);
if (aggregate == null)
{
throw new Exception($"{typeof(TAggregate).Name} not found");
}
return(aggregate);
}
private static DebugEntry CreateDebugEntry(IDebugEntryData data, IDomainId domainId)
{
return(new DebugEntry()
{
BlobIdentifier = data.BlobIdentifier,
ClientKey = data.ClientKey,
InformationLevel = data.InformationLevel,
DomainId = domainId,
});
}
/// <inheritdoc />
public Task <IEnumerable <Request> > GetAllRequestsAsync(CancellationToken cancellationToken,
SizeOptions sizeOptions = null,
ExpirationDateOptions expirationDateOptions = null,
IDomainId domainIdOption = null,
RetrievalOptions retrievalOptions = RetrievalOptions.ExcludeSoftDeleted,
RequestStatus?requestStatus = null)
{
return(RetryAsync(
nameof(ISymbolServiceClient.GetAllRequestsAsync),
(client, ct) => client.GetAllRequestsAsync(ct, sizeOptions, expirationDateOptions, domainIdOption, retrievalOptions, requestStatus),
cancellationToken));
}
protected Event(IDomainId aggregateId, Guid correlationId, string who = "anonymous")
{
Headers = new EventHeaders()
{
Who = who,
CorrelationId = correlationId,
When = DateTime.UtcNow,
AggregateType = GetType().Name
};
MessageId = Guid.NewGuid();
AggregateId = aggregateId;
}
protected Command(IDomainId aggregateId, Guid correlationId, string who = "anonymous")
{
Headers = new CommandHeaders()
{
Who = who,
CorrelationId = correlationId,
When = DateTime.UtcNow,
AggregateType = GetType().Name
};
CommitId = Guid.NewGuid();
CommitDate = DateTime.UtcNow;
AggregateId = aggregateId;
}
/// <summary>
/// Queries the endpoint for the RequestId.
/// This method should be called only after the request has been created, otherwise, it will throw an exception.
/// </summary>
/// <remarks>
/// On workers, m_requestId / m_domainId won't be initialized, so we need to query the server for the right values.
/// </remarks>
private async Task EnsureRequestIdAndDomainIdAreInitalizedAsync()
{
if (string.IsNullOrEmpty(m_requestId) || m_domainId == null)
{
using (m_counters.StartStopwatch(SymbolClientCounter.GetRequestIdDuration))
{
var result = await m_symbolClient.GetRequestByNameAsync(RequestName, CancellationToken);
m_requestId = result.Id;
m_domainId = result.DomainId;
}
}
}
/// <inheritdoc />
public Task <IEnumerable <Request> > GetRequestPaginatedAsync(
String continueFromRequestId,
int pageSize,
CancellationToken cancellationToken,
SizeOptions sizeOptions = null,
ExpirationDateOptions expirationDateOptions = null,
IDomainId domainIdOption = null,
RetrievalOptions retrievalOptions = RetrievalOptions.ExcludeSoftDeleted,
RequestStatus?requestStatus = null)
{
return(RetryAsync(
nameof(ISymbolServiceClient.GetRequestPaginatedAsync),
(client, ct) => client.GetRequestPaginatedAsync(continueFromRequestId, pageSize, ct, sizeOptions, expirationDateOptions, domainIdOption, retrievalOptions, requestStatus),
cancellationToken));
}
/// <inheritdoc />
public Task <IEnumerable <DropItem> > ListAsync(
string dropNamePrefix,
PathOptions pathOptions,
bool includeNonFinalizedDrops,
CancellationToken cancellationToken,
RetrievalOptions retrievalOptions,
SizeOptions sizeOptions,
ExpirationDateOptions expirationDateOptions,
IDomainId domainId,
int pageSize = -1,
string continueFromDropName = null)
{
return(RetryAsync(
nameof(IDropServiceClient.ListAsync),
(client, ct) => client.ListAsync(dropNamePrefix, pathOptions, includeNonFinalizedDrops, ct, retrievalOptions, sizeOptions, expirationDateOptions, domainId, pageSize, continueFromDropName),
cancellationToken));
}
public Task <TAggregate> GetById <TAggregate>(IDomainId id, int version) where TAggregate : class, IAggregate
{
if (version <= 0)
{
throw new InvalidOperationException("Cannot get version <= 0");
}
var streamName = aggregateIdToStreamName(typeof(TAggregate), id.Value);
var aggregate = ConstructAggregate <TAggregate>();
long sliceStart = 0;
StreamEventsSlice currentSlice;
do
{
var sliceCount = (int)(sliceStart + ReadPageSize <= version ? ReadPageSize : version - sliceStart + 1);
currentSlice = eventStoreConnection.ReadStreamEventsForwardAsync(streamName, sliceStart, sliceCount, false).Result;
if (currentSlice.Status == SliceReadStatus.StreamNotFound)
{
throw new AggregateNotFoundException(id.Value, typeof(TAggregate));
}
if (currentSlice.Status == SliceReadStatus.StreamDeleted)
{
throw new AggregateDeletedException(id.Value, typeof(TAggregate));
}
sliceStart = currentSlice.NextEventNumber;
foreach (var @event in currentSlice.Events)
{
aggregate.ApplyEvent(DeserializeEvent(@event.OriginalEvent.Metadata, @event.OriginalEvent.Data));
}
} while (version >= currentSlice.NextEventNumber && !currentSlice.IsEndOfStream);
if (aggregate.Version != version && version < int.MaxValue)
{
throw new AggregateVersionException(id.Value, typeof(TAggregate), aggregate.Version, version);
}
return(Task.FromResult(aggregate));
}
/// <summary>
/// Queries the endpoint for the RequestId.
/// This method should be called only after the request has been created, otherwise, it will throw an exception.
/// </summary>
/// <remarks>
/// On workers, m_requestId / m_domainId won't be initialized, so we need to query the server for the right values.
/// </remarks>
private async Task EnsureRequestIdAndDomainIdAreInitalizedAsync()
{
// Check whether the field is initialized, so we are not wastefully acquire the semaphore.
if (string.IsNullOrEmpty(m_requestId))
{
using (await m_requestIdAcquisitionMutex.AcquireAsync())
{
// check whether we still need to query the server
if (string.IsNullOrEmpty(m_requestId) || m_domainId == null)
{
using (m_counters.StartStopwatch(SymbolClientCounter.GetRequestIdDuration))
{
var result = await m_symbolClient.GetRequestByNameAsync(RequestName, CancellationToken);
m_requestId = result.Id;
m_domainId = result.DomainId;
}
}
}
}
}
/// <summary>
/// Invokes <see cref="IDropServiceClient.CreateAsync(string, bool, DateTime?, bool, CancellationToken)"/>.
/// </summary>
public async Task <DropItem> CreateAsync(CancellationToken token)
{
var startTime = DateTime.UtcNow;
if (!m_config.DomainId.HasValue)
{
m_logger.Verbose("Domain ID is not specified. Creating drop using a default domain id.");
}
IDomainId domainId = m_config.DomainId.HasValue
? new ByteDomainId(m_config.DomainId.Value)
: WellKnownDomainIds.DefaultDomainId;
var result = await m_dropClient.CreateAsync(
domainId,
DropName,
isAppendOnly : true,
expirationDate : DateTime.UtcNow.Add(m_config.Retention),
chunkDedup : m_config.EnableChunkDedup,
cancellationToken : token);
Interlocked.Add(ref Stats.CreateTimeMs, ElapsedMillis(startTime));
return(result);
}
public FakeResponseError(IDomainId aggregateId, Guid correlationId, string value1, string who = "anonymous") : base(aggregateId, correlationId, who)
{
Value1 = value1;
}
public Task <DropItem> CreateAsync(IDomainId domainId, string dropName, bool isAppendOnly, DateTime?expirationDate, bool chunkDedup, bool enableSymbolicLinkPreservation, bool enableExecutionBitPreservation, CancellationToken cancellationToken)
{
throw new NotImplementedException();
}
public Task <DropItem> CreateAsync(IDomainId domainId, string dropName, bool isAppendOnly, DateTime?expirationDate, bool chunkDedup, CancellationToken cancellationToken)
{
throw new NotImplementedException();
}
Task <IEnumerable <DropItem> > IDropServiceClient.ListAsync(string dropNamePrefix, PathOptions pathOptions, bool includeNonFinalizedDrops, CancellationToken cancellationToken, RetrievalOptions retrievalOptions, SizeOptions sizeOptions, ExpirationDateOptions expirationDateOptions, IDomainId domainId, int pageSize, string continueFromDropName)
{
throw new NotImplementedException();
}
/// <summary>
/// Initializes a new instance of the <see cref="BuildCacheCache"/> class.
/// </summary>
/// <param name="fileSystem">Filesystem used to read/write files.</param>
/// <param name="cacheNamespace">the namespace of the cache that is communicated with.</param>
/// <param name="buildCacheHttpClientFactory">Factory for creatign a backing BuildCache http client.</param>
/// <param name="backingContentStoreHttpClientFactory">Factory for creating a backing store http client.</param>
/// <param name="maxFingerprintSelectorsToFetch">Maximum number of selectors to enumerate.</param>
/// <param name="timeToKeepUnreferencedContent">Initial time-to-live for unreferenced content.</param>
/// <param name="pinInlineThreshold">Maximum time-to-live to inline pin calls.</param>
/// <param name="ignorePinThreshold">Minimum time-to-live to ignore pin calls.</param>
/// <param name="minimumTimeToKeepContentHashLists">Minimum time-to-live for created or referenced ContentHashLists.</param>
/// <param name="rangeOfTimeToKeepContentHashLists">Range of time beyond the minimum for the time-to-live of created or referenced ContentHashLists.</param>
/// <param name="logger">A logger for tracing.</param>
/// <param name="fingerprintIncorporationEnabled">Feature flag to enable fingerprints incorporation on shutdown</param>
/// <param name="maxDegreeOfParallelismForIncorporateRequests">Throttle the number of fingerprints chunks sent in parallel</param>
/// <param name="maxFingerprintsPerIncorporateRequest">Max fingerprints allowed per chunk</param>
/// <param name="domain">Domain ID to use against BlobStore or DedupStore</param>
/// <param name="writeThroughContentStoreFunc">Optional write-through store to allow writing-behind to BlobStore</param>
/// <param name="sealUnbackedContentHashLists">If true, the client will attempt to seal any unbacked ContentHashLists that it sees.</param>
/// <param name="useBlobContentHashLists">use blob based content hash lists.</param>
/// <param name="useDedupStore">If true, gets content through DedupStore. If false, gets content from BlobStore.</param>
/// <param name="overrideUnixFileAccessMode">If true, overrides default Unix file access modes.</param>
/// <param name="enableEagerFingerprintIncorporation"><see cref="BuildCacheServiceConfiguration.EnableEagerFingerprintIncorporation"/></param>
/// <param name="inlineFingerprintIncorporationExpiry"><see cref="BuildCacheServiceConfiguration.InlineFingerprintIncorporationExpiry"/></param>
/// <param name="eagerFingerprintIncorporationNagleInterval"><see cref="BuildCacheServiceConfiguration.EagerFingerprintIncorporationNagleInterval"/></param>
/// <param name="eagerFingerprintIncorporationNagleBatchSize"><see cref="BuildCacheServiceConfiguration.EagerFingerprintIncorporationNagleBatchSize"/></param>
/// <param name="downloadBlobsUsingHttpClient"><see cref="BuildCacheServiceConfiguration.DownloadBlobsUsingHttpClient"/></param>
public BuildCacheCache(
IAbsFileSystem fileSystem,
string cacheNamespace,
IBuildCacheHttpClientFactory buildCacheHttpClientFactory,
IArtifactHttpClientFactory backingContentStoreHttpClientFactory,
int maxFingerprintSelectorsToFetch,
TimeSpan timeToKeepUnreferencedContent,
TimeSpan pinInlineThreshold,
TimeSpan ignorePinThreshold,
TimeSpan minimumTimeToKeepContentHashLists,
TimeSpan rangeOfTimeToKeepContentHashLists,
ILogger logger,
bool fingerprintIncorporationEnabled,
int maxDegreeOfParallelismForIncorporateRequests,
int maxFingerprintsPerIncorporateRequest,
IDomainId domain,
Func <IContentStore> writeThroughContentStoreFunc = null,
bool sealUnbackedContentHashLists = false,
bool useBlobContentHashLists = false,
bool useDedupStore = false,
bool overrideUnixFileAccessMode = false,
bool enableEagerFingerprintIncorporation = false,
TimeSpan inlineFingerprintIncorporationExpiry = default,
TimeSpan eagerFingerprintIncorporationNagleInterval = default,
int eagerFingerprintIncorporationNagleBatchSize = 100,
bool downloadBlobsUsingHttpClient = false)
{
Contract.Requires(fileSystem != null);
Contract.Requires(buildCacheHttpClientFactory != null);
Contract.Requires(backingContentStoreHttpClientFactory != null);
_fileSystem = fileSystem;
_cacheNamespace = cacheNamespace;
_buildCacheHttpClientFactory = buildCacheHttpClientFactory;
_tracer = new BuildCacheCacheTracer(logger, nameof(BuildCacheCache));
_backingContentStore = new BackingContentStore(
new BackingContentStoreConfiguration()
{
FileSystem = fileSystem,
ArtifactHttpClientFactory = backingContentStoreHttpClientFactory,
TimeToKeepContent = timeToKeepUnreferencedContent,
PinInlineThreshold = pinInlineThreshold,
IgnorePinThreshold = ignorePinThreshold,
UseDedupStore = useDedupStore,
DownloadBlobsUsingHttpClient = downloadBlobsUsingHttpClient
});
_manuallyExtendContentLifetime = false;
if (useDedupStore)
{
// Guaranteed content is only available for BlobSessions. (bug 144396)
_sealUnbackedContentHashLists = false;
// BuildCache is incompatible with Dedup hashes.
// This is because BuildCache would not know to look for the blob in DedupStore instead of BlobStore
_useBlobContentHashLists = false;
_manuallyExtendContentLifetime = true;
}
else
{
_sealUnbackedContentHashLists = sealUnbackedContentHashLists;
_useBlobContentHashLists = useBlobContentHashLists;
}
if (!domain.Equals(WellKnownDomainIds.OriginalDomainId))
{
// BuildCache is incompatible with multi-domain
_useBlobContentHashLists = false;
_manuallyExtendContentLifetime = true;
}
_maxFingerprintSelectorsToFetch = maxFingerprintSelectorsToFetch;
_minimumTimeToKeepContentHashLists = minimumTimeToKeepContentHashLists;
_rangeOfTimeToKeepContentHashLists = rangeOfTimeToKeepContentHashLists;
if (writeThroughContentStoreFunc != null)
{
_writeThroughContentStore = writeThroughContentStoreFunc();
Contract.Assert(_writeThroughContentStore != null);
}
_fingerprintIncorporationEnabled = fingerprintIncorporationEnabled;
_maxDegreeOfParallelismForIncorporateRequests = maxDegreeOfParallelismForIncorporateRequests;
_maxFingerprintsPerIncorporateRequest = maxFingerprintsPerIncorporateRequest;
_overrideUnixFileAccessMode = overrideUnixFileAccessMode;
_enableEagerFingerprintIncorporation = enableEagerFingerprintIncorporation;
_inlineFingerprintIncorporationExpiry = inlineFingerprintIncorporationExpiry;
_eagerFingerprintIncorporationNagleInterval = eagerFingerprintIncorporationNagleInterval;
_eagerFingerprintIncorporationNagleBatchSize = eagerFingerprintIncorporationNagleBatchSize;
}
/// <inheritdoc />
public Task <IEnumerable <Request> > GetAllRequestsAsync(CancellationToken cancellationToken, SizeOptions sizeOptions = null, ExpirationDateOptions expirationDateOptions = null, IDomainId domainIdOption = null)
{
return(RetryAsync(
nameof(ISymbolServiceClient.GetAllRequestsAsync),
(client, ct) => client.GetAllRequestsAsync(ct, sizeOptions, expirationDateOptions, domainIdOption),
cancellationToken));
}
public async Task <TAggregate> GetById <TAggregate>(IDomainId id) where TAggregate : class, IAggregate
{
return(await GetById <TAggregate>(id, 0));
}
public FakeAggregateRoot(IDomainId id)
{
Id = id;
}
public FakeStep2Command(IDomainId aggregateId, Guid correlationId, string value1, string who = "anonymous") : base(aggregateId, correlationId, who)
{
Value1 = value1;
}
protected Event(IDomainId aggregateId, string who = "anonymous")
: this(aggregateId, Guid.NewGuid(), who)
{
}
public DifferentFakeAggregateRoot(IDomainId id)
{
Id = id;
}
protected IntegrationEvent(IDomainId aggregateId, Guid correlationId, string who = "anonymous") : base(aggregateId, correlationId, who)
{
}
