private void RunGraphFingerprintValueTest(string[] expectedValues, string commandLineFilter, string[] commandLineValues, bool expectSuccess = true)
{
var pathTable = new PathTable();
// EngineSchedulTest operate on the real filesystem for now
var fileSystem = new PassThroughFileSystem(pathTable);
BuildXLContext context = EngineContext.CreateNew(CancellationToken.None, pathTable, fileSystem);
var config = new CommandLineConfiguration
{
Filter = commandLineFilter,
Startup = { ImplicitFilters = new List <string>(commandLineValues) },
Engine = { DefaultFilter = null }
};
EvaluationFilter evaluationFilter;
if (!expectSuccess)
{
XAssert.IsFalse(EngineSchedule.TryGetEvaluationFilter(BuildXL.TestUtilities.Xunit.XunitBuildXLTest.CreateLoggingContextForTest(), context, config, config, out evaluationFilter));
}
else
{
XAssert.IsTrue(EngineSchedule.TryGetEvaluationFilter(BuildXL.TestUtilities.Xunit.XunitBuildXLTest.CreateLoggingContextForTest(), context, config, config, out evaluationFilter));
foreach (FullSymbol symbol in evaluationFilter.ValueNamesToResolve)
{
string value = symbol.ToString(context.SymbolTable);
XAssert.IsTrue(expectedValues.Contains(value));
}
XAssert.AreEqual(expectedValues.Length, evaluationFilter.ValueNamesToResolve.Count());
}
}
private RootFilter ParseFilter(IEnumerable <string> values, string commandLineFilter, string defaultFilter, out SymbolTable symbolTable, out PathTable pathTable)
{
var loggingContext = new LoggingContext("EngineScheduleTests.ParseFilter");
pathTable = new PathTable();
// EngineSchedulTest operate on the real filesystem for now
var fileSystem = new PassThroughFileSystem(pathTable);
BuildXLContext context = EngineContext.CreateNew(CancellationToken.None, pathTable, fileSystem);
symbolTable = context.SymbolTable;
var config = new CommandLineConfiguration
{
Filter = commandLineFilter,
Startup =
{
ImplicitFilters = new List <string>(values ?? Enumerable.Empty <string>()),
},
Engine =
{
DefaultFilter = defaultFilter,
}
};
RootFilter rootFilter;
XAssert.IsTrue(EngineSchedule.TryGetPipFilter(loggingContext, context, config, config, TryGetPathByMountName,
rootFilter: out rootFilter));
return(rootFilter);
}
public async Task PutFileThenPlaceTestAsync()
{
using var fileSystem = new PassThroughFileSystem(Logger);
using var dir = new DisposableDirectory(fileSystem);
var cacheDir = dir.Path / "cache";
var file = dir.Path / "theFile.txt";
fileSystem.WriteAllText(file, "Foo");
var args = new Dictionary <string, string>
{
["cachePath"] = cacheDir.Path,
["path"] = file.Path,
["hashType"] = "MD5",
["LogSeverity"] = "Diagnostic",
};
var hash = "1356C67D7AD1638D816BFB822DD2C25D";
await RunAppAsync("PutFile", args, Logger);
var destination = dir.Path / "destination.txt";
args["hash"] = hash;
args["path"] = destination.Path;
await RunAppAsync("PlaceFile", args, Logger);
fileSystem.ReadAllText(destination).Should().Be("Foo");
}
private async Task WithContentLocationEventStore(Func <Context, IClock, IAbsFileSystem, ContentLocationEventStore, Task> action, ContentLocationEventStoreConfiguration configuration, IContentLocationEventHandler eventHandler, string localMachineName = "Worker")
{
string centralStateKeyBase = "ThisIsUnused";
var clock = new MemoryClock();
using var fileSystem = new PassThroughFileSystem(TestGlobal.Logger);
var tracingContext = new Context(TestGlobal.Logger);
{
using var localDiskCentralStoreWorkingDirectory = new DisposableDirectory(fileSystem);
var localDiskCentralStoreConfiguration = new LocalDiskCentralStoreConfiguration(localDiskCentralStoreWorkingDirectory.Path, centralStateKeyBase);
var centralStorage = new LocalDiskCentralStorage(localDiskCentralStoreConfiguration);
{
using var eventHubWorkingDirectory = new DisposableDirectory(fileSystem);
var eventStore = CreateEventStore(configuration, eventHandler, localMachineName, centralStorage, eventHubWorkingDirectory, clock);
(await eventStore.StartupAsync(tracingContext)).ShouldBeSuccess();
await action(tracingContext, clock, fileSystem, eventStore);
if (!eventStore.ShutdownStarted)
{
(await eventStore.ShutdownAsync(tracingContext)).ShouldBeSuccess();
}
}
}
}
/// <summary>
/// Initializes a new instance of the <see cref="LocalCache" /> class backed by <see cref="TwoContentStore"/> implemented as <see cref="StreamPathContentStore"/>
/// </summary>
public static LocalCache CreateStreamPathContentStoreInProcMemoizationStoreCache(ILogger logger,
AbsolutePath rootPathForStream,
AbsolutePath rootPathForPath,
MemoizationStoreConfiguration memoConfig,
ConfigurationModel configurationModelForStream = null,
ConfigurationModel configurationModelForPath = null,
IClock clock = null,
bool checkLocalFiles = true,
bool emptyFileHashShortcutEnabled = false)
{
var fileSystem = new PassThroughFileSystem(logger);
clock = clock ?? SystemClock.Instance;
var contentStoreSettings = new ContentStoreSettings()
{
CheckFiles = checkLocalFiles, UseEmptyFileHashShortcut = emptyFileHashShortcutEnabled
};
Func <IContentStore> contentStoreFactory = () => new StreamPathContentStore(
() => new FileSystemContentStore(fileSystem, clock, rootPathForStream, configurationModelForStream, settings: contentStoreSettings),
() => new FileSystemContentStore(fileSystem, clock, rootPathForPath, configurationModelForPath, settings: contentStoreSettings));
var memoizationStoreFactory = CreateInProcessLocalMemoizationStoreFactory(logger, clock, memoConfig);
return(new LocalCache(fileSystem, contentStoreFactory, memoizationStoreFactory, LoadPersistentCacheGuid(rootPathForStream, fileSystem)));
}
public static int ListCheckpoints(
[Required] string storageConnectionString,
[DefaultValue("checkpoints")] string containerName)
{
using var consoleLog = new ConsoleLog();
using var logger = new Logger(new[] { consoleLog });
var loggingContext = new Context(logger);
var context = new OperationContext(loggingContext);
var fileSystem = new PassThroughFileSystem(logger);
try
{
WithCentralStorage(
context,
storageConnectionString,
containerName,
centralStorage => PrintCheckpointsAsync(context, centralStorage))
.Result
.ThrowIfFailure();
return(0);
}
catch (Exception e)
{
Tracer.Error(context, $"Application closing due to exception: {e}");
return(-1);
}
}
public async Task HashBytesOfVariousSizes(int size)
{
using var fileSystem = new PassThroughFileSystem();
using var tempDirectory = new DisposableDirectory(fileSystem);
var content = ThreadSafeRandom.GetBytes(size);
var path = tempDirectory.Path / "1.txt";
fileSystem.WriteAllBytes(path, content);
var hashType = HashType.Vso0;
var contentHasher = HashInfoLookup.GetContentHasher(hashType);
var h1 = contentHasher.GetContentHash(content);
var h2 = CalculateHashWithMemoryMappedFile(fileSystem, path, hashType);
Assert.Equal(h1, h2);
#if NET_COREAPP
h2 = contentHasher.GetContentHash(content.AsSpan());
Assert.Equal(h1, h2);
#endif
using var memoryStream = new MemoryStream(content);
var h3 = await contentHasher.GetContentHashAsync(memoryStream);
Assert.Equal(h1, h3);
// Using an old style hashing to make sure it works the same as the new and optimized version.
using var fileStream = fileSystem.OpenForHashing(path);
var h4 = await contentHasher.GetContentHashAsync(fileStream);
Assert.Equal(h1, h4);
}
// ReSharper disable once UnusedParameter.Local
static int Main(string[] args)
{
int resultCode;
using (var log = new ConsoleLog())
using (var logger = new Logger(log))
{
try
{
using (var fileSystem = new PassThroughFileSystem())
using (var directory = new DisposableDirectory(fileSystem))
using (var store = new FileSystemContentStore(fileSystem, logger, SystemClock.Instance, directory.Path))
{
var context = new Context(logger);
// ReSharper disable once AccessToDisposedClosure
resultCode = TaskSafetyHelpers.SyncResultOnThreadPool(() => RunStore(context, store));
}
}
catch (Exception exception)
{
logger.Error($"Unexpected error: {exception}");
resultCode = 1;
}
}
return(resultCode);
}
internal static BuildXL.Cache.MemoizationStore.Interfaces.Caches.ICache CreateBuildCacheCache <T>(T cacheConfig, ILogger logger, string pat = null) where T : BuildCacheCacheConfig
{
// TODO: Remove check when all clients are updated with unified Dedup flag
if ((ContentHashingUtilities.HashInfo.HashType.IsValidDedup()) ^ cacheConfig.UseDedupStore)
{
var store = cacheConfig.UseDedupStore ? "DedupStore" : "BlobStore";
throw new ArgumentException($"HashType {ContentHashingUtilities.HashInfo.HashType} cannot be used with {store}");
}
var credentialProviderHelper = new CredentialProviderHelper(m => logger.Debug(m));
if (credentialProviderHelper.IsCredentialProviderSpecified())
{
logger.Debug($"Credential providers path specified: '{credentialProviderHelper.CredentialHelperPath}'");
}
else
{
logger.Debug("Using current user's credentials for obtaining AAD token");
}
var credentialsFactory = new VssCredentialsFactory(pat, credentialProviderHelper, m => logger.Debug(m));
logger.Diagnostic("Creating BuildCacheCache factory");
var fileSystem = new PassThroughFileSystem(logger);
// TODO: Once write-behind is implemented send a contentstorefunc down to the create.
Func <IContentStore> writeThroughStore = null;
if (!string.IsNullOrWhiteSpace(cacheConfig.CacheName))
{
ServiceClientRpcConfiguration rpcConfiguration;
if (cacheConfig.GrpcPort != 0)
{
rpcConfiguration = new ServiceClientRpcConfiguration((int)cacheConfig.GrpcPort);
}
else
{
var factory = new MemoryMappedFileGrpcPortSharingFactory(logger, cacheConfig.GrpcPortFileName);
var portReader = factory.GetPortReader();
var port = portReader.ReadPort();
rpcConfiguration = new ServiceClientRpcConfiguration(port);
}
writeThroughStore = () =>
new ServiceClientContentStore(
logger,
fileSystem,
cacheConfig.CacheName,
rpcConfiguration,
cacheConfig.ConnectionRetryIntervalSeconds,
cacheConfig.ConnectionRetryCount,
scenario: cacheConfig.ScenarioName);
}
BuildCacheServiceConfiguration buildCacheServiceConfiguration = cacheConfig.AsBuildCacheServiceConfigurationFile();
return(BuildCacheCacheFactory.Create(fileSystem, logger, credentialsFactory, buildCacheServiceConfiguration, writeThroughStore));
}
public void RollingWithMaxFileCount()
{
foreach (var maxFileCount in new[] { 0, 10 })
{
using (var fileSystem = new PassThroughFileSystem(Logger))
using (var uniqueTempDirectory = new DisposableDirectory(fileSystem))
{
var message = new string('h', 1000);
using (var log = new FileLog(
uniqueTempDirectory.Path.Path,
"test",
maxFileSize: 1000,
maxFileCount: maxFileCount,
dateInFileName: false,
processIdInFileName: false))
{
for (var i = 0; i < 12; i++)
{
log.Write(DateTime.Now, Thread.CurrentThread.ManagedThreadId, Severity.Always, message);
}
}
var actualFileNames = fileSystem.EnumerateFiles(uniqueTempDirectory.Path, EnumerateOptions.None)
.Select(fileInfo => fileInfo.FullPath.FileName).ToHashSet();
if (maxFileCount == 0)
{
actualFileNames.Count.Should().Be(12);
actualFileNames.Contains("test-0.log").Should().BeTrue();
actualFileNames.Contains("test-1.log").Should().BeTrue();
actualFileNames.Contains("test-2.log").Should().BeTrue();
actualFileNames.Contains("test-3.log").Should().BeTrue();
actualFileNames.Contains("test-4.log").Should().BeTrue();
actualFileNames.Contains("test-5.log").Should().BeTrue();
actualFileNames.Contains("test-6.log").Should().BeTrue();
actualFileNames.Contains("test-7.log").Should().BeTrue();
actualFileNames.Contains("test-8.log").Should().BeTrue();
actualFileNames.Contains("test-9.log").Should().BeTrue();
actualFileNames.Contains("test-10.log").Should().BeTrue();
actualFileNames.Contains("test-11.log").Should().BeTrue();
}
else
{
actualFileNames.Count.Should().Be(10);
actualFileNames.Contains("test-02.log").Should().BeTrue();
actualFileNames.Contains("test-03.log").Should().BeTrue();
actualFileNames.Contains("test-04.log").Should().BeTrue();
actualFileNames.Contains("test-05.log").Should().BeTrue();
actualFileNames.Contains("test-06.log").Should().BeTrue();
actualFileNames.Contains("test-07.log").Should().BeTrue();
actualFileNames.Contains("test-08.log").Should().BeTrue();
actualFileNames.Contains("test-09.log").Should().BeTrue();
actualFileNames.Contains("test-10.log").Should().BeTrue();
actualFileNames.Contains("test-11.log").Should().BeTrue();
}
}
}
}
public static AppState TryCreateWorkspace(
TextDocumentManager documentManager,
Uri rootFolder,
EventHandler <WorkspaceProgressEventArgs> progressHandler,
TestContext?testContext = null,
DScriptSettings settings = null)
{
documentManager = documentManager ?? new TextDocumentManager(new PathTable());
// Need to clear event handlers to detach old instances like IncrementalWorkspaceProvider from the new events.
documentManager.ClearEventHandlers();
var pathTable = documentManager.PathTable;
// Check if we need to prepopulate the document manager
// with some documents (for testing)
if (testContext.HasValue)
{
foreach (var document in testContext.Value.PrePopulatedDocuments)
{
documentManager.Add(AbsolutePath.Create(pathTable, document.Uri), document);
}
}
// Bootstrap the DScript frontend to parse/type-check the workspace given a build config
var loggingContext = new LoggingContext("LanguageServer");
var fileSystem = new PassThroughFileSystem(pathTable);
var engineContext = EngineContext.CreateNew(
cancellationToken: System.Threading.CancellationToken.None,
pathTable: pathTable,
fileSystem: fileSystem);
var engineAbstraction = new LanguageServiceEngineAbstraction(documentManager, pathTable, engineContext.FileSystem);
var frontEndContext = engineContext.ToFrontEndContext(loggingContext);
var rootFolderAsAbsolutePath = rootFolder.ToAbsolutePath(pathTable);
if (!WorkspaceBuilder.TryBuildWorkspaceForIde(
frontEndContext: frontEndContext,
engineContext: engineContext,
frontEndEngineAbstraction: engineAbstraction,
rootFolder: rootFolderAsAbsolutePath,
progressHandler: progressHandler,
workspace: out var workspace,
skipNuget: settings?.SkipNuget ?? false, // Do not skip nuget by default.
controller: out var controller))
{
// The workspace builder fails when the workspace contains any errors
// however, we bail out only if the workspace is null, which happens only
// when the config could not be parsed
if (workspace == null || controller == null)
{
return(null);
}
}
var incrementalWorkspaceProvider = new IncrementalWorkspaceProvider(controller, pathTable, workspace, documentManager, testContext);
return(new AppState(engineAbstraction, documentManager, pathTable, incrementalWorkspaceProvider, workspace));
}
public VirtualizedContentSession(VirtualizedContentStore store, IContentSession session, VfsContentManager contentManager, string name)
: base(name)
{
_store = store;
_innerSession = session;
_contentManager = contentManager;
_fileSystem = new PassThroughFileSystem();
}
/// <summary>
/// Both content and metadata are entirely backed by an out-of-proc cache.
/// </summary>
public static ICache CreateRpcCache(
ILogger logger,
ServiceClientContentStoreConfiguration serviceClientCacheConfiguration)
{
var fileSystem = new PassThroughFileSystem(logger);
return(new ServiceClientCache(logger, fileSystem, serviceClientCacheConfiguration));
}
public VfsContentManager(ILogger logger, VfsCasConfiguration configuration, VfsTree tree, IContentSession contentSession)
{
_logger = logger;
_configuration = configuration;
Tree = tree;
_contentSession = contentSession;
_fileSystem = new PassThroughFileSystem();
_tempDirectory = new DisposableDirectory(_fileSystem, configuration.DataRootPath / "temp");
}
public CacheServerFactory(DistributedCacheServiceArguments arguments)
{
_arguments = arguments;
_logger = arguments.Logger;
// Enable POSIX delete to ensure that files are removed even when there are open handles
PassThroughFileSystem.EnablePosixDelete();
_fileSystem = new PassThroughFileSystem(_logger);
}
private void Init(ILogger logger, IClock clock, LocalRedisFixture redisFixture)
{
_fileSystem = new PassThroughFileSystem(logger);
_logger = logger;
_tempDirectory = new DisposableDirectory(_fileSystem, "RedisTests");
_clock = clock;
_redisFixture = redisFixture;
_disposed = false;
}
/// <nodoc />
public VfsContentManager(ILogger logger, VfsCasConfiguration configuration, IReadOnlyContentSession placer)
{
Tree = new VfsTree(configuration);
_logger = logger;
_configuration = configuration;
_placer = placer;
_fileSystem = new PassThroughFileSystem();
_tempDirectory = new DisposableDirectory(_fileSystem, configuration.DataRootPath / "temp");
Provider = new VfsProvider(logger, configuration, this);
}
[InlineData(2 * 1024 * 1024 + 1)] // File size is 1024 * 1024 - max chunk is 1024K * 2
public async Task CanGet1024kChunkHashFromFile(int fileLength)
{
using (var fileSystem = new PassThroughFileSystem())
using (var disposableDir = new DisposableDirectory(fileSystem))
{
var fileName = disposableDir.CreateRandomFileName();
fileSystem.WriteAllBytes(fileName, new byte[fileLength]);
var node = await DedupContentSession.GetDedupNodeFromFileAsync(HashType.Dedup1024K, fileName.Path);
node.HashString.Should().NotBeNullOrEmpty();
}
}
private void Init(ILogger logger, IClock clock, LocalRedisFixture redisFixture)
{
_fileSystem = new PassThroughFileSystem(logger);
_logger = logger;
_tempDirectory = new DisposableDirectory(_fileSystem, "RedisTests");
_clock = clock;
_redisFixture = redisFixture;
_disposed = false;
// The instance is re-initialized, so we need to re-register it for finalization to detect resource leaks.
GC.ReRegisterForFinalize(this);
}
/// <summary>
/// Content Stores:
/// - <see cref="ServiceClientContentStore"/>
/// Memoization Stores:
/// - <see cref="CreateInProcessLocalMemoizationStoreFactory(ILogger, IClock, MemoizationStoreConfiguration)"/>
/// </summary>
public static LocalCache CreateRpcContentStoreInProcMemoizationStoreCache(ILogger logger,
AbsolutePath rootPath,
ServiceClientContentStoreConfiguration serviceClientContentStoreConfiguration,
MemoizationStoreConfiguration memoizationStoreConfiguration,
IClock clock = null)
{
var fileSystem = new PassThroughFileSystem(logger);
clock = clock ?? SystemClock.Instance;
Func <IContentStore> remoteContentStoreFactory = () => new ServiceClientContentStore(logger, fileSystem, serviceClientContentStoreConfiguration);
var localMemoizationStoreFactory = CreateInProcessLocalMemoizationStoreFactory(logger, clock, memoizationStoreConfiguration);
return(new LocalCache(fileSystem, remoteContentStoreFactory, localMemoizationStoreFactory, LoadPersistentCacheGuid(rootPath, fileSystem)));
}
public async Task RunAsync(VfsCasConfiguration configuration)
{
// Create VFS root
using (var fileLog = new FileLog((configuration.RootPath / "Bvfs.log").Path))
using (var logger = new Logger(fileLog))
{
var fileSystem = new PassThroughFileSystem(logger);
var context = new OperationContext(new Context(logger));
// Map junctions into VFS root
foreach (var mount in configuration.VirtualizationMounts)
{
CreateJunction(context, source: mount.Value, target: configuration.VfsMountRootPath / mount.Key);
}
var clientContentStore = new ServiceClientContentStore(
logger,
fileSystem,
new ServiceClientContentStoreConfiguration(
configuration.CacheName,
new ServiceClientRpcConfiguration(
configuration.BackingGrpcPort),
scenario: configuration.Scenario));
// Client is startup/shutdown with wrapping VFS content store
using (var server = new LocalContentServer(
fileSystem,
logger,
scenario: "bvfs" + configuration.ServerGrpcPort,
path => new VirtualizedContentStore(clientContentStore, logger, configuration),
new LocalServerConfiguration(
configuration.DataRootPath,
new Dictionary <string, AbsolutePath>()
{
{ configuration.CacheName, configuration.ServerRootPath }
},
configuration.ServerGrpcPort,
fileSystem)))
{
await server.StartupAsync(context).ThrowIfFailure();
await WaitForTerminationAsync(context);
await server.ShutdownAsync(context).ThrowIfFailure();
}
}
}
/// <summary>
/// Content Stores:
/// - <see cref="ServiceClientContentStore"/> if <see cref="LocalCacheConfiguration.EnableContentServer"/>
/// - <see cref="FileSystemContentStore"/> otherwise
/// Memoization Stores:
/// - <see cref="CreateInProcessLocalMemoizationStoreFactory(ILogger, IClock, MemoizationStoreConfiguration)"/>
/// </summary>
public static LocalCache CreateUnknownContentStoreInProcMemoizationStoreCache(
ILogger logger,
AbsolutePath rootPath,
MemoizationStoreConfiguration memoizationStoreConfiguration,
LocalCacheConfiguration localCacheConfiguration,
ConfigurationModel configurationModel = null,
IClock clock = null,
bool checkLocalFiles = true,
bool emptyFileHashShortcutEnabled = false)
{
clock = clock ?? SystemClock.Instance;
var fileSystem = new PassThroughFileSystem(logger);
var contentStoreSettings = new ContentStoreSettings()
{
CheckFiles = checkLocalFiles,
UseEmptyFileHashShortcut = emptyFileHashShortcutEnabled
};
Func <IContentStore> contentStoreFactory = () =>
{
if (localCacheConfiguration.EnableContentServer)
{
return(new ServiceClientContentStore(
logger,
fileSystem,
localCacheConfiguration.CacheName,
new ServiceClientRpcConfiguration(localCacheConfiguration.GrpcPort),
(uint)localCacheConfiguration.RetryIntervalSeconds,
(uint)localCacheConfiguration.RetryCount,
scenario: localCacheConfiguration.ScenarioName));
}
else
{
return(new FileSystemContentStore(
fileSystem,
clock,
rootPath,
configurationModel: configurationModel,
settings: contentStoreSettings));
}
};
var memoizationStoreFactory = CreateInProcessLocalMemoizationStoreFactory(logger, clock, memoizationStoreConfiguration);
return(new LocalCache(fileSystem, contentStoreFactory, memoizationStoreFactory, LoadPersistentCacheGuid(rootPath, fileSystem)));
}
public Task WithConfiguration(Func <AzureBlobStorageLogConfiguration, OperationContext, IClock, IAbsFileSystem, ITelemetryFieldsProvider, AzureBlobStorageCredentials, Task> action)
{
var fileSystem = new PassThroughFileSystem();
using var workspace = new DisposableDirectory(fileSystem);
// See: https://docs.microsoft.com/en-us/azure/storage/common/storage-use-emulator#connect-to-the-emulator-account-using-a-shortcut
var credentials = new AzureBlobStorageCredentials(connectionString: "UseDevelopmentStorage=true");
var tracingContext = new Context(Logger);
var context = new OperationContext(tracingContext);
var configuration = new AzureBlobStorageLogConfiguration(workspace.Path);
var clock = SystemClock.Instance;
var telemetryFieldsProvider = new MockTelemetryFieldsProvider();
return(action(configuration, context, clock, fileSystem, telemetryFieldsProvider, credentials));
}
/// <summary>
/// Content Stores:
/// - <see cref="ServiceClientContentStore"/> if <see cref="LocalCacheConfiguration.ServiceClientContentStoreConfiguration"/>
/// - <see cref="FileSystemContentStore"/> otherwise
/// Memoization Stores:
/// - <see cref="CreateInProcessLocalMemoizationStoreFactory(ILogger, IClock, MemoizationStoreConfiguration)"/>
/// </summary>
public static LocalCache CreateUnknownContentStoreInProcMemoizationStoreCache(
ILogger logger,
AbsolutePath rootPath,
MemoizationStoreConfiguration memoizationStoreConfiguration,
LocalCacheConfiguration localCacheConfiguration,
ConfigurationModel configurationModel = null,
IClock clock = null,
bool checkLocalFiles = true)
{
clock = clock ?? SystemClock.Instance;
var fileSystem = new PassThroughFileSystem(logger);
var contentStoreSettings = new ContentStoreSettings()
{
CheckFiles = checkLocalFiles,
};
Func <IContentStore> contentStoreFactory = () =>
{
if (localCacheConfiguration.ServiceClientContentStoreConfiguration != null)
{
return(new ServiceClientContentStore(logger, fileSystem, localCacheConfiguration.ServiceClientContentStoreConfiguration));
}
else
{
return(new FileSystemContentStore(
fileSystem,
clock,
rootPath,
configurationModel: configurationModel,
settings: contentStoreSettings));
}
};
var memoizationStoreFactory = CreateInProcessLocalMemoizationStoreFactory(logger, clock, memoizationStoreConfiguration);
return(new LocalCache(fileSystem, contentStoreFactory, memoizationStoreFactory, LoadPersistentCacheGuid(rootPath, fileSystem)));
}
public static int DownloadCheckpoint(
[Required] string outputPath,
[Required] string storageConnectionString,
[DefaultValue(null)] string?checkpointId,
[DefaultValue("checkpoints")] string containerName,
[DefaultValue(8)] int downloadParallelism,
[DefaultValue(false)] bool overwrite
)
{
using var consoleLog = new ConsoleLog();
using var logger = new Logger(new[] { consoleLog });
var loggingContext = new Context(logger);
var context = new OperationContext(loggingContext);
var fileSystem = new PassThroughFileSystem(logger);
try
{
var outputDirectoryPath = new AbsolutePath(outputPath);
fileSystem.CreateDirectory(outputDirectoryPath);
WithCentralStorage(
context,
storageConnectionString,
containerName,
centralStorage => DownloadNewestMatchingCheckpointAsync(context, fileSystem, centralStorage, outputDirectoryPath, downloadParallelism, checkpointId, overwrite))
.Result
.ThrowIfFailure();
return(0);
}
catch (Exception e)
{
Tracer.Error(context, $"Application closing due to exception: {e}");
return(-1);
}
}
/// <summary>
/// Generates pip graph fragment.
/// </summary>
public static bool TryGeneratePipGraphFragment(
PathTable pathTable,
ICommandLineConfiguration commandLineConfig,
PipGraphFragmentGeneratorConfiguration pipGraphFragmentConfig)
{
var loggingContext = new LoggingContext(nameof(PipGraphFragmentGenerator));
var fileSystem = new PassThroughFileSystem(pathTable);
var engineContext = EngineContext.CreateNew(CancellationToken.None, pathTable, fileSystem);
FrontEndContext context = engineContext.ToFrontEndContext(loggingContext);
// Parse filter string.
var evaluationFilter = EvaluationFilter.Empty;
if (!string.IsNullOrWhiteSpace(commandLineConfig.Filter))
{
if (!TryGetEvaluationFilter(loggingContext, engineContext, commandLineConfig.Filter, out evaluationFilter))
{
// Error should have been been reported already.
return(false);
}
}
if (!TryBuildPipGraphFragment(
commandLineConfig,
pipGraphFragmentConfig,
context,
engineContext,
evaluationFilter))
{
return(false);
}
return(true);
}
public LifetimeTrackerTests(ITestOutputHelper output)
: base(output)
{
_fileSystem = new PassThroughFileSystem(TestGlobal.Logger);
}
/// <summary>
/// Builds a workspace and uses filter to find specs to evaluate.
/// </summary>
public static bool TryBuildWorkspaceAndCollectFilesToAnalyze(
Tracing.Logger logger,
PathTable pathTable,
ICommandLineConfiguration configuation,
out Workspace workspace,
out IPipGraph pipGraph,
out IReadOnlyDictionary <AbsolutePath, ISourceFile> filesToAnalyze,
out FrontEndContext context)
{
workspace = null;
pipGraph = null;
filesToAnalyze = null;
var loggingContext = new LoggingContext("DScriptAnalyzer");
var fileSystem = new PassThroughFileSystem(pathTable);
var engineContext = EngineContext.CreateNew(CancellationToken.None, pathTable, fileSystem);
context = engineContext.ToFrontEndContext(loggingContext);
// Parse filter string into EvaluationFilter
var evaluationFilter = EvaluationFilter.Empty;
if (!string.IsNullOrEmpty(configuation.Filter))
{
if (!TryGetEvaluationFilter(logger, loggingContext, engineContext, configuation.Filter, out evaluationFilter))
{
// Error has been reported already
return(false);
}
}
// Try parsing the workspace from config and evaluation filter
if (!TryBuildWorkspace(
configuation,
context,
engineContext,
evaluationFilter,
progressHandler: null,
workspace: out workspace,
frontEndHostController: out _,
pipGraph: out pipGraph,
configuration: GetDefaultConfiguration()))
{
return(false);
}
if (configuation.Engine.Phase == EnginePhases.AnalyzeWorkspace)
{
// Find strict subset of specs in workspace that should be analyzed
var collectedFilesToAnalyze = CollectFilesToAnalyze(
workspace,
pathTable,
configuation.Startup.ConfigFile,
evaluationFilter);
if (collectedFilesToAnalyze.Count == 0)
{
logger.ErrorFilterHasNoMatchingSpecs(loggingContext, configuation.Filter);
return(false);
}
filesToAnalyze = collectedFilesToAnalyze;
}
else
{
filesToAnalyze = new Dictionary <AbsolutePath, ISourceFile>();
}
return(true);
}
public TestEnv(
string name,
string rootPath,
bool enableLazyOutputMaterialization = false,
int maxRelativeOutputDirectoryLength = 260,
List <IMount> mounts = null,
PathTable pathTable = null)
{
Contract.Requires(name != null);
Contract.Requires(!string.IsNullOrEmpty(rootPath));
LoggingContext = new LoggingContext("TestLogger." + name);
PathTable = pathTable ?? new PathTable();
PipDataBuilderPool = new ObjectPool <PipDataBuilder>(() => new PipDataBuilder(PathTable.StringTable), _ => { });
// The tests that use TestEnv need to be modernized to take a filesystem
var fileSystem = new PassThroughFileSystem(PathTable);
Context = EngineContext.CreateNew(CancellationToken.None, PathTable, fileSystem);
// Add some well-known paths with fixed casing to the Context.PathTable
AbsolutePath.Create(Context.PathTable, rootPath.ToLowerInvariant());
var root = AbsolutePath.Create(Context.PathTable, rootPath);
var configuration = ConfigHelpers.CreateDefaultForXml(Context.PathTable, root);
configuration.Layout.SourceDirectory = root.Combine(PathTable, PathAtom.Create(PathTable.StringTable, "src")); // These tests have non-standard src folder
configuration.Engine.MaxRelativeOutputDirectoryLength = maxRelativeOutputDirectoryLength;
configuration.Schedule.EnableLazyOutputMaterialization = enableLazyOutputMaterialization;
configuration.Schedule.UnsafeDisableGraphPostValidation = false;
configuration.Schedule.ComputePipStaticFingerprints = true;
configuration.Sandbox.FileAccessIgnoreCodeCoverage = true;
BuildXLEngine.PopulateFileSystemCapabilities(configuration, configuration, Context.PathTable, LoggingContext);
BuildXLEngine.PopulateLoggingAndLayoutConfiguration(configuration, Context.PathTable, bxlExeLocation: null, inTestMode: true);
BuildXLEngine.PopulateAndValidateConfiguration(configuration, configuration, Context.PathTable, LoggingContext);
Configuration = configuration;
var mountsTable = MountsTable.CreateAndRegister(LoggingContext, Context, Configuration, null);
if (mounts != null)
{
foreach (var mount in mounts)
{
mountsTable.AddResolvedMount(mount);
}
}
AbsolutePath specFile = SourceRoot.CreateRelative(Context.PathTable, "TestSpecFile.dsc");
var graph = TestSchedulerFactory.CreateEmptyPipGraph(Context, configuration, mountsTable.MountPathExpander);
PipTable = graph.PipTable;
PipGraph = graph;
var locationData = new LocationData(specFile, 0, 0);
var modulePip = ModulePip.CreateForTesting(Context.StringTable, specFile);
PipGraph.AddModule(modulePip);
PipGraph.AddSpecFile(new SpecFilePip(FileArtifact.CreateSourceFile(specFile), locationData, modulePip.Module));
PipConstructionHelper = PipConstructionHelper.CreateForTesting(
Context,
ObjectRoot,
redirectedRoot: Configuration.Layout.RedirectedDirectory,
pipGraph: PipGraph,
moduleName: modulePip.Identity.ToString(Context.StringTable),
symbol: name,
specPath: specFile);
Paths = new Paths(PathTable);
mountsTable.CompleteInitialization();
}
internal static BuildXL.Cache.MemoizationStore.Interfaces.Caches.ICache CreateBuildCacheCache <T>(T cacheConfig, ILogger logger, string pat = null) where T : BuildCacheCacheConfig
{
// TODO: Remove check when all clients are updated with unified Dedup flag
if (ContentHashingUtilities.HashInfo.HashType == HashType.DedupNodeOrChunk ^ cacheConfig.UseDedupStore)
{
var store = cacheConfig.UseDedupStore ? "DedupStore" : "BlobStore";
throw new ArgumentException($"HashType {ContentHashingUtilities.HashInfo.HashType} cannot be used with {store}");
}
string credentialProviderPath = Environment.GetEnvironmentVariable(CredentialProvidersPathEnvVariable);
if (!string.IsNullOrWhiteSpace(credentialProviderPath))
{
logger.Debug($"Credential providers path specified: {credentialProviderPath}");
}
else
{
logger.Debug("Using current user's credentials for obtaining AAD token");
}
VssCredentialsFactory credentialsFactory;
#if !PLATFORM_OSX
string userName = null; // when running on .NET Framework, user name doesn't have to be explicitly provided
#if FEATURE_CORECLR
userName = GetAadUserNameUpn();
#endif
credentialsFactory = new VssCredentialsFactory(new VsoCredentialHelper(s => logger.Debug(s)), userName);
#else
var secPat = new SecureString();
if (!string.IsNullOrWhiteSpace(pat))
{
foreach (char c in pat)
{
secPat.AppendChar(c);
}
}
else
{
throw new ArgumentException("PAT must be supplied when running with CoreCLR");
}
credentialsFactory = new VssCredentialsFactory(new VssBasicCredential(new NetworkCredential(string.Empty, secPat)));
#endif
logger.Diagnostic("Creating BuildCacheCache factory");
var fileSystem = new PassThroughFileSystem(logger);
// TODO: Once write-behind is implemented send a contentstorefunc down to the create.
Func <IContentStore> writeThroughStore = null;
if (!string.IsNullOrWhiteSpace(cacheConfig.CacheName))
{
ServiceClientRpcConfiguration rpcConfiguration;
if (cacheConfig.GrpcPort != 0)
{
rpcConfiguration = new ServiceClientRpcConfiguration((int)cacheConfig.GrpcPort);
}
else
{
var factory = new MemoryMappedFileGrpcPortSharingFactory(logger, cacheConfig.GrpcPortFileName);
var portReader = factory.GetPortReader();
var port = portReader.ReadPort();
rpcConfiguration = new ServiceClientRpcConfiguration(port);
}
writeThroughStore = () =>
new ServiceClientContentStore(
logger,
fileSystem,
cacheConfig.CacheName,
rpcConfiguration,
cacheConfig.ConnectionRetryIntervalSeconds,
cacheConfig.ConnectionRetryCount,
scenario: cacheConfig.ScenarioName);
}
BuildCacheServiceConfiguration buildCacheServiceConfiguration = cacheConfig.AsBuildCacheServiceConfigurationFile();
return(BuildCacheCacheFactory.Create(fileSystem, logger, credentialsFactory, buildCacheServiceConfiguration, writeThroughStore));
}
