/// <nodoc />
public ApiServer(
IIpcProvider ipcProvider,
string ipcMonikerId,
FileContentManager fileContentManager,
PipExecutionContext context,
IServerConfig config,
EngineCache engineCache,
Tracing.IExecutionLogTarget executionLog,
Tracing.BuildManifestGenerator buildManifestGenerator)
{
Contract.Requires(ipcMonikerId != null);
Contract.Requires(fileContentManager != null);
Contract.Requires(context != null);
Contract.Requires(config != null);
Contract.Requires(engineCache != null);
Contract.Requires(executionLog != null);
m_fileContentManager = fileContentManager;
m_server = ipcProvider.GetServer(ipcProvider.LoadAndRenderMoniker(ipcMonikerId), config);
m_context = context;
m_engineCache = engineCache;
m_executionLog = executionLog;
m_buildManifestGenerator = buildManifestGenerator;
m_inMemoryBuildManifestStore = new ConcurrentDictionary <ContentHash, ContentHash>();
}
/// <nodoc/>
public MaterializationDaemon(
IParser parser,
DaemonConfig daemonConfig,
MaterializationDaemonConfig materializationConfig,
IIpcProvider rpcProvider = null,
Client bxlClient = null)
: base(parser,
daemonConfig,
!string.IsNullOrWhiteSpace(materializationConfig.LogDir) ? new FileLogger(materializationConfig.LogDir, LogFileName, daemonConfig.Moniker, logVerbose: true, MaterializationDaemonLogPrefix) : daemonConfig.Logger,
rpcProvider,
bxlClient)
{
m_config = materializationConfig;
m_actionQueue = new ActionQueue(m_config.MaxDegreeOfParallelism);
m_materializationStatus = new ConcurrentBigMap <string, bool>();
m_counters = new CounterCollection <MaterializationDaemonCounter>();
m_macros = new Dictionary <string, string>
{
["$(build.nttree)"] = Environment.GetEnvironmentVariable("_NTTREE")
};
m_logger.Info($"MaterializationDaemon config: {JsonConvert.SerializeObject(m_config)}");
m_logger.Info($"Defined macros (count={m_macros.Count}):{Environment.NewLine}{string.Join(Environment.NewLine, m_macros.Select(kvp => $"{kvp.Key}={kvp.Value}"))}");
}
public void TestWithExecutionError(IIpcProvider provider)
{
var testName = nameof(TestWithExecutionError);
WithIpcServer(
provider,
CrashingExecutor,
ServerConfigWithLogger(testName),
(moniker, server) =>
{
using (var client = provider.GetClient(provider.RenderConnectionString(moniker), ClientConfigWithLogger(testName)))
{
var syncOp = new IpcOperation("sync", waitForServerAck: true);
var asyncOp = new IpcOperation("async", waitForServerAck: false);
var syncResult = SendWithTimeout(client, syncOp);
var asyncResult = SendWithTimeout(client, asyncOp);
Assert.True(asyncResult.Succeeded, "Asynchronous operation is expected to succeed if executor crashes");
Assert.False(syncResult.Succeeded, "Synchronous operation is expected to fail if executor crashes");
Assert.Equal(IpcResultStatus.ExecutionError, syncResult.ExitCode);
Assert.True(syncResult.Payload.Contains("System.Exception")); // because CrashingExecutor throws System.Exception
Assert.True(syncResult.Payload.Contains(syncOp.Payload)); // because CrashingExecutor throws System.Exception whose message is equal to syncOp.Payload
client.RequestStop();
client.Completion.GetAwaiter().GetResult();
}
});
}
/// <nodoc />
public ApiServer(
IIpcProvider ipcProvider,
string ipcMonikerId,
FileContentManager fileContentManager,
PipExecutionContext context,
IServerConfig config,
PipTwoPhaseCache pipTwoPhaseCache,
Tracing.IExecutionLogTarget executionLog,
Tracing.BuildManifestGenerator buildManifestGenerator,
ServiceManager serviceManger,
bool verifyFileContentOnBuildManifestHashComputation)
{
Contract.Requires(ipcMonikerId != null);
Contract.Requires(fileContentManager != null);
Contract.Requires(context != null);
Contract.Requires(config != null);
Contract.Requires(pipTwoPhaseCache != null);
Contract.Requires(executionLog != null);
m_fileContentManager = fileContentManager;
m_server = ipcProvider.GetServer(ipcProvider.LoadAndRenderMoniker(ipcMonikerId), config);
m_context = context;
m_executionLog = executionLog;
m_buildManifestGenerator = buildManifestGenerator;
m_serviceManger = serviceManger;
m_pipTwoPhaseCache = pipTwoPhaseCache;
m_inMemoryBuildManifestStore = new ConcurrentBigMap <ContentHash, IReadOnlyList <ContentHash> >();
m_receivedStatistics = new ConcurrentBigMap <string, long>();
m_verifyFileContentOnBuildManifestHashComputation = verifyFileContentOnBuildManifestHashComputation;
}
public void TestConcurrentSynchronousOperations(IIpcProvider provider)
{
var testName = nameof(TestConcurrentSynchronousOperations);
WithIpcServer(
provider,
EchoingExecutor,
ServerConfigWithLogger(testName),
(moniker, server) =>
{
using (IClient client = provider.GetClient(provider.RenderConnectionString(moniker), ClientConfigWithLogger(testName)))
{
var threads = Enumerable
.Range(1, 10)
.Select(i => new Thread(() =>
{
var message = "hi" + i;
var op = new IpcOperation(message, waitForServerAck: true);
var result = SendWithTimeout(client, op);
Assert.True(result.Succeeded, "error: " + result.Payload);
Assert.Equal(op.Payload, result.Payload);
}))
.ToArray();
Start(threads);
Join(threads);
client.RequestStop();
client.Completion.GetAwaiter().GetResult();
}
});
}
private WeakContentFingerprint GenerateSaltedWeakFingerprint(ContentHash hash) => new WeakContentFingerprint(FingerprintUtilities.Hash($"Hash: '{hash.ToHex()}' Salt: '{m_buildManifestHashCacheSalt}'")); // Changes to this string will invalidate all existing cache entries
/// <nodoc />
public ApiServer(
IIpcProvider ipcProvider,
string ipcMonikerId,
FileContentManager fileContentManager,
PipExecutionContext context,
IServerConfig config,
EngineCache engineCache,
Tracing.IExecutionLogTarget executionLog,
Tracing.BuildManifestGenerator buildManifestGenerator)
{
Contract.Requires(ipcMonikerId != null);
Contract.Requires(fileContentManager != null);
Contract.Requires(context != null);
Contract.Requires(config != null);
Contract.Requires(engineCache != null);
Contract.Requires(executionLog != null);
m_fileContentManager = fileContentManager;
m_server = ipcProvider.GetServer(ipcProvider.LoadAndRenderMoniker(ipcMonikerId), config);
m_context = context;
m_engineCache = engineCache;
m_executionLog = executionLog;
m_buildManifestGenerator = buildManifestGenerator;
m_inMemoryBuildManifestStore = new ConcurrentBigMap <ContentHash, ContentHash>();
m_buildManifestHashCacheSalt = string.IsNullOrEmpty(Utilities.Configuration.EngineEnvironmentSettings.BuildManifestHashCacheSalt)
? string.Empty
: Utilities.Configuration.EngineEnvironmentSettings.BuildManifestHashCacheSalt;
}
public void TestCreateDisposeClient(IIpcProvider provider)
{
var m1 = provider.CreateNewConnectionString();
using (provider.GetClient(m1, new ClientConfig()))
{
}
}
public void TestCreateDisposeServer(IIpcProvider provider)
{
var m1 = provider.CreateNewConnectionString();
using (var server = provider.GetServer(m1, new ServerConfig()))
{
}
}
public void TestCreateNewMonikerReturnsUniqueMonikers(IIpcProvider provider)
{
var m1 = provider.CreateNewMoniker();
var m2 = provider.CreateNewMoniker();
Assert.NotNull(m1);
Assert.NotNull(m2);
Assert.NotEqual(m1, m2);
}
public void TestServerConsideredNotCompletedBeforeStarted(IIpcProvider provider)
{
var m1 = provider.CreateNewConnectionString();
using (var server = provider.GetServer(m1, new ServerConfig()))
{
Assert.False(server.Completion.IsCompleted);
}
}
public void TestRenderConnectionStringReturnsSameString(IIpcProvider provider)
{
var m1 = provider.CreateNewMoniker();
var connStr1 = provider.RenderConnectionString(m1);
var connStr2 = provider.RenderConnectionString(m1);
Assert.NotNull(connStr1);
Assert.NotNull(connStr2);
Assert.Equal(connStr1, connStr2);
}
private IEnumerable <Thread> GetClientThreads(IIpcProvider ipcProvider, IEnumerable <string> ipcMonikers, int numServices, int numRequests, string cmdLine)
{
return(ipcMonikers
.SelectMany(moniker =>
Enumerable
.Range(1, numRequests)
.Select(i => CreateThreadForCommand(cmdLine.Replace("<moniker>", moniker), ipcProvider.GetClient(moniker, new ClientConfig())))
.ToList())
.ToList());
}
public void TestServerMultipleDispose(IIpcProvider provider)
{
var m1 = provider.CreateNewConnectionString();
using (var server = provider.GetServer(m1, new ServerConfig()))
{
server.Start(EchoingExecutor);
server.RequestStop();
WaitServerDone(server);
server.Dispose();
}
}
public void TestCreateStartStopDisposeServer(IIpcProvider provider)
{
var m1 = provider.CreateNewConnectionString();
using (var server = provider.GetServer(m1, new ServerConfig()))
{
server.Start(EchoingExecutor);
Assert.False(server.Completion.IsCompleted);
server.RequestStop();
WaitServerDone(server);
}
}
/// <summary>
/// 1. creates a server
/// 2. starts the server
/// 3. invokes 'testAction'
/// 4. shuts down the server
/// 5. waits for the server to complete.
/// </summary>
protected static void WithIpcServer(IIpcProvider provider, IIpcOperationExecutor executor, IServerConfig config, Action <IIpcMoniker, IServer> testAction)
{
WithIpcServer(
provider,
executor,
config,
(moniker, server) =>
{
testAction(moniker, server);
return(Task.FromResult(1));
}).GetAwaiter().GetResult();
}
public TestScheduler(
PipGraph graph,
TestPipQueue pipQueue,
PipExecutionContext context,
FileContentTable fileContentTable,
EngineCache cache,
IConfiguration configuration,
FileAccessWhitelist fileAccessWhitelist,
DirectoryMembershipFingerprinterRuleSet directoryMembershipFingerprinterRules = null,
ITempCleaner tempCleaner = null,
PipRuntimeTimeTable runningTimeTable = null,
JournalState journalState = null,
PerformanceCollector performanceCollector = null,
string fingerprintSalt = null,
PreserveOutputsInfo?previousInputsSalt = null,
IEnumerable <Pip> successfulPips = null,
IEnumerable <Pip> failedPips = null,
LoggingContext loggingContext = null,
IIpcProvider ipcProvider = null,
DirectoryTranslator directoryTranslator = null,
VmInitializer vmInitializer = null,
SchedulerTestHooks testHooks = null) : base(graph, pipQueue, context, fileContentTable, cache,
configuration, fileAccessWhitelist, loggingContext, null, directoryMembershipFingerprinterRules,
tempCleaner, AsyncLazy <PipRuntimeTimeTable> .FromResult(runningTimeTable), performanceCollector, fingerprintSalt, previousInputsSalt,
ipcProvider: ipcProvider,
directoryTranslator: directoryTranslator,
journalState: journalState,
vmInitializer: vmInitializer,
testHooks: testHooks)
{
m_testPipQueue = pipQueue;
if (successfulPips != null)
{
foreach (var pip in successfulPips)
{
Contract.Assume(pip.PipId.IsValid, "Override results must be added after the pip has been added to the scheduler");
m_overridePipResults.Add(pip.PipId, PipResultStatus.Succeeded);
}
}
if (failedPips != null)
{
foreach (var pip in failedPips)
{
Contract.Assume(pip.PipId.IsValid, "Override results must be added after the pip has been added to the scheduler");
m_overridePipResults.Add(pip.PipId, PipResultStatus.Failed);
}
}
m_loggingContext = loggingContext;
}
/// <summary>
/// Creates an execution environment for a single pip. To run pips incrementally, the <paramref name="fileContentTable"/> and <paramref name="pipCache"/> should be specified.
/// </summary>
public DummyPipExecutionEnvironment(
LoggingContext loggingContext,
PipExecutionContext context,
IConfiguration config,
FileContentTable fileContentTable = null,
EngineCache pipCache = null,
SemanticPathExpander semanticPathExpander = null,
PipContentFingerprinter.PipDataLookup pipDataLookup = null,
FileAccessWhitelist fileAccessWhitelist = null,
bool allowUnspecifiedSealedDirectories = false,
PipTable pipTable = null,
IIpcProvider ipcProvider = null,
(string substSource, string substTarget)?subst = default,
/// <nodoc/>
public MaterializationDaemon(
IParser parser,
DaemonConfig daemonConfig,
MaterializationDaemonConfig materializationConfig,
IIpcProvider rpcProvider = null,
Client bxlClient = null)
: base(parser,
daemonConfig,
!string.IsNullOrWhiteSpace(materializationConfig.LogDir) ? new FileLogger(materializationConfig.LogDir, LogFileName, daemonConfig.Moniker, logVerbose: true, MaterializationDaemonLogPrefix) : daemonConfig.Logger,
rpcProvider,
bxlClient)
{
}
public void TestServerMultipleStartFails(IIpcProvider provider)
{
var testName = nameof(TestServerMultipleStartFails);
var m1 = provider.CreateNewConnectionString();
using (var server = provider.GetServer(m1, ServerConfigWithLogger(testName)))
{
server.Start(EchoingExecutor);
var ex = Assert.Throws <IpcException>(() => server.Start(EchoingExecutor));
Assert.Equal(IpcException.IpcExceptionKind.MultiStart, ex.Kind);
server.RequestStop();
WaitServerDone(server);
}
}
public void TestClientMultipleDispose(IIpcProvider provider)
{
var testName = nameof(TestClientMultipleDispose);
WithIpcServer(
provider,
EchoingExecutor,
ServerConfigWithLogger(testName),
(moniker, server) =>
{
using (var client = provider.GetClient(provider.RenderConnectionString(moniker), ClientConfigWithLogger(testName)))
{
client.Dispose();
}
});
}
public void TestLoadOrCreteMoniker(IIpcProvider provider)
{
var monikerId = "some arbitrary string";
var moniker = provider.LoadOrCreateMoniker(monikerId);
XAssert.AreEqual(monikerId, moniker.Id);
var monikerClone = provider.LoadOrCreateMoniker(monikerId);
XAssert.AreEqual(moniker, monikerClone);
XAssert.AreEqual(moniker.GetHashCode(), monikerClone.GetHashCode());
var differentMoniker = provider.CreateNewMoniker();
XAssert.AreNotEqual(moniker, differentMoniker);
XAssert.AreNotEqual(moniker.GetHashCode(), differentMoniker.GetHashCode());
}
/// <nodoc />
public ApiServer(
IIpcProvider ipcProvider,
string ipcMonikerId,
FileContentManager fileContentManager,
PipExecutionContext context,
IServerConfig config)
{
Contract.Requires(ipcMonikerId != null);
Contract.Requires(fileContentManager != null);
Contract.Requires(context != null);
Contract.Requires(config != null);
m_fileContentManager = fileContentManager;
m_server = ipcProvider.GetServer(ipcProvider.LoadAndRenderMoniker(ipcMonikerId), config);
m_context = context;
}
/// <summary>
/// Async version of <see cref="WithIpcServer(IIpcProvider, IIpcOperationExecutor, IServerConfig, Action{IIpcMoniker, IServer})"/>
/// </summary>
protected static async Task WithIpcServer(IIpcProvider provider, IIpcOperationExecutor executor, IServerConfig config, Func <IIpcMoniker, IServer, Task> testAction)
{
var moniker = provider.CreateNewMoniker();
var server = provider.GetServer(provider.RenderConnectionString(moniker), config);
server.Start(executor);
try
{
await testAction(moniker, server);
}
finally
{
server.RequestStop();
await server.Completion;
server.Dispose();
}
}
public async Task TestWithConnectionErrorAsync(IIpcProvider provider)
{
var testName = nameof(TestWithConnectionErrorAsync);
var connectionString = provider.CreateNewConnectionString();
var config = new ClientConfig()
{
Logger = VerboseLogger(testName),
MaxConnectRetries = 2,
ConnectRetryDelay = TimeSpan.FromMilliseconds(1)
};
using var client = provider.GetClient(connectionString, config);
var syncOpResult = await client.Send(new IpcOperation("sync hi", waitForServerAck : true));
var asyncOpResult = await client.Send(new IpcOperation("async hi", waitForServerAck : false));
// operations should fail because no server was started
XAssert.AreEqual(IpcResultStatus.ConnectionError, syncOpResult.ExitCode);
XAssert.AreEqual(IpcResultStatus.ConnectionError, asyncOpResult.ExitCode);
XAssert.IsFalse(syncOpResult.Succeeded);
XAssert.IsFalse(asyncOpResult.Succeeded);
}
public void TestSimpleSyncOperation(IIpcProvider provider)
{
var testName = nameof(TestSimpleSyncOperation);
WithIpcServer(
provider,
EchoingExecutor,
ServerConfigWithLogger(testName),
(moniker, server) =>
{
using (var client = provider.GetClient(provider.RenderConnectionString(moniker), ClientConfigWithLogger(testName)))
{
var payload = "sync";
var syncOp = new IpcOperation(payload, waitForServerAck: true);
var syncResult = SendWithTimeout(client, syncOp);
Assert.True(syncResult.Succeeded, syncResult.Payload);
Assert.Equal(syncResult.Payload, payload);
client.RequestStop();
client.Completion.GetAwaiter().GetResult();
}
});
}
/// <summary>
/// Creates an execution environment for a single pip. To run pips incrementally, the <paramref name="fileContentTable"/> and <paramref name="pipCache"/> should be specified.
/// </summary>
public DummyPipExecutionEnvironment(
LoggingContext loggingContext,
PipExecutionContext context,
IConfiguration config,
FileContentTable fileContentTable = null,
EngineCache pipCache = null,
SemanticPathExpander semanticPathExpander = null,
PipContentFingerprinter.PipDataLookup pipDataLookup = null,
FileAccessWhitelist fileAccessWhitelist = null,
bool allowUnspecifiedSealedDirectories = false,
PipTable pipTable = null,
IIpcProvider ipcProvider = null,
IKextConnection sandboxedKextConnection = null)
{
Contract.Requires(context != null);
Contract.Requires(config != null);
LoggingContext = loggingContext;
Context = context;
// Ensure paths visible when debugging
PathTable.DebugPathTable = Context.PathTable;
Configuration = config;
PipTable = pipTable;
PathExpander = semanticPathExpander ?? SemanticPathExpander.Default;
ContentFingerprinter = new PipContentFingerprinter(
Context.PathTable,
artifact => State.FileContentManager.GetInputContent(artifact).FileContentInfo,
new ExtraFingerprintSalts(config, PipFingerprintingVersion.TwoPhaseV2, fingerprintSalt: null, searchPathToolsHash: null),
pathExpander: PathExpander,
pipDataLookup: pipDataLookup);
PipFragmentRenderer = this.CreatePipFragmentRenderer();
IpcProvider = ipcProvider ?? IpcFactory.GetProvider();
FileContentTable = fileContentTable ?? FileContentTable.CreateNew();
Cache = pipCache;
FileAccessWhitelist = fileAccessWhitelist;
m_allowUnspecifiedSealedDirectories = allowUnspecifiedSealedDirectories;
m_sandboxedKextConnection = sandboxedKextConnection;
if (Cache == null)
{
Cache = InMemoryCacheFactory.Create(context);
}
var tracker = FileChangeTracker.CreateDisabledTracker(LoggingContext);
LocalDiskContentStore = new LocalDiskContentStore(loggingContext, context.PathTable, FileContentTable, tracker);
PipGraphView = new TestPipGraphFilesystemView(Context.PathTable);
m_operationTracker = new OperationTracker(loggingContext);
var fileSystemView = new FileSystemView(Context.PathTable, PipGraphView, LocalDiskContentStore);
var preserveOutputsSalt = UnsafeOptions.PreserveOutputsNotUsed;
if (config.Sandbox.UnsafeSandboxConfiguration.PreserveOutputs != PreserveOutputsMode.Disabled)
{
preserveOutputsSalt = ContentHashingUtilities.HashString(Guid.NewGuid().ToString());
}
State = new PipExecutionState(
config,
cache: new PipTwoPhaseCache(loggingContext, Cache, context, PathExpander),
fileAccessWhitelist: FileAccessWhitelist,
directoryMembershipFingerprinter: this,
pathExpander: PathExpander,
executionLog: ExecutionLogRecorder,
fileSystemView: fileSystemView,
fileContentManager: GetFileContentManager(),
directoryMembershipFinterprinterRuleSet: null,
unsafeConfiguration: config.Sandbox.UnsafeSandboxConfiguration,
preserveOutputsSalt: preserveOutputsSalt,
serviceManager: new DummyServiceManager());
m_sealContentsById = new ConcurrentBigMap <DirectoryArtifact, int[]>();
ProcessInContainerManager = new ProcessInContainerManager(LoggingContext, context.PathTable);
}
/// <nodoc />
public IpcProviderWithMemoization(IIpcProvider provider, IIpcLogger defaultClientLogger = null)
{
m_provider = provider;
m_defaultClientLogger = defaultClientLogger;
}
/// <nodoc />
public DropDaemon(IParser parser, DaemonConfig daemonConfig, DropConfig dropConfig, Task <IDropClient> dropClientTask, IIpcProvider rpcProvider = null, Client client = null)
: base(parser,
daemonConfig,
!string.IsNullOrWhiteSpace(dropConfig.LogDir) ? new FileLogger(dropConfig.LogDir, LogFileName, daemonConfig.Moniker, dropConfig.Verbose, DropDLogPrefix) : daemonConfig.Logger,
rpcProvider,
client)
{
Contract.Requires(dropConfig != null);
DropConfig = dropConfig;
m_logger.Info("Using DropDaemon config: " + JsonConvert.SerializeObject(Config));
m_dropClientTask = dropClientTask ?? Task.Run(() => (IDropClient) new VsoClient(m_logger, dropConfig));
}
/// <nodoc />
public Daemon(IParser parser, DaemonConfig daemonConfig, DropConfig dropConfig, Task <IDropClient> dropClientTask, IIpcProvider rpcProvider = null, Client client = null)
{
Contract.Requires(daemonConfig != null);
Contract.Requires(dropConfig != null);
Config = daemonConfig;
DropConfig = dropConfig;
m_parser = parser;
ApiClient = client;
m_logger = !string.IsNullOrWhiteSpace(dropConfig.LogDir) ? new FileLogger(dropConfig.LogDir, LogFileName, Config.Moniker, dropConfig.Verbose, DropDLogPrefix) : Config.Logger;
m_logger.Info("Using DropDaemon config: " + JsonConvert.SerializeObject(Config));
rpcProvider = rpcProvider ?? IpcFactory.GetProvider();
m_server = rpcProvider.GetServer(Config.Moniker, Config);
m_etwLogger = new BuildXLBasedCloudBuildLogger(Config.Logger, Config.EnableCloudBuildIntegration);
m_dropClientTask = dropClientTask ?? Task.Run(() => (IDropClient) new VsoClient(m_logger, dropConfig));
}
/// <nodoc />
public ServicePipDaemon(IParser parser, DaemonConfig daemonConfig, IIpcLogger logger, IIpcProvider rpcProvider = null, Client client = null)
{
Contract.Requires(daemonConfig != null);
Config = daemonConfig;
m_parser = parser;
ApiClient = client;
m_logger = logger;
rpcProvider = rpcProvider ?? IpcFactory.GetProvider();
m_server = rpcProvider.GetServer(Config.Moniker, Config);
m_etwLogger = new BuildXLBasedCloudBuildLogger(Config.Logger, Config.EnableCloudBuildIntegration);
}
