/// <nodoc /> public ApiServer( IIpcProvider ipcProvider, string ipcMonikerId, FileContentManager fileContentManager, PipExecutionContext context, IServerConfig config, EngineCache engineCache, Tracing.IExecutionLogTarget executionLog, Tracing.BuildManifestGenerator buildManifestGenerator) { Contract.Requires(ipcMonikerId != null); Contract.Requires(fileContentManager != null); Contract.Requires(context != null); Contract.Requires(config != null); Contract.Requires(engineCache != null); Contract.Requires(executionLog != null); m_fileContentManager = fileContentManager; m_server = ipcProvider.GetServer(ipcProvider.LoadAndRenderMoniker(ipcMonikerId), config); m_context = context; m_engineCache = engineCache; m_executionLog = executionLog; m_buildManifestGenerator = buildManifestGenerator; m_inMemoryBuildManifestStore = new ConcurrentDictionary <ContentHash, ContentHash>(); }
/// <nodoc/> public MaterializationDaemon( IParser parser, DaemonConfig daemonConfig, MaterializationDaemonConfig materializationConfig, IIpcProvider rpcProvider = null, Client bxlClient = null) : base(parser, daemonConfig, !string.IsNullOrWhiteSpace(materializationConfig.LogDir) ? new FileLogger(materializationConfig.LogDir, LogFileName, daemonConfig.Moniker, logVerbose: true, MaterializationDaemonLogPrefix) : daemonConfig.Logger, rpcProvider, bxlClient) { m_config = materializationConfig; m_actionQueue = new ActionQueue(m_config.MaxDegreeOfParallelism); m_materializationStatus = new ConcurrentBigMap <string, bool>(); m_counters = new CounterCollection <MaterializationDaemonCounter>(); m_macros = new Dictionary <string, string> { ["$(build.nttree)"] = Environment.GetEnvironmentVariable("_NTTREE") }; m_logger.Info($"MaterializationDaemon config: {JsonConvert.SerializeObject(m_config)}"); m_logger.Info($"Defined macros (count={m_macros.Count}):{Environment.NewLine}{string.Join(Environment.NewLine, m_macros.Select(kvp => $"{kvp.Key}={kvp.Value}"))}"); }
public void TestWithExecutionError(IIpcProvider provider) { var testName = nameof(TestWithExecutionError); WithIpcServer( provider, CrashingExecutor, ServerConfigWithLogger(testName), (moniker, server) => { using (var client = provider.GetClient(provider.RenderConnectionString(moniker), ClientConfigWithLogger(testName))) { var syncOp = new IpcOperation("sync", waitForServerAck: true); var asyncOp = new IpcOperation("async", waitForServerAck: false); var syncResult = SendWithTimeout(client, syncOp); var asyncResult = SendWithTimeout(client, asyncOp); Assert.True(asyncResult.Succeeded, "Asynchronous operation is expected to succeed if executor crashes"); Assert.False(syncResult.Succeeded, "Synchronous operation is expected to fail if executor crashes"); Assert.Equal(IpcResultStatus.ExecutionError, syncResult.ExitCode); Assert.True(syncResult.Payload.Contains("System.Exception")); // because CrashingExecutor throws System.Exception Assert.True(syncResult.Payload.Contains(syncOp.Payload)); // because CrashingExecutor throws System.Exception whose message is equal to syncOp.Payload client.RequestStop(); client.Completion.GetAwaiter().GetResult(); } }); }
/// <nodoc /> public ApiServer( IIpcProvider ipcProvider, string ipcMonikerId, FileContentManager fileContentManager, PipExecutionContext context, IServerConfig config, PipTwoPhaseCache pipTwoPhaseCache, Tracing.IExecutionLogTarget executionLog, Tracing.BuildManifestGenerator buildManifestGenerator, ServiceManager serviceManger, bool verifyFileContentOnBuildManifestHashComputation) { Contract.Requires(ipcMonikerId != null); Contract.Requires(fileContentManager != null); Contract.Requires(context != null); Contract.Requires(config != null); Contract.Requires(pipTwoPhaseCache != null); Contract.Requires(executionLog != null); m_fileContentManager = fileContentManager; m_server = ipcProvider.GetServer(ipcProvider.LoadAndRenderMoniker(ipcMonikerId), config); m_context = context; m_executionLog = executionLog; m_buildManifestGenerator = buildManifestGenerator; m_serviceManger = serviceManger; m_pipTwoPhaseCache = pipTwoPhaseCache; m_inMemoryBuildManifestStore = new ConcurrentBigMap <ContentHash, IReadOnlyList <ContentHash> >(); m_receivedStatistics = new ConcurrentBigMap <string, long>(); m_verifyFileContentOnBuildManifestHashComputation = verifyFileContentOnBuildManifestHashComputation; }
public void TestConcurrentSynchronousOperations(IIpcProvider provider) { var testName = nameof(TestConcurrentSynchronousOperations); WithIpcServer( provider, EchoingExecutor, ServerConfigWithLogger(testName), (moniker, server) => { using (IClient client = provider.GetClient(provider.RenderConnectionString(moniker), ClientConfigWithLogger(testName))) { var threads = Enumerable .Range(1, 10) .Select(i => new Thread(() => { var message = "hi" + i; var op = new IpcOperation(message, waitForServerAck: true); var result = SendWithTimeout(client, op); Assert.True(result.Succeeded, "error: " + result.Payload); Assert.Equal(op.Payload, result.Payload); })) .ToArray(); Start(threads); Join(threads); client.RequestStop(); client.Completion.GetAwaiter().GetResult(); } }); }
private WeakContentFingerprint GenerateSaltedWeakFingerprint(ContentHash hash) => new WeakContentFingerprint(FingerprintUtilities.Hash($"Hash: '{hash.ToHex()}' Salt: '{m_buildManifestHashCacheSalt}'")); // Changes to this string will invalidate all existing cache entries /// <nodoc /> public ApiServer( IIpcProvider ipcProvider, string ipcMonikerId, FileContentManager fileContentManager, PipExecutionContext context, IServerConfig config, EngineCache engineCache, Tracing.IExecutionLogTarget executionLog, Tracing.BuildManifestGenerator buildManifestGenerator) { Contract.Requires(ipcMonikerId != null); Contract.Requires(fileContentManager != null); Contract.Requires(context != null); Contract.Requires(config != null); Contract.Requires(engineCache != null); Contract.Requires(executionLog != null); m_fileContentManager = fileContentManager; m_server = ipcProvider.GetServer(ipcProvider.LoadAndRenderMoniker(ipcMonikerId), config); m_context = context; m_engineCache = engineCache; m_executionLog = executionLog; m_buildManifestGenerator = buildManifestGenerator; m_inMemoryBuildManifestStore = new ConcurrentBigMap <ContentHash, ContentHash>(); m_buildManifestHashCacheSalt = string.IsNullOrEmpty(Utilities.Configuration.EngineEnvironmentSettings.BuildManifestHashCacheSalt) ? string.Empty : Utilities.Configuration.EngineEnvironmentSettings.BuildManifestHashCacheSalt; }
public void TestCreateDisposeClient(IIpcProvider provider) { var m1 = provider.CreateNewConnectionString(); using (provider.GetClient(m1, new ClientConfig())) { } }
public void TestCreateDisposeServer(IIpcProvider provider) { var m1 = provider.CreateNewConnectionString(); using (var server = provider.GetServer(m1, new ServerConfig())) { } }
public void TestCreateNewMonikerReturnsUniqueMonikers(IIpcProvider provider) { var m1 = provider.CreateNewMoniker(); var m2 = provider.CreateNewMoniker(); Assert.NotNull(m1); Assert.NotNull(m2); Assert.NotEqual(m1, m2); }
public void TestServerConsideredNotCompletedBeforeStarted(IIpcProvider provider) { var m1 = provider.CreateNewConnectionString(); using (var server = provider.GetServer(m1, new ServerConfig())) { Assert.False(server.Completion.IsCompleted); } }
public void TestRenderConnectionStringReturnsSameString(IIpcProvider provider) { var m1 = provider.CreateNewMoniker(); var connStr1 = provider.RenderConnectionString(m1); var connStr2 = provider.RenderConnectionString(m1); Assert.NotNull(connStr1); Assert.NotNull(connStr2); Assert.Equal(connStr1, connStr2); }
private IEnumerable <Thread> GetClientThreads(IIpcProvider ipcProvider, IEnumerable <string> ipcMonikers, int numServices, int numRequests, string cmdLine) { return(ipcMonikers .SelectMany(moniker => Enumerable .Range(1, numRequests) .Select(i => CreateThreadForCommand(cmdLine.Replace("<moniker>", moniker), ipcProvider.GetClient(moniker, new ClientConfig()))) .ToList()) .ToList()); }
public void TestServerMultipleDispose(IIpcProvider provider) { var m1 = provider.CreateNewConnectionString(); using (var server = provider.GetServer(m1, new ServerConfig())) { server.Start(EchoingExecutor); server.RequestStop(); WaitServerDone(server); server.Dispose(); } }
public void TestCreateStartStopDisposeServer(IIpcProvider provider) { var m1 = provider.CreateNewConnectionString(); using (var server = provider.GetServer(m1, new ServerConfig())) { server.Start(EchoingExecutor); Assert.False(server.Completion.IsCompleted); server.RequestStop(); WaitServerDone(server); } }
/// <summary> /// 1. creates a server /// 2. starts the server /// 3. invokes 'testAction' /// 4. shuts down the server /// 5. waits for the server to complete. /// </summary> protected static void WithIpcServer(IIpcProvider provider, IIpcOperationExecutor executor, IServerConfig config, Action <IIpcMoniker, IServer> testAction) { WithIpcServer( provider, executor, config, (moniker, server) => { testAction(moniker, server); return(Task.FromResult(1)); }).GetAwaiter().GetResult(); }
public TestScheduler( PipGraph graph, TestPipQueue pipQueue, PipExecutionContext context, FileContentTable fileContentTable, EngineCache cache, IConfiguration configuration, FileAccessWhitelist fileAccessWhitelist, DirectoryMembershipFingerprinterRuleSet directoryMembershipFingerprinterRules = null, ITempCleaner tempCleaner = null, PipRuntimeTimeTable runningTimeTable = null, JournalState journalState = null, PerformanceCollector performanceCollector = null, string fingerprintSalt = null, PreserveOutputsInfo?previousInputsSalt = null, IEnumerable <Pip> successfulPips = null, IEnumerable <Pip> failedPips = null, LoggingContext loggingContext = null, IIpcProvider ipcProvider = null, DirectoryTranslator directoryTranslator = null, VmInitializer vmInitializer = null, SchedulerTestHooks testHooks = null) : base(graph, pipQueue, context, fileContentTable, cache, configuration, fileAccessWhitelist, loggingContext, null, directoryMembershipFingerprinterRules, tempCleaner, AsyncLazy <PipRuntimeTimeTable> .FromResult(runningTimeTable), performanceCollector, fingerprintSalt, previousInputsSalt, ipcProvider: ipcProvider, directoryTranslator: directoryTranslator, journalState: journalState, vmInitializer: vmInitializer, testHooks: testHooks) { m_testPipQueue = pipQueue; if (successfulPips != null) { foreach (var pip in successfulPips) { Contract.Assume(pip.PipId.IsValid, "Override results must be added after the pip has been added to the scheduler"); m_overridePipResults.Add(pip.PipId, PipResultStatus.Succeeded); } } if (failedPips != null) { foreach (var pip in failedPips) { Contract.Assume(pip.PipId.IsValid, "Override results must be added after the pip has been added to the scheduler"); m_overridePipResults.Add(pip.PipId, PipResultStatus.Failed); } } m_loggingContext = loggingContext; }
/// <summary> /// Creates an execution environment for a single pip. To run pips incrementally, the <paramref name="fileContentTable"/> and <paramref name="pipCache"/> should be specified. /// </summary> public DummyPipExecutionEnvironment( LoggingContext loggingContext, PipExecutionContext context, IConfiguration config, FileContentTable fileContentTable = null, EngineCache pipCache = null, SemanticPathExpander semanticPathExpander = null, PipContentFingerprinter.PipDataLookup pipDataLookup = null, FileAccessWhitelist fileAccessWhitelist = null, bool allowUnspecifiedSealedDirectories = false, PipTable pipTable = null, IIpcProvider ipcProvider = null, (string substSource, string substTarget)?subst = default,
/// <nodoc/> public MaterializationDaemon( IParser parser, DaemonConfig daemonConfig, MaterializationDaemonConfig materializationConfig, IIpcProvider rpcProvider = null, Client bxlClient = null) : base(parser, daemonConfig, !string.IsNullOrWhiteSpace(materializationConfig.LogDir) ? new FileLogger(materializationConfig.LogDir, LogFileName, daemonConfig.Moniker, logVerbose: true, MaterializationDaemonLogPrefix) : daemonConfig.Logger, rpcProvider, bxlClient) { }
public void TestServerMultipleStartFails(IIpcProvider provider) { var testName = nameof(TestServerMultipleStartFails); var m1 = provider.CreateNewConnectionString(); using (var server = provider.GetServer(m1, ServerConfigWithLogger(testName))) { server.Start(EchoingExecutor); var ex = Assert.Throws <IpcException>(() => server.Start(EchoingExecutor)); Assert.Equal(IpcException.IpcExceptionKind.MultiStart, ex.Kind); server.RequestStop(); WaitServerDone(server); } }
public void TestClientMultipleDispose(IIpcProvider provider) { var testName = nameof(TestClientMultipleDispose); WithIpcServer( provider, EchoingExecutor, ServerConfigWithLogger(testName), (moniker, server) => { using (var client = provider.GetClient(provider.RenderConnectionString(moniker), ClientConfigWithLogger(testName))) { client.Dispose(); } }); }
public void TestLoadOrCreteMoniker(IIpcProvider provider) { var monikerId = "some arbitrary string"; var moniker = provider.LoadOrCreateMoniker(monikerId); XAssert.AreEqual(monikerId, moniker.Id); var monikerClone = provider.LoadOrCreateMoniker(monikerId); XAssert.AreEqual(moniker, monikerClone); XAssert.AreEqual(moniker.GetHashCode(), monikerClone.GetHashCode()); var differentMoniker = provider.CreateNewMoniker(); XAssert.AreNotEqual(moniker, differentMoniker); XAssert.AreNotEqual(moniker.GetHashCode(), differentMoniker.GetHashCode()); }
/// <nodoc /> public ApiServer( IIpcProvider ipcProvider, string ipcMonikerId, FileContentManager fileContentManager, PipExecutionContext context, IServerConfig config) { Contract.Requires(ipcMonikerId != null); Contract.Requires(fileContentManager != null); Contract.Requires(context != null); Contract.Requires(config != null); m_fileContentManager = fileContentManager; m_server = ipcProvider.GetServer(ipcProvider.LoadAndRenderMoniker(ipcMonikerId), config); m_context = context; }
/// <summary> /// Async version of <see cref="WithIpcServer(IIpcProvider, IIpcOperationExecutor, IServerConfig, Action{IIpcMoniker, IServer})"/> /// </summary> protected static async Task WithIpcServer(IIpcProvider provider, IIpcOperationExecutor executor, IServerConfig config, Func <IIpcMoniker, IServer, Task> testAction) { var moniker = provider.CreateNewMoniker(); var server = provider.GetServer(provider.RenderConnectionString(moniker), config); server.Start(executor); try { await testAction(moniker, server); } finally { server.RequestStop(); await server.Completion; server.Dispose(); } }
public async Task TestWithConnectionErrorAsync(IIpcProvider provider) { var testName = nameof(TestWithConnectionErrorAsync); var connectionString = provider.CreateNewConnectionString(); var config = new ClientConfig() { Logger = VerboseLogger(testName), MaxConnectRetries = 2, ConnectRetryDelay = TimeSpan.FromMilliseconds(1) }; using var client = provider.GetClient(connectionString, config); var syncOpResult = await client.Send(new IpcOperation("sync hi", waitForServerAck : true)); var asyncOpResult = await client.Send(new IpcOperation("async hi", waitForServerAck : false)); // operations should fail because no server was started XAssert.AreEqual(IpcResultStatus.ConnectionError, syncOpResult.ExitCode); XAssert.AreEqual(IpcResultStatus.ConnectionError, asyncOpResult.ExitCode); XAssert.IsFalse(syncOpResult.Succeeded); XAssert.IsFalse(asyncOpResult.Succeeded); }
public void TestSimpleSyncOperation(IIpcProvider provider) { var testName = nameof(TestSimpleSyncOperation); WithIpcServer( provider, EchoingExecutor, ServerConfigWithLogger(testName), (moniker, server) => { using (var client = provider.GetClient(provider.RenderConnectionString(moniker), ClientConfigWithLogger(testName))) { var payload = "sync"; var syncOp = new IpcOperation(payload, waitForServerAck: true); var syncResult = SendWithTimeout(client, syncOp); Assert.True(syncResult.Succeeded, syncResult.Payload); Assert.Equal(syncResult.Payload, payload); client.RequestStop(); client.Completion.GetAwaiter().GetResult(); } }); }
/// <summary> /// Creates an execution environment for a single pip. To run pips incrementally, the <paramref name="fileContentTable"/> and <paramref name="pipCache"/> should be specified. /// </summary> public DummyPipExecutionEnvironment( LoggingContext loggingContext, PipExecutionContext context, IConfiguration config, FileContentTable fileContentTable = null, EngineCache pipCache = null, SemanticPathExpander semanticPathExpander = null, PipContentFingerprinter.PipDataLookup pipDataLookup = null, FileAccessWhitelist fileAccessWhitelist = null, bool allowUnspecifiedSealedDirectories = false, PipTable pipTable = null, IIpcProvider ipcProvider = null, IKextConnection sandboxedKextConnection = null) { Contract.Requires(context != null); Contract.Requires(config != null); LoggingContext = loggingContext; Context = context; // Ensure paths visible when debugging PathTable.DebugPathTable = Context.PathTable; Configuration = config; PipTable = pipTable; PathExpander = semanticPathExpander ?? SemanticPathExpander.Default; ContentFingerprinter = new PipContentFingerprinter( Context.PathTable, artifact => State.FileContentManager.GetInputContent(artifact).FileContentInfo, new ExtraFingerprintSalts(config, PipFingerprintingVersion.TwoPhaseV2, fingerprintSalt: null, searchPathToolsHash: null), pathExpander: PathExpander, pipDataLookup: pipDataLookup); PipFragmentRenderer = this.CreatePipFragmentRenderer(); IpcProvider = ipcProvider ?? IpcFactory.GetProvider(); FileContentTable = fileContentTable ?? FileContentTable.CreateNew(); Cache = pipCache; FileAccessWhitelist = fileAccessWhitelist; m_allowUnspecifiedSealedDirectories = allowUnspecifiedSealedDirectories; m_sandboxedKextConnection = sandboxedKextConnection; if (Cache == null) { Cache = InMemoryCacheFactory.Create(context); } var tracker = FileChangeTracker.CreateDisabledTracker(LoggingContext); LocalDiskContentStore = new LocalDiskContentStore(loggingContext, context.PathTable, FileContentTable, tracker); PipGraphView = new TestPipGraphFilesystemView(Context.PathTable); m_operationTracker = new OperationTracker(loggingContext); var fileSystemView = new FileSystemView(Context.PathTable, PipGraphView, LocalDiskContentStore); var preserveOutputsSalt = UnsafeOptions.PreserveOutputsNotUsed; if (config.Sandbox.UnsafeSandboxConfiguration.PreserveOutputs != PreserveOutputsMode.Disabled) { preserveOutputsSalt = ContentHashingUtilities.HashString(Guid.NewGuid().ToString()); } State = new PipExecutionState( config, cache: new PipTwoPhaseCache(loggingContext, Cache, context, PathExpander), fileAccessWhitelist: FileAccessWhitelist, directoryMembershipFingerprinter: this, pathExpander: PathExpander, executionLog: ExecutionLogRecorder, fileSystemView: fileSystemView, fileContentManager: GetFileContentManager(), directoryMembershipFinterprinterRuleSet: null, unsafeConfiguration: config.Sandbox.UnsafeSandboxConfiguration, preserveOutputsSalt: preserveOutputsSalt, serviceManager: new DummyServiceManager()); m_sealContentsById = new ConcurrentBigMap <DirectoryArtifact, int[]>(); ProcessInContainerManager = new ProcessInContainerManager(LoggingContext, context.PathTable); }
/// <nodoc /> public IpcProviderWithMemoization(IIpcProvider provider, IIpcLogger defaultClientLogger = null) { m_provider = provider; m_defaultClientLogger = defaultClientLogger; }
/// <nodoc /> public DropDaemon(IParser parser, DaemonConfig daemonConfig, DropConfig dropConfig, Task <IDropClient> dropClientTask, IIpcProvider rpcProvider = null, Client client = null) : base(parser, daemonConfig, !string.IsNullOrWhiteSpace(dropConfig.LogDir) ? new FileLogger(dropConfig.LogDir, LogFileName, daemonConfig.Moniker, dropConfig.Verbose, DropDLogPrefix) : daemonConfig.Logger, rpcProvider, client) { Contract.Requires(dropConfig != null); DropConfig = dropConfig; m_logger.Info("Using DropDaemon config: " + JsonConvert.SerializeObject(Config)); m_dropClientTask = dropClientTask ?? Task.Run(() => (IDropClient) new VsoClient(m_logger, dropConfig)); }
/// <nodoc /> public Daemon(IParser parser, DaemonConfig daemonConfig, DropConfig dropConfig, Task <IDropClient> dropClientTask, IIpcProvider rpcProvider = null, Client client = null) { Contract.Requires(daemonConfig != null); Contract.Requires(dropConfig != null); Config = daemonConfig; DropConfig = dropConfig; m_parser = parser; ApiClient = client; m_logger = !string.IsNullOrWhiteSpace(dropConfig.LogDir) ? new FileLogger(dropConfig.LogDir, LogFileName, Config.Moniker, dropConfig.Verbose, DropDLogPrefix) : Config.Logger; m_logger.Info("Using DropDaemon config: " + JsonConvert.SerializeObject(Config)); rpcProvider = rpcProvider ?? IpcFactory.GetProvider(); m_server = rpcProvider.GetServer(Config.Moniker, Config); m_etwLogger = new BuildXLBasedCloudBuildLogger(Config.Logger, Config.EnableCloudBuildIntegration); m_dropClientTask = dropClientTask ?? Task.Run(() => (IDropClient) new VsoClient(m_logger, dropConfig)); }
/// <nodoc /> public ServicePipDaemon(IParser parser, DaemonConfig daemonConfig, IIpcLogger logger, IIpcProvider rpcProvider = null, Client client = null) { Contract.Requires(daemonConfig != null); Config = daemonConfig; m_parser = parser; ApiClient = client; m_logger = logger; rpcProvider = rpcProvider ?? IpcFactory.GetProvider(); m_server = rpcProvider.GetServer(Config.Moniker, Config); m_etwLogger = new BuildXLBasedCloudBuildLogger(Config.Logger, Config.EnableCloudBuildIntegration); }