protected FrontEndHostController RunEngineAndGetFrontEndHostController(
ICommandLineConfiguration config,
AppDeployment appDeployment,
string testRootDirectory,
bool rememberAllChangedTrackedInputs,
Action <EngineTestHooksData> verifyEngineTestHooksData = null)
{
var result = CreateAndRunEngine(
config,
appDeployment,
testRootDirectory,
rememberAllChangedTrackedInputs,
out var engine,
verifyEngineTestHooksData);
m_frontEndContext = engine.Context.ToFrontEndContext(LoggingContext);
verifyEngineTestHooksData?.Invoke(engine.TestHooks);
// If the engine reloaded and created a new pipGraph we need to udpate the Test file system with that new pipgraph as well
if (!engine.TestHooks.GraphReuseResult.IsNoReuse)
{
FileSystem = (IMutableFileSystem)engine.Context.FileSystem;
}
if (!result.IsSuccess)
{
Assert.True(false, $"Failed to run the engine. See '{testRootDirectory ?? TestOutputDirectory}' for more details.");
}
return((FrontEndHostController)engine.FrontEndController);
}
protected virtual BuildXLEngineResult CreateAndRunEngine(
ICommandLineConfiguration config,
AppDeployment appDeployment,
string testRootDirectory,
bool rememberAllChangedTrackedInputs,
out BuildXLEngine engine,
Action <EngineTestHooksData> verifyEngineTestHooksData = null,
TestCache testCache = null)
{
testCache = testCache ?? TestCache;
using (EngineTestHooksData testHooks = new EngineTestHooksData
{
AppDeployment = appDeployment,
CacheFactory = (context) => new EngineCache(
testCache.GetArtifacts(context),
testCache.Fingerprints)
})
{
engine = CreateEngine(config, appDeployment, testRootDirectory, rememberAllChangedTrackedInputs, verifyEngineTestHooksData);
// Ignore DX222 for csc.exe being outside of src directory
IgnoreWarnings();
engine.TestHooks = testHooks;
var result = engine.RunForFrontEndTests(LoggingContext);
return(result);
}
}
protected virtual BuildXLEngine CreateEngine(ICommandLineConfiguration config,
AppDeployment appDeployment,
string testRootDirectory,
bool rememberAllChangedTrackedInputs,
Action <EngineTestHooksData> verifyEngineTestHooksData = null)
{
var engineContext = EngineContext.CreateNew(CancellationToken.None, PathTable, FileSystem);
var factory = FrontEndControllerFactory.Create(
FrontEndMode.NormalMode,
LoggingContext,
config,
// Set the timeout to a large number to avoid useless performance collections in tests.
new PerformanceCollector(TimeSpan.FromHours(1)),
collectMemoryAsSoonAsPossible: false);
var engine = BuildXLEngine.Create(
LoggingContext,
engineContext,
config,
factory,
rememberAllChangedTrackedInputs: rememberAllChangedTrackedInputs
);
return(engine);
}
private void RunAndAssertGraphCacheHit(ICommandLineConfiguration config, AppDeployment appDeployment, bool rememberAllChangedTrackedInputs = false)
{
using (var hostController = RunEngineAndGetFrontEndHostController(config, appDeployment, null, rememberAllChangedTrackedInputs))
{
AssertNotLogged(LogEventId.EndSerializingPipGraph);
XAssert.IsNull(hostController.Workspace);
}
}
protected static AppDeployment CreateAppDeployment(TempFileStorage tempFiles)
{
string manifestPath = tempFiles.GetFileName(AppDeployment.DeploymentManifestFileName);
File.WriteAllText(
Path.Combine(Path.GetDirectoryName(manifestPath), AppDeployment.DeploymentManifestFileName),
AssemblyHelper.GetAssemblyLocation(Assembly.GetExecutingAssembly()));
AppDeployment appDeployment = AppDeployment.ReadDeploymentManifest(Path.GetDirectoryName(manifestPath), AppDeployment.DeploymentManifestFileName, skipManifestCheckTestHook: true);
return appDeployment;
}
private void updateCheck_DoWork(object sender, DoWorkEventArgs e)
{
try
{
e.Result = AppDeployment.CheckForDetailedUpdate();
}
catch (Exception x)
{
e.Result = x;
}
}
public void Test()
{
string root = Path.Combine(TemporaryDirectory, "testDeployment");
Directory.CreateDirectory(root);
File.WriteAllText(
Path.Combine(root, TestServerManifestName),
@"included1.dll
included1.pdb
excluded1.txt
subdir" + Path.DirectorySeparatorChar + "included2.exe" + Environment.NewLine +
AppDeployment.BuildXLBrandingManifestFileName);
File.WriteAllText(Path.Combine(root, "included1.dll"), "test");
// Include pdbs to aid with call stacks when debugging
File.WriteAllText(Path.Combine(root, "included1.pdb"), "test");
File.WriteAllText(Path.Combine(root, "excluded1.txt"), "test");
File.WriteAllText(Path.Combine(root, "excluded2.dll"), "test");
File.WriteAllText(Path.Combine(root, AppDeployment.BuildXLBrandingManifestFileName), "test");
Directory.CreateDirectory(Path.Combine(root, "subdir"));
File.WriteAllText(Path.Combine(root, "subdir", "included2.exe"), "test");
// Create an initial app deployment and verify it is correct
AppDeployment deployment = AppDeployment.ReadDeploymentManifest(root, TestServerManifestName);
// Verify the file count. PDB files should only be included for the server deployment.
XAssert.AreEqual(4, deployment.GetRelevantRelativePaths(forServerDeployment: true).Count());
XAssert.AreEqual(3, deployment.GetRelevantRelativePaths(forServerDeployment: false).Count());
Fingerprint originalHash = deployment.TimestampBasedHash;
// Now mess with files that should not impact the state and make sure they are excluded
// This file is excluded because of the extension. It is in the manifest
UpdateFile(Path.Combine(root, "excluded1.txt"));
// This file is excluded because it isn't in the deployment manifest
UpdateFile(Path.Combine(root, "excluded2.dll"));
AppDeployment deployment2 = AppDeployment.ReadDeploymentManifest(root, TestServerManifestName);
XAssert.AreEqual(
deployment.TimestampBasedHash.ToHex(),
deployment2.TimestampBasedHash.ToHex());
// Mess with a file that is in the deployment and check that the hash does change
UpdateFile(Path.Combine(root, "subdir", "included2.exe"));
AppDeployment deployment3 = AppDeployment.ReadDeploymentManifest(root, TestServerManifestName);
XAssert.AreNotEqual(
deployment.TimestampBasedHash.ToHex(),
deployment3.TimestampBasedHash.ToHex());
}
private void update_ProgressChanged(UpdateProgress e, ILongWaitBroker broker)
{
if (broker.IsCanceled)
{
AppDeployment.UpdateAsyncCancel();
}
else
{
long updateBytes = Math.Max(_updateInfo.UpdateSizeBytes ?? 0, e.BytesTotal);
broker.Message = GetProgressMessage(e.BytesCompleted, updateBytes);
broker.ProgressValue = updateBytes == 0
? 0 : (int)Math.Min(99, e.BytesCompleted * 100 / updateBytes);
}
}
public void IdentityManifestMissing(bool includeInDeploymentManifest)
{
string root = Path.Combine(TemporaryDirectory, "testDeployment");
Directory.CreateDirectory(root);
File.WriteAllText(
Path.Combine(root, TestServerManifestName),
"included1.dll" + Environment.NewLine +
(includeInDeploymentManifest ? AppDeployment.BuildXLBrandingManifestFileName : string.Empty));
if (!includeInDeploymentManifest)
{
File.WriteAllText(Path.Combine(root, AppDeployment.BuildXLBrandingManifestFileName), "whatever");
}
Assert.Throws <BuildXLException>(() => AppDeployment.ReadDeploymentManifest(root, TestServerManifestName));
}
public void TestMissingManifestFile()
{
var manifestRootDir = TemporaryDirectory;
var manifestPath = Path.Combine(manifestRootDir, AppDeployment.DeploymentManifestFileName);
File.WriteAllText(manifestPath, AssemblyHelper.GetAssemblyLocation(Assembly.GetExecutingAssembly()));
var appDeployment = AppDeployment.ReadDeploymentManifest(
manifestRootDir,
AppDeployment.DeploymentManifestFileName,
skipManifestCheckTestHook: true);
string deploymentDir = ServerDeployment.ComputeDeploymentDir(manifestRootDir);
Directory.CreateDirectory(deploymentDir);
XAssert.IsTrue(ServerDeployment.IsServerDeploymentOutOfSync(manifestRootDir, appDeployment, out deploymentDir));
}
/// <summary>
/// Checks if the deployed server bits are still up-to-date.
/// </summary>
public static bool IsServerDeploymentOutOfSync(string serverDeploymentRoot, AppDeployment clientApp, out string deploymentDir)
{
deploymentDir = ComputeDeploymentDir(serverDeploymentRoot);
if (Directory.Exists(deploymentDir))
{
try
{
// Calculates a hash of the contents of the BuildXL binaries from the server deployment directory.
AppDeployment serverDeployment = AppDeployment.ReadDeploymentManifest(deploymentDir, AppDeployment.ServerDeploymentManifestFileName);
return(clientApp.TimestampBasedHash.ToHex() != serverDeployment.TimestampBasedHash.ToHex());
}
catch (BuildXLException)
{
// This case occurs when the deployment directory is missing or the ServerDeploymentManifestFile is missing
return(true);
}
}
return(true);
}
/// <summary>
/// If the deployment hash of the client is not the same as the deployment hash of the server cache, creates a new deployment cache. Otherwise, does nothing.
/// </summary>
/// <exception cref="IOException">
/// Throws if the copy fails</exception>
public static ServerDeployment GetOrCreateServerDeploymentCache(string serverDeploymentRoot, AppDeployment clientApp)
{
var deploymentDir = ComputeDeploymentDir(serverDeploymentRoot);
ServerDeploymentCacheCreated?cacheCreated = null;
if (!Directory.Exists(deploymentDir) || clientApp.TimestampBasedHash.ToHex() != GetDeploymentCacheHash(deploymentDir))
{
cacheCreated = CreateServerDeployment(deploymentDir, clientApp);
}
return(new ServerDeployment(deploymentDir, cacheCreated));
}
private void BeginUpdate(ILongWaitBroker broker)
{
AppDeployment.UpdateAsync(ev => update_ProgressChanged(ev, broker), update_Complete);
}
private void updateCheck_Complete(object sender, RunWorkerCompletedEventArgs e)
{
var exTrust = e.Result as TrustNotGrantedException;
if (exTrust != null)
{
if (ShowUpgradeForm(AppDeployment.GetVersionFromUpdateLocation(), false, true))
{
AppDeployment.OpenInstallLink(ParentWindow);
}
return;
}
var ex = e.Result as Exception;
if (ex != null)
{
// Show an error message box to allow a user to inspect the exception stack trace
MessageDlg.ShowWithException(ParentWindow,
Resources.UpgradeManager_updateCheck_Complete_Failed_attempting_to_check_for_an_upgrade_, ex);
// Show no upgrade found message to allow a user to turn off or on this checking
ShowUpgradeForm(null, false, false);
return;
}
_updateInfo = e.Result as UpdateCheckDetails;
if (_updateInfo != null && _updateInfo.UpdateAvailable)
{
if (!ShowUpgradeForm(_updateInfo.AvailableVersion, true, true))
{
return;
}
using (var longWaitUpdate = new LongWaitDlg
{
Text = string.Format(Resources.UpgradeManager_updateCheck_Complete_Upgrading__0_, Program.Name),
Message = GetProgressMessage(0, _updateInfo.UpdateSizeBytes ?? 0),
ProgressValue = 0
})
{
longWaitUpdate.PerformWork(ParentWindow, 500, broker =>
{
BeginUpdate(broker);
_endUpdateEvent.WaitOne();
_endUpdateEvent.Dispose();
broker.ProgressValue = 100;
});
}
if (_completeArgs == null || _completeArgs.Cancelled)
{
return;
}
if (_completeArgs.Error != null)
{
MessageDlg.ShowWithException(ParentWindow,
Resources.UpgradeManager_updateCheck_Complete_Failed_attempting_to_upgrade_, _completeArgs.Error);
if (ShowUpgradeForm(null, false, true))
{
AppDeployment.OpenInstallLink(ParentWindow);
}
return;
}
AppDeployment.Restart();
}
else if (!_startup)
{
ShowUpgradeForm(null, false, false);
}
}
/// <summary>
/// Checks if the deployed server bits are still up-to-date.
/// </summary>
public static bool IsServerDeploymentOutOfSync(string serverDeploymentRoot, AppDeployment clientApp, out string deploymentDir)
{
deploymentDir = ComputeDeploymentDir(serverDeploymentRoot);
return(!Directory.Exists(deploymentDir) || clientApp.TimestampBasedHash.ToHex() != GetDeploymentCacheHash(deploymentDir));
}
protected IList <AbsolutePath> RunAndRetrieveSpecs(ICommandLineConfiguration config, AppDeployment appDeployment, bool rememberAllChangedTrackedInputs = false)
{
IList <AbsolutePath> specs;
using (var controller = RunEngineAndGetFrontEndHostController(config, appDeployment, null, rememberAllChangedTrackedInputs))
{
var workspace = controller.Workspace;
specs = workspace.GetAllSpecFiles().Where(spec => !workspace.PreludeModule.Specs.ContainsKey(spec)).ToList();
}
Assert.True(specs != null);
return(specs);
}
/// <summary>
/// If the deployment hash of the client is not the same as the deployment hash of the server cache, creates a new deployment cache. Otherwise, does nothing.
/// </summary>
/// <exception cref="IOException">
/// Throws if the copy fails</exception>
public static ServerDeployment GetOrCreateServerDeploymentCache(string serverDeploymentRoot, AppDeployment clientApp)
{
ServerDeploymentCacheCreated?cacheCreated = null;
if (IsServerDeploymentOutOfSync(serverDeploymentRoot, clientApp, out var deploymentDir))
{
cacheCreated = CreateServerDeployment(deploymentDir, clientApp);
}
return(new ServerDeployment(deploymentDir, cacheCreated));
}
private static ServerDeploymentCacheCreated CreateServerDeployment(string destDir, AppDeployment clientApp)
{
Stopwatch st = Stopwatch.StartNew();
// Every time the server cache gets created, a file with the result of GetDeploymentHash is created, so we avoid computing this again
// The assumption is that nobody but this process modifies the server cache folder
string serverDeploymentHashFile = Path.Combine(destDir, ServerDeploymentHashFilename);
// Deletes the existing cache directory if it exists, so we avoid accumulating garbage.
if (Directory.Exists(destDir))
{
// Check if the main root process (likely bxl.exe) is in use before attempting to delete, so we avoid partially deleting files
// Not completely bullet proof (there can be a race) but it is highly unlikely the process starts to be in use right after this check
KillServer(destDir);
// Delete the deployment hash file first to make sure the deployment cache cannot be used in case the
// cleanup of the other files is interrupted.
PoisonServerDeployment(destDir);
// Remove all files regardless of files being readonly
FileUtilities.DeleteDirectoryContents(destDir, true);
}
// Perform the deployment
AppDeployment serverDeployment = AppDeployment.ReadDeploymentManifest(clientApp.BaseDirectory, AppDeployment.ServerDeploymentManifestFileName);
HashSet <string> directories = new HashSet <string>();
List <KeyValuePair <string, string> > filesToCopy = new List <KeyValuePair <string, string> >();
foreach (string path in serverDeployment.GetRelevantRelativePaths(forServerDeployment: true).Concat(new string[] { AppDeployment.ServerDeploymentManifestFileName }))
{
string targetPath = Path.Combine(destDir, path);
string sourcePath = Path.Combine(clientApp.BaseDirectory, path);
string directory = Path.GetDirectoryName(targetPath);
if (directories.Add(directory))
{
FileUtilities.CreateDirectory(directory);
}
filesToCopy.Add(new KeyValuePair <string, string>(sourcePath, targetPath));
}
// Because some deployments use virtualized vpak, using a very parallelized copy is beneficial
Parallel.ForEach(
filesToCopy,
new ParallelOptions()
{
MaxDegreeOfParallelism = 50,
},
(fileToCopy) =>
{
if (File.Exists(fileToCopy.Key))
{
File.Copy(fileToCopy.Key, fileToCopy.Value);
}
});
#if !FEATURE_CORECLR
var ngenExe = Path.Combine(System.Runtime.InteropServices.RuntimeEnvironment.GetRuntimeDirectory(), @"ngen.exe");
var destExe = Path.Combine(destDir, System.AppDomain.CurrentDomain.FriendlyName);
// queue:1 means it runs in the background
var ngenArgs = "install " + destExe + " /queue:1";
bool runNgen = File.Exists(ngenExe);
if (runNgen)
{
ProcessStartInfo startInfo = new ProcessStartInfo(ngenExe, ngenArgs);
startInfo.UseShellExecute = false;
startInfo.CreateNoWindow = true;
Process.Start(startInfo);
}
#endif
using (var file = new StreamWriter(File.OpenWrite(serverDeploymentHashFile)))
{
file.WriteLine(clientApp.TimestampBasedHash);
// This isn't actually consumed. It is only used for debugging
file.WriteLine("Debug info:");
#if !FEATURE_CORECLR
if (runNgen)
{
file.WriteLine("Ran Ngen: " + ngenExe + " " + ngenArgs);
}
#endif
file.WriteLine(clientApp.TimestampBasedHashDebug);
}
ServerDeploymentCacheCreated cacheCreated = default(ServerDeploymentCacheCreated);
cacheCreated.TimeToCreateServerCacheMilliseconds = st.ElapsedMilliseconds;
return(cacheCreated);
}
/// <summary>
/// Calculates a hash of the contents of the BuildXL binaries from the server deployment directory.
/// </summary>
public static string GetDeploymentCacheHash(string deploymentDir)
{
AppDeployment serverDeployment = AppDeployment.ReadDeploymentManifest(deploymentDir, AppDeployment.ServerDeploymentManifestFileName);
return(serverDeployment.TimestampBasedHash.ToHex());
}
private static ServerDeploymentCacheCreated CreateServerDeployment(string destDir, AppDeployment clientApp)
{
Stopwatch st = Stopwatch.StartNew();
// Check if the main server process is in use before attempting to delete the deployment, this way we avoid partially deleting files
// due to access permission issues. This is not completely bullet proof (there can be a race) but it is highly unlikely the
// process starts to be in use right after this check
KillServer(destDir);
// Deletes the existing cache directory if it exists, so we avoid accumulating garbage.
if (Directory.Exists(destDir))
{
// Remove all files regardless of files being readonly
FileUtilities.DeleteDirectoryContents(destDir, true);
}
// Perform the deployment
AppDeployment serverDeployment = AppDeployment.ReadDeploymentManifest(clientApp.BaseDirectory, AppDeployment.ServerDeploymentManifestFileName);
HashSet <string> directories = new HashSet <string>();
List <KeyValuePair <string, string> > filesToCopy = new List <KeyValuePair <string, string> >();
foreach (string path in serverDeployment.GetRelevantRelativePaths(forServerDeployment: true).Concat(new string[] { AppDeployment.ServerDeploymentManifestFileName }))
{
string targetPath = Path.Combine(destDir, path);
string sourcePath = Path.Combine(clientApp.BaseDirectory, path);
string directory = Path.GetDirectoryName(targetPath);
if (directories.Add(directory))
{
FileUtilities.CreateDirectory(directory);
}
filesToCopy.Add(new KeyValuePair <string, string>(sourcePath, targetPath));
}
// Because some deployments use virtualized vpak, using a very parallelized copy is beneficial
Parallel.ForEach(
filesToCopy,
new ParallelOptions()
{
MaxDegreeOfParallelism = 50,
},
(fileToCopy) =>
{
if (File.Exists(fileToCopy.Key))
{
File.Copy(fileToCopy.Key, fileToCopy.Value);
}
});
#if NET_FRAMEWORK
var ngenExe = Path.Combine(System.Runtime.InteropServices.RuntimeEnvironment.GetRuntimeDirectory(), @"ngen.exe");
var destExe = Path.Combine(destDir, System.AppDomain.CurrentDomain.FriendlyName);
// queue:1 means it runs in the background
if (File.Exists(ngenExe))
{
var ngenArgs = "install " + destExe + " /queue:1";
ProcessStartInfo startInfo = new ProcessStartInfo(ngenExe, ngenArgs);
startInfo.UseShellExecute = false;
startInfo.CreateNoWindow = true;
Process.Start(startInfo);
}
#endif
ServerDeploymentCacheCreated cacheCreated = default(ServerDeploymentCacheCreated);
cacheCreated.TimeToCreateServerCacheMilliseconds = st.ElapsedMilliseconds;
return(cacheCreated);
}
