/// <summary>
/// Removes a corrupt file and attempts to save it to the appropriate logs directory for future debugging.
/// WARNING: This should only be called if it is ok to delete the corrupt file.
/// </summary>
/// <param name="corruptFile">
/// The string path of the corrupt file to copy.
/// </param>
/// <param name="configuration">
/// The BuildXL configuration being used to determine logs layout.
/// </param>
/// <param name="pathTable">
/// PathTable for expanding the BuildXL logs path.
/// </param>
/// <param name="loggingContext">
/// LoggingContext for verbose and warning messages.
/// </param>
/// <param name="removeFile">
/// Whether or not the corrupt file should be removed.
/// Conservatively defaults to false.
/// </param>
public static bool TryLogAndMaybeRemoveCorruptFile(string corruptFile, IConfiguration configuration, PathTable pathTable, LoggingContext loggingContext, bool removeFile = false)
{
// Build the destination path
var destFolder = configuration.Logging.EngineCacheCorruptFilesLogDirectory.ToString(pathTable);
var possibleFileName = FileUtilities.GetFileName(corruptFile);
var fileName = possibleFileName.Succeeded ? possibleFileName.Result : Path.GetFileName(corruptFile);
var destination = Path.Combine(destFolder, fileName);
Logger.Log.MovingCorruptFile(loggingContext, corruptFile, destination);
try
{
FileUtilities.CreateDirectory(destFolder);
if (removeFile)
{
FileUtilities.MoveFileAsync(corruptFile, destination).GetAwaiter().GetResult();
}
else
{
FileUtilities.CopyFileAsync(corruptFile, destination).GetAwaiter().GetResult();
}
return(true);
}
catch (Exception moveException)
{
Logger.Log.FailedToMoveCorruptFile(loggingContext, corruptFile, destination, moveException.ToStringDemystified());
}
// If moving the file to logs for debugging failed, still attempt to delete the file since it's corrupt
if (removeFile)
{
try
{
FileUtilities.DeleteFile(corruptFile, waitUntilDeletionFinished: true);
}
catch (Exception deleteException)
{
Logger.Log.FailedToDeleteCorruptFile(loggingContext, corruptFile, deleteException.ToStringDemystified());
}
}
return(false);
}
public void Test_CreateDirectory_AlreadyExists()
{
//---------------Set up test pack-------------------
string path = GetTestPath("Existing");
if (Directory.Exists(path))
{
Directory.Delete(path, true);
}
FileUtilities.CreateDirectory(path);
//---------------Assert Precondition----------------
AssertDirectoryExists(path);
//---------------Execute Test ----------------------
FileUtilities.CreateDirectory(path);
//---------------Test Result -----------------------
AssertDirectoryExists(path);
}
private bool TryPrepareWorkingDirectory(SandboxedProcessInfo info)
{
if (!Directory.Exists(info.WorkingDirectory))
{
try
{
FileUtilities.CreateDirectory(info.WorkingDirectory);
}
catch (BuildXLException e)
{
m_logger.LogError($"Failed to prepare temporary folder '{info.WorkingDirectory}': {e.ToStringDemystified()}");
return(false);
}
}
return(true);
}
private Task <SandboxedProcessResult> RunNinjaGraphBuilderAsync(AbsolutePath outputFile)
{
AbsolutePath pathToTool = Configuration.Layout.BuildEngineDirectory.Combine(Context.PathTable, m_relativePathToGraphConstructionTool);
string rootString = ProjectRoot.ToString(Context.PathTable);
AbsolutePath outputDirectory = outputFile.GetParent(Context.PathTable);
FileUtilities.CreateDirectory(outputDirectory.ToString(Context.PathTable)); // Ensure it exists
AbsolutePath argumentsFile = outputDirectory.Combine(Context.PathTable, Guid.NewGuid().ToString());
SerializeToolArguments(outputFile, argumentsFile);
// After running the tool we'd like to remove some files
void CleanUpOnResult()
{
try
{
var shouldKeepArgs = m_resolverSettings.KeepToolFiles ?? false;
if (!shouldKeepArgs)
{
FileUtilities.DeleteFile(argumentsFile.ToString(Context.PathTable));
}
}
catch (BuildXLException e)
{
Tracing.Logger.Log.CouldNotDeleteToolArgumentsFile(
Context.LoggingContext,
m_resolverSettings.Location(Context.PathTable),
argumentsFile.ToString(Context.PathTable),
e.Message);
}
}
return(FrontEndUtilities.RunSandboxedToolAsync(
Context,
pathToTool.ToString(Context.PathTable),
buildStorageDirectory: outputDirectory.ToString(Context.PathTable),
fileAccessManifest: GenerateFileAccessManifest(pathToTool.GetParent(Context.PathTable), outputFile),
arguments: I($@"""{argumentsFile.ToString(Context.PathTable)}"""),
workingDirectory: SpecFile.GetParent(Context.PathTable).ToString(Context.PathTable),
description: "Ninja graph builder",
BuildParameters.GetFactory().PopulateFromEnvironment(),
onResult: CleanUpOnResult));
}
/// <summary>
/// Serializes sandboxed process info to file.
/// </summary>
protected void SerializeSandboxedProcessInfoToFile()
{
string file = GetSandboxedProcessInfoFile();
FileUtilities.CreateDirectory(Path.GetDirectoryName(file));
try
{
using (FileStream stream = File.OpenWrite(file))
{
SandboxedProcessInfo.Serialize(stream);
}
}
catch (IOException ioException)
{
ThrowBuildXLException($"Failed to serialize sandboxed process info '{file}'", ioException);
}
}
/// <summary>
/// Serializes sandboxed process input files.
/// </summary>
protected void SerializeSandboxedProcessInputFile(string path, Action <FileStream> serialize)
{
string file = SandboxedProcessInfoFile;
FileUtilities.CreateDirectory(Path.GetDirectoryName(file));
try
{
using (FileStream stream = File.OpenWrite(path))
{
serialize(stream);
}
}
catch (IOException ioException)
{
ThrowBuildXLException($"Failed to serialize sandboxed process input '{file}'", ioException);
}
}
public void ValidateResourcesAreReportedWithSharedCompilation()
{
string embeddedFilePath = Path.Combine(SourceRoot, "dummy.txt");
FileUtilities.CreateDirectory(SourceRoot);
File.WriteAllText(embeddedFilePath, "-");
var collector = RunManagedCompilation(
useSharedCompilation: true,
out string _,
out AbsolutePath _,
additionalArgs: $"Resources='{embeddedFilePath}'");
AbsolutePath dummyPath = AbsolutePath.Create(PathTable, embeddedFilePath);
var resourceAccesses = collector.GetFileAccessPaths().Where(p => p.Equals(dummyPath));
XAssert.IsTrue(resourceAccesses.Any());
}
public void ValidateSharedCompilationWithRelativePaths()
{
// An empty manifest should be enough
var manifestPath = Path.Combine(SourceRoot, "app.manifest");
FileUtilities.CreateDirectory(SourceRoot);
File.WriteAllText(manifestPath, string.Empty);
// Let's pass the manifest as a relative path
var collector = RunManagedCompilation(
useSharedCompilation: true,
absolutePathToReference: out _,
usedTempDirectory: out _,
additionalArgs: "Win32Manifest='app.manifest'");
// Verify the access was made against the correct absolute path
Assert.True(collector.GetAllFileAccessPaths().Select(path => path.ToUpperInvariant()).Any(input => input.Contains(manifestPath.ToUpperInvariant())));
}
private void SetupEnvironment(StreamWriter writer, Process pip, string outputFile)
{
var pipEnviornment = new PipEnvironment(LoggingContext);
var environment = pipEnviornment.GetEffectiveEnvironmentVariables(PathTable, pip).ToDictionary();
writer.WriteLine($"{CommentPrefix} Environment Variables");
if (string.IsNullOrEmpty(JsonEnvironmentScript))
{
writer.WriteLine($"{CommentPrefix} Clear Existing Environment Variables");
#if PLATFORM_WIN
writer.WriteLine(@"for /f ""tokens=1* delims=="" %%a in ('set') do (");
writer.WriteLine(" set %%a=");
writer.WriteLine(")");
#else
writer.WriteLine(@"for c in $(set | cut -d '=' -f 1); do unset $c 2> /dev/null; done");
#endif
writer.WriteLine();
writer.WriteLine($"{CommentPrefix} Setting PIP Environment Variables");
foreach (var environmentVariable in environment)
{
writer.WriteLine($"{ScriptVariableExportKeyword} {SanitizeEnvironmentVariableValue(environmentVariable.Key)}={SanitizeEnvironmentVariableValue(environmentVariable.Value)}");
}
}
else
{
FileUtilities.CreateDirectory(Path.GetDirectoryName(outputFile));
using (var jsonFile = File.Create($"{outputFile}.env.json"))
using (var jsonStream = new StreamWriter(jsonFile))
using (var json = new JsonTextWriter(jsonStream))
{
json.WriteStartObject();
foreach (var environmentVariable in environment)
{
json.WritePropertyName(environmentVariable.Key);
json.WriteValue(environmentVariable.Value);
}
json.WriteEndObject();
}
writer.WriteLine($"{ExecuteScriptCommand} {JsonEnvironmentScript} {outputFile}.env.json");
}
writer.WriteLine();
}
private Task <SandboxedProcessResult> RunNinjaGraphBuilderAsync(AbsolutePath outputFile)
{
AbsolutePath outputDirectory = outputFile.GetParent(m_context.PathTable);
FileUtilities.CreateDirectory(outputDirectory.ToString(m_context.PathTable)); // Ensure it exists
AbsolutePath argumentsFile = outputDirectory.Combine(m_context.PathTable, Guid.NewGuid().ToString());
SerializeToolArguments(outputFile, argumentsFile);
// After running the tool we'd like to remove some files
void cleanUpOnResult()
{
try
{
var shouldKeepArgs = m_resolverSettings.KeepProjectGraphFile ?? false;
if (!shouldKeepArgs)
{
FileUtilities.DeleteFile(argumentsFile.ToString(m_context.PathTable));
}
}
catch (BuildXLException e)
{
Tracing.Logger.Log.CouldNotDeleteToolArgumentsFile(
m_context.LoggingContext,
m_resolverSettings.Location(m_context.PathTable),
argumentsFile.ToString(m_context.PathTable),
e.Message);
}
}
return(FrontEndUtilities.RunSandboxedToolAsync(
m_context,
m_pathToTool.ToString(m_context.PathTable),
buildStorageDirectory: outputDirectory.ToString(m_context.PathTable),
fileAccessManifest: GenerateFileAccessManifest(m_pathToTool.GetParent(m_context.PathTable), outputFile),
arguments: I($@"""{argumentsFile.ToString(m_context.PathTable)}"""),
workingDirectory: SpecFile.GetParent(m_context.PathTable).ToString(m_context.PathTable),
description: "Ninja graph builder",
RetrieveBuildParameters(),
onResult: cleanUpOnResult));
}
private void SerializeComputedGraph(GenericJavaScriptGraph <DeserializedJavaScriptProject, YarnConfiguration> graph)
{
AbsolutePath outputDirectory = m_host.GetFolderForFrontEnd(Name);
AbsolutePath outputFile = outputDirectory.Combine(m_context.PathTable, Guid.NewGuid().ToString());
// Make sure the directories are there
FileUtilities.CreateDirectory(outputDirectory.ToString(m_context.PathTable));
try
{
File.WriteAllText(outputFile.ToString(m_context.PathTable), JObject.FromObject(graph, ConstructProjectGraphSerializer(JsonSerializerSettings)).ToString());
// Graph-related files are requested to be left on disk. Let's print a message with their location.
JavaScript.Tracing.Logger.Log.GraphBuilderFilesAreNotRemoved(m_context.LoggingContext, outputFile.ToString(m_context.PathTable));
}
catch (Exception ex)
{
// Serializing the graph is done on a best-effort basis. If there is any issues with it, just log it and move on.
Tracing.Logger.Log.CannotSerializeGraphFile(m_context.LoggingContext, m_resolverSettings.Location(m_context.PathTable), outputFile.ToString(m_context.PathTable), ex.ToString());
}
}
private bool TryPrepareTemporaryFolders(SandboxedProcessInfo info)
{
Contract.Requires(info != null);
if (info.RedirectedTempFolders != null)
{
foreach (var redirection in info.RedirectedTempFolders)
{
try
{
FileUtilities.DeleteDirectoryContents(redirection.target, deleteRootDirectory: false);
FileUtilities.CreateDirectory(redirection.target);
}
catch (BuildXLException e)
{
m_logger.LogError($"Failed to prepare temporary folder '{redirection.target}': {e.ToStringDemystified()}");
return(false);
}
}
}
foreach (var tmpEnvVar in BuildParameters.DisallowedTempVariables)
{
if (info.EnvironmentVariables.ContainsKey(tmpEnvVar))
{
string tempPath = info.EnvironmentVariables[tmpEnvVar];
try
{
FileUtilities.CreateDirectory(tempPath);
}
catch (BuildXLException e)
{
m_logger.LogError($"Failed to prepare temporary folder '{tempPath}': {e.ToStringDemystified()}");
return(false);
}
}
}
return(true);
}
private async Task <bool> SaveCgManifetsFileAsync(string generatedCgManifest)
{
// Overwrite or create new cgmanifest.json file with updated nuget package and version info
try
{
string targetFilePath = Configuration.FrontEnd.GenerateCgManifestForNugets.ToString(Context.PathTable);
FileUtilities.CreateDirectory(Path.GetDirectoryName(targetFilePath));
await FileUtilities.WriteAllTextAsync(targetFilePath, generatedCgManifest, Encoding.UTF8);
FrontEndHost.Engine.RecordFrontEndFile(
Configuration.FrontEnd.GenerateCgManifestForNugets,
CGManifestResolverName);
}
catch (BuildXLException e)
{
Logger.ReportComponentGovernanceGenerationError(Context.LoggingContext, "Could not write Component Governance Manifest file to disk\n" + e.ToString());
return(false);
}
return(true);
}
private void SetupEnvironment(StreamWriter writer, Process pip, string outputFile)
{
var pipEnviornment = new PipEnvironment();
var environment = pipEnviornment.GetEffectiveEnvironmentVariables(PathTable, pip).ToDictionary();
writer.WriteLine(":::: Environment Variables");
if (string.IsNullOrEmpty(JsonEnvironmentScript))
{
writer.WriteLine(":: Clear Existing Environment Variables");
writer.WriteLine(@"for /f ""tokens=1* delims=="" %%a in ('set') do (");
writer.WriteLine(" set %%a=");
writer.WriteLine(")");
writer.WriteLine();
writer.WriteLine(":: Setting PIP Environment Variables");
foreach (var environmentVariable in environment)
{
writer.WriteLine("set {0}={1}", SanitizeEnvironmentVariableValue(environmentVariable.Key), SanitizeEnvironmentVariableValue(environmentVariable.Value));
}
}
else
{
FileUtilities.CreateDirectory(Path.GetDirectoryName(outputFile));
using (var jsonFile = File.Create($"{outputFile}.env.json"))
using (var jsonStream = new StreamWriter(jsonFile))
using (var json = new JsonTextWriter(jsonStream))
{
json.WriteStartObject();
foreach (var environmentVariable in environment)
{
json.WritePropertyName(environmentVariable.Key);
json.WriteValue(environmentVariable.Value);
}
json.WriteEndObject();
}
writer.WriteLine($"call {JsonEnvironmentScript} {outputFile}.env.json");
}
writer.WriteLine();
}
public async Task <Possible <string, Failure> > ProduceFileAsync(
CasHash hash,
string filename,
FileState fileState,
UrgencyHint urgencyHint,
Guid activityId)
{
Contract.Requires(!IsClosed);
Contract.Requires(filename != null);
using (var counter = m_counters.ProduceFileCounter())
{
using (var eventing = new ProduceFileActivity(BasicFilesystemCache.EventSource, activityId, this))
{
eventing.Start(hash, filename, fileState, urgencyHint);
if (!m_pinnedToCas.ContainsKey(hash))
{
counter.Miss();
return(eventing.StopFailure(new UnpinnedCasEntryFailure(CacheId, hash)));
}
try
{
FileUtilities.CreateDirectory(Path.GetDirectoryName(filename));
await m_cache.CopyFromCasAsync(hash, filename);
counter.FileSize(new FileInfo(filename).Length);
}
catch (Exception e)
{
counter.Fail();
return(eventing.StopFailure(new ProduceFileFailure(CacheId, hash, filename, e)));
}
return(eventing.Returns(filename));
}
}
}
public void Test_CreateDirectory_AlreadyExists_WithFiles()
{
//---------------Set up test pack-------------------
string path = GetTestPath("Existing");
if (Directory.Exists(path))
{
Directory.Delete(path, true);
}
FileUtilities.CreateDirectory(path);
string filePath = Path.Combine(path, "TestFile.txt");
System.IO.File.AppendAllText(filePath, "File Contents");
//---------------Assert Precondition----------------
AssertDirectoryExists(path);
AssertFileExists(filePath);
//---------------Execute Test ----------------------
FileUtilities.CreateDirectory(path);
//---------------Test Result -----------------------
AssertDirectoryExists(path);
AssertFileExists(filePath);
}
public void LongPathAccessControlTest()
{
var longPath = Enumerable.Range(0, NativeIOConstants.MaxDirectoryPath).Aggregate(TemporaryDirectory, (path, _) => Path.Combine(path, "dir"));
var file = Path.Combine(longPath, "fileWithWriteAccess.txt");
FileUtilities.CreateDirectory(longPath);
SafeFileHandle fileHandle;
var result = FileUtilities.TryCreateOrOpenFile(
file,
FileDesiredAccess.GenericWrite,
FileShare.Delete,
FileMode.Create,
FileFlagsAndAttributes.FileAttributeNormal,
out fileHandle);
XAssert.IsTrue(result.Succeeded);
FileUtilities.SetFileAccessControl(file, FileSystemRights.WriteAttributes, true);
XAssert.IsTrue(FileUtilities.HasWritableAccessControl(file));
//Delete the created directory
fileHandle.Close();
FileUtilities.DeleteDirectoryContents(longPath, deleteRootDirectory: true);
}
private static ServerDeploymentCacheCreated CreateServerDeployment(string destDir, AppDeployment clientApp)
{
Stopwatch st = Stopwatch.StartNew();
// Every time the server cache gets created, a file with the result of GetDeploymentHash is created, so we avoid computing this again
// The assumption is that nobody but this process modifies the server cache folder
string serverDeploymentHashFile = Path.Combine(destDir, ServerDeploymentHashFilename);
// Deletes the existing cache directory if it exists, so we avoid accumulating garbage.
if (Directory.Exists(destDir))
{
// Check if the main root process (likely bxl.exe) is in use before attempting to delete, so we avoid partially deleting files
// Not completely bullet proof (there can be a race) but it is highly unlikely the process starts to be in use right after this check
KillServer(destDir);
// Delete the deployment hash file first to make sure the deployment cache cannot be used in case the
// cleanup of the other files is interrupted.
PoisonServerDeployment(destDir);
// Remove all files regardless of files being readonly
FileUtilities.DeleteDirectoryContents(destDir, true);
}
// Perform the deployment
AppDeployment serverDeployment = AppDeployment.ReadDeploymentManifest(clientApp.BaseDirectory, AppDeployment.ServerDeploymentManifestFileName);
HashSet <string> directories = new HashSet <string>();
List <KeyValuePair <string, string> > filesToCopy = new List <KeyValuePair <string, string> >();
foreach (string path in serverDeployment.GetRelevantRelativePaths(forServerDeployment: true).Concat(new string[] { AppDeployment.ServerDeploymentManifestFileName }))
{
string targetPath = Path.Combine(destDir, path);
string sourcePath = Path.Combine(clientApp.BaseDirectory, path);
string directory = Path.GetDirectoryName(targetPath);
if (directories.Add(directory))
{
FileUtilities.CreateDirectory(directory);
}
filesToCopy.Add(new KeyValuePair <string, string>(sourcePath, targetPath));
}
// Because some deployments use virtualized vpak, using a very parallelized copy is beneficial
Parallel.ForEach(
filesToCopy,
new ParallelOptions()
{
MaxDegreeOfParallelism = 50,
},
(fileToCopy) =>
{
if (File.Exists(fileToCopy.Key))
{
File.Copy(fileToCopy.Key, fileToCopy.Value);
}
});
#if !FEATURE_CORECLR
var ngenExe = Path.Combine(System.Runtime.InteropServices.RuntimeEnvironment.GetRuntimeDirectory(), @"ngen.exe");
var destExe = Path.Combine(destDir, System.AppDomain.CurrentDomain.FriendlyName);
// queue:1 means it runs in the background
var ngenArgs = "install " + destExe + " /queue:1";
bool runNgen = File.Exists(ngenExe);
if (runNgen)
{
ProcessStartInfo startInfo = new ProcessStartInfo(ngenExe, ngenArgs);
startInfo.UseShellExecute = false;
startInfo.CreateNoWindow = true;
Process.Start(startInfo);
}
#endif
using (var file = new StreamWriter(File.OpenWrite(serverDeploymentHashFile)))
{
file.WriteLine(clientApp.TimestampBasedHash);
// This isn't actually consumed. It is only used for debugging
file.WriteLine("Debug info:");
#if !FEATURE_CORECLR
if (runNgen)
{
file.WriteLine("Ran Ngen: " + ngenExe + " " + ngenArgs);
}
#endif
file.WriteLine(clientApp.TimestampBasedHashDebug);
}
ServerDeploymentCacheCreated cacheCreated = default(ServerDeploymentCacheCreated);
cacheCreated.TimeToCreateServerCacheMilliseconds = st.ElapsedMilliseconds;
return(cacheCreated);
}
private MergeResult TryCreateHardlinkForOutput(ExpandedAbsolutePath fileOutput, int rewriteCount, string sourcePath, Process process, PipExecutionContext pipExecutionContext, HashSet <AbsolutePath> createdDirectories)
{
if (!CanMerge(fileOutput.ExpandedPath, rewriteCount, process, pipExecutionContext, out bool isDisallowed, out bool shouldDelete))
{
if (isDisallowed)
{
Tracing.Logger.Log.DisallowedDoubleWriteOnMerge(m_loggingContext, process.SemiStableHash, process.GetDescription(pipExecutionContext), fileOutput.ExpandedPath, sourcePath);
return(MergeResult.DisallowedDoubleWrite);
}
// The file cannot be merged, but no merging is allowed
return(MergeResult.Success);
}
if (shouldDelete)
{
FileUtilities.DeleteFile(fileOutput.ExpandedPath, waitUntilDeletionFinished: true);
}
else
{
// If we need to delete the target file, the directory is already created. So only do this otherwise.
// Make sure the destination directory exists before trying to create the hardlink
var containingDirectory = fileOutput.Path.GetParent(m_pathTable);
if (!createdDirectories.Contains(containingDirectory))
{
string destinationDirectory = containingDirectory.ToString(m_pathTable);
if (!FileUtilities.DirectoryExistsNoFollow(destinationDirectory))
{
FileUtilities.CreateDirectory(destinationDirectory);
}
createdDirectories.Add(containingDirectory);
}
}
var createHardlinkStatus = FileUtilities.TryCreateHardLink(fileOutput.ExpandedPath, sourcePath);
if (createHardlinkStatus != CreateHardLinkStatus.Success)
{
// Maybe somebody else raced and successfully created the hardlink since the last time we checked,
// so let's check again before failing
if (!CanMerge(fileOutput.ExpandedPath, rewriteCount, process, pipExecutionContext, out isDisallowed, out bool _))
{
if (isDisallowed)
{
Tracing.Logger.Log.DisallowedDoubleWriteOnMerge(m_loggingContext, process.SemiStableHash, process.GetDescription(pipExecutionContext), fileOutput.ExpandedPath, sourcePath);
return(MergeResult.DisallowedDoubleWrite);
}
// The file cannot be merged, but no merging is allowed
return(MergeResult.Success);
}
Tracing.Logger.Log.FailedToCreateHardlinkOnMerge(m_loggingContext, process.SemiStableHash, process.GetDescription(pipExecutionContext), fileOutput.ExpandedPath, sourcePath, createHardlinkStatus.ToString());
return(MergeResult.Failure);
}
return(MergeResult.Success);
}
private static Possible <KeyValueStoreAccessor> OpenInternal(
string storeDirectory,
int storeVersion,
bool defaultColumnKeyTracked,
IEnumerable <string> additionalColumns,
IEnumerable <string> additionalKeyTrackedColumns,
Action <Failure> failureHandler,
bool openReadOnly,
bool dropMismatchingColumns,
bool createNew)
{
KeyValueStoreAccessor accessor = null;
bool useVersioning = storeVersion != VersionConstants.IgnoreStore;
try
{
var persistedStoreVersion = -1;
if (createNew)
{
accessor?.Dispose();
if (FileUtilities.FileExistsNoFollow(storeDirectory))
{
FileUtilities.DeleteFile(storeDirectory);
}
else if (FileUtilities.DirectoryExistsNoFollow(storeDirectory))
{
FileUtilities.DeleteDirectoryContents(storeDirectory);
}
FileUtilities.CreateDirectory(storeDirectory);
if (useVersioning)
{
WriteVersionFile(storeDirectory, storeVersion);
}
persistedStoreVersion = storeVersion;
}
else
{
var possibleStoreVersion = ReadStoreVersion(storeDirectory);
if (possibleStoreVersion.Succeeded)
{
persistedStoreVersion = possibleStoreVersion.Result;
// Even if the store does not use the built in versioning, checks for an invalid store will be done to ensure a corrupt store is not opened
if (persistedStoreVersion == VersionConstants.InvalidStore)
{
return(new Failure <string>("The existing store is invalid and and may not be safe to open."));
}
// First check for invalid (corrupt) stores before incompatible store format versions
if (useVersioning && persistedStoreVersion != storeVersion)
{
return(new Failure <string>($"The existing store format version is incompatible expected format version. Existing store version: {persistedStoreVersion}, expected format version: {storeVersion}."));
}
}
else
{
return(possibleStoreVersion.Failure);
}
}
accessor = new KeyValueStoreAccessor(
storeDirectory,
persistedStoreVersion,
defaultColumnKeyTracked,
additionalColumns,
additionalKeyTrackedColumns,
failureHandler,
openReadOnly,
dropMismatchingColumns,
createNew);
}
catch (Exception ex)
{
return(new Failure <Exception>(ex));
}
return(accessor);
}
private bool TryExtractToDisk(ExtractorArgs arguments)
{
var archive = arguments.PathToFileToExtract;
var target = arguments.ExtractDirectory;
try
{
FileUtilities.DeleteDirectoryContents(target, false);
FileUtilities.CreateDirectory(target);
}
catch (BuildXLException e)
{
ErrorExtractingArchive(archive, target, e.Message);
return(false);
}
switch (arguments.ArchiveType)
{
case DownloadArchiveType.Zip:
try
{
new FastZip().ExtractZip(archive, target, null);
}
catch (ZipException e)
{
ErrorExtractingArchive(archive, target, e.Message);
return(false);
}
break;
case DownloadArchiveType.Gzip:
try
{
var targetFile = Path.Combine(target, Path.GetFileNameWithoutExtension(arguments.PathToFileToExtract));
using (var reader = new StreamReader(arguments.PathToFileToExtract))
using (var gzipStream = new GZipInputStream(reader.BaseStream))
using (var output = FileUtilities.CreateFileStream(targetFile, FileMode.Create, FileAccess.Write, FileShare.Read))
{
byte[] buffer = new byte[4096];
StreamUtils.Copy(gzipStream, output, buffer);
}
}
catch (GZipException e)
{
ErrorExtractingArchive(archive, target, e.Message);
return(false);
}
break;
case DownloadArchiveType.Tar:
try
{
using (var reader = new StreamReader(arguments.PathToFileToExtract))
using (var tar = TarArchive.CreateInputTarArchive(reader.BaseStream, nameEncoding: null))
{
tar.ExtractContents(target);
}
}
catch (TarException e)
{
ErrorExtractingArchive(archive, target, e.Message);
return(false);
}
break;
case DownloadArchiveType.Tgz:
try
{
using (var reader = new StreamReader(arguments.PathToFileToExtract))
using (var gzipStream = new GZipInputStream(reader.BaseStream))
using (var tar = TarArchive.CreateInputTarArchive(gzipStream, nameEncoding: null))
{
tar.ExtractContents(target);
}
}
catch (GZipException e)
{
ErrorExtractingArchive(archive, target, e.Message);
return(false);
}
catch (TarException e)
{
ErrorExtractingArchive(archive, target, e.Message);
return(false);
}
break;
case DownloadArchiveType.File:
Console.WriteLine("Specified download archive type is 'File'. Nothing to extract.");
return(true);
default:
throw Contract.AssertFailure($"Unexpected archive type '{arguments.ArchiveType}'");
}
try
{
if (!FileUtilities.DirectoryExistsNoFollow(target))
{
ErrorNothingExtracted(archive, target);
return(false);
}
}
catch (BuildXLException e)
{
ErrorExtractingArchive(archive, target, e.Message);
return(false);
}
return(true);
}
/// <summary>
/// Extract files to disk
/// </summary>
/// <remarks>
/// At the point of authoring (Jan 2019) the BCL does not implement tar-files or bz2.
/// https://github.com/dotnet/corefx/issues/3253 has been discussed since Sept 2015
/// Therefore we rely here on 3rd party library: https://github.com/icsharpcode/SharpZipLib
/// </remarks>
private bool TryExtractToDisk(DownloadData downloadData)
{
var archive = downloadData.DownloadedFilePath.ToString(m_context.PathTable);
var target = downloadData.ContentsFolder.Path.ToString(m_context.PathTable);
try
{
FileUtilities.DeleteDirectoryContents(target, false);
FileUtilities.CreateDirectory(target);
}
catch (BuildXLException e)
{
m_logger.ErrorExtractingArchive(m_context.LoggingContext, downloadData.Settings.ModuleName, archive, target, e.Message);
return(false);
}
switch (downloadData.Settings.ArchiveType)
{
case DownloadArchiveType.Zip:
try
{
new FastZip().ExtractZip(archive, target, null);
}
catch (ZipException e)
{
m_logger.ErrorExtractingArchive(m_context.LoggingContext, downloadData.Settings.ModuleName, archive, target, e.Message);
return(false);
}
break;
case DownloadArchiveType.Gzip:
try
{
var targetFile = Path.Combine(
target,
downloadData.DownloadedFilePath.GetName(m_context.PathTable).RemoveExtension(m_context.StringTable)
.ToString(m_context.StringTable));
using (var reader = m_context.FileSystem.OpenText(downloadData.DownloadedFilePath))
using (var gzipStream = new GZipInputStream(reader.BaseStream))
using (var output = FileUtilities.CreateFileStream(targetFile, FileMode.Create, FileAccess.Write, FileShare.Read))
{
byte[] buffer = new byte[4096];
StreamUtils.Copy(gzipStream, output, buffer);
}
}
catch (GZipException e)
{
m_logger.ErrorExtractingArchive(m_context.LoggingContext, downloadData.Settings.ModuleName, archive, target, e.Message);
return(false);
}
break;
case DownloadArchiveType.Tar:
try
{
using (var reader = m_context.FileSystem.OpenText(downloadData.DownloadedFilePath))
using (var tar = TarArchive.CreateInputTarArchive(reader.BaseStream))
{
tar.ExtractContents(target);
}
}
catch (TarException e)
{
m_logger.ErrorExtractingArchive(m_context.LoggingContext, downloadData.Settings.ModuleName, archive, target, e.Message);
return(false);
}
break;
case DownloadArchiveType.Tgz:
try
{
using (var reader = m_context.FileSystem.OpenText(downloadData.DownloadedFilePath))
using (var gzipStream = new GZipInputStream(reader.BaseStream))
using (var tar = TarArchive.CreateInputTarArchive(gzipStream))
{
tar.ExtractContents(target);
}
}
catch (GZipException e)
{
m_logger.ErrorExtractingArchive(m_context.LoggingContext, downloadData.Settings.ModuleName, archive, target, e.Message);
return(false);
}
catch (TarException e)
{
m_logger.ErrorExtractingArchive(m_context.LoggingContext, downloadData.Settings.ModuleName, archive, target, e.Message);
return(false);
}
break;
default:
throw Contract.AssertFailure("Unexpected archive type");
}
try
{
if (!FileUtilities.DirectoryExistsNoFollow(target))
{
m_logger.ErrorNothingExtracted(m_context.LoggingContext, downloadData.Settings.ModuleName, archive, target);
return(false);
}
}
catch (BuildXLException e)
{
m_logger.ErrorExtractingArchive(m_context.LoggingContext, downloadData.Settings.ModuleName, archive, target, e.Message);
return(false);
}
return(true);
}
private static Possible <KeyValueStoreAccessor> OpenInternal(
RocksDbStoreConfiguration storeConfiguration,
int storeVersion,
Action <RocksDbFailureEvent>?failureHandler,
bool createNewStore,
Action <Failure <Exception> >?invalidationHandler)
{
KeyValueStoreAccessor?accessor = null;
bool useVersioning = storeVersion != VersionConstants.IgnoreStore;
try
{
var persistedStoreVersion = -1;
if (createNewStore)
{
if (FileUtilities.FileExistsNoFollow(storeConfiguration.StoreDirectory))
{
FileUtilities.DeleteFile(storeConfiguration.StoreDirectory);
}
else if (FileUtilities.DirectoryExistsNoFollow(storeConfiguration.StoreDirectory))
{
FileUtilities.DeleteDirectoryContents(storeConfiguration.StoreDirectory);
}
FileUtilities.CreateDirectory(storeConfiguration.StoreDirectory);
if (useVersioning)
{
WriteVersionFile(storeConfiguration.StoreDirectory, storeVersion);
}
persistedStoreVersion = storeVersion;
}
else
{
var possibleStoreVersion = ReadStoreVersion(storeConfiguration.StoreDirectory);
if (possibleStoreVersion.Succeeded)
{
persistedStoreVersion = possibleStoreVersion.Result;
// Even if the store does not use the built in versioning, checks for an invalid store will be done to ensure a corrupt store is not opened
if (persistedStoreVersion == VersionConstants.InvalidStore)
{
return(new Failure <string>("The existing store is invalid and and may not be safe to open."));
}
// First check for invalid (corrupt) stores before incompatible store format versions
if (useVersioning && persistedStoreVersion != storeVersion)
{
return(new Failure <string>($"The existing store format version is incompatible expected format version. Existing store version: {persistedStoreVersion}, expected format version: {storeVersion}."));
}
}
else
{
return(possibleStoreVersion.Failure);
}
}
accessor = new KeyValueStoreAccessor(
storeConfiguration,
persistedStoreVersion,
failureHandler,
createNewStore,
invalidationHandler);
}
catch (Exception ex)
{
return(new Failure <Exception>(ex));
}
return(accessor);
}
/// <summary>
/// Attempts to downoad the file to disk.
/// </summary>
/// <returns>Returns EvaluationResult.Continue if we sucessfully downloaded and need to continue to store the incremental information, else the result will be what should be returned</returns>
private async Task <EvaluationResult> TryDownloadFileToDiskAsync(DownloadData downloadData)
{
var downloadFilePathAsString = downloadData.DownloadedFilePath.ToString(m_context.PathTable);
try
{
FileUtilities.CreateDirectory(Path.GetDirectoryName(downloadFilePathAsString));
FileUtilities.DeleteFile(downloadFilePathAsString, waitUntilDeletionFinished: true);
}
catch (BuildXLException e)
{
m_logger.ErrorPreppingForDownload(m_context.LoggingContext, downloadData.Settings.ModuleName, e.Message);
return(EvaluationResult.Error);
}
// We have to download the file.
m_logger.StartDownload(m_context.LoggingContext, downloadData.Settings.ModuleName, downloadData.Settings.Url);
var stopwatch = Stopwatch.StartNew();
try
{
using (var httpClient = new HttpClient())
{
httpClient.Timeout = TimeSpan.FromMinutes(10);
var response = await httpClient.GetAsync((Uri)downloadData.DownloadUri, m_context.CancellationToken);
response.EnsureSuccessStatusCode();
var stream = await response.Content.ReadAsStreamAsync();
using (var targetStream = new FileStream(downloadFilePathAsString, FileMode.Create, FileAccess.Write, FileShare.None))
{
await stream.CopyToAsync(targetStream);
}
m_logger.Downloaded(
m_context.LoggingContext,
downloadData.Settings.ModuleName,
downloadData.Settings.Url,
stopwatch.ElapsedMilliseconds,
new FileInfo(downloadFilePathAsString).Length);
}
}
catch (TaskCanceledException e)
{
string message = m_context.CancellationToken.IsCancellationRequested ? "Download manually canceled." : e.Message;
m_logger.DownloadFailed(m_context.LoggingContext, downloadData.Settings.ModuleName, downloadData.Settings.Url, message);
return(EvaluationResult.Canceled);
}
catch (HttpRequestException e)
{
var message = e.InnerException == null
? e.Message
: e.Message + " " + e.InnerException?.Message;
m_logger.DownloadFailed(m_context.LoggingContext, downloadData.Settings.ModuleName, downloadData.Settings.Url, message);
return(EvaluationResult.Error);
}
catch (IOException e)
{
m_logger.DownloadFailed(m_context.LoggingContext, downloadData.Settings.ModuleName, downloadData.Settings.Url, e.Message);
return(EvaluationResult.Error);
}
catch (UnauthorizedAccessException e)
{
m_logger.DownloadFailed(m_context.LoggingContext, downloadData.Settings.ModuleName, downloadData.Settings.Url, e.Message);
return(EvaluationResult.Error);
}
// Indicate we should continue to store the incremental information
return(EvaluationResult.Continue);
}
private async Task <SerializationResult> SerializeToFileInternalAsync(GraphCacheFile fileType, Action <BuildXLWriter> serializer, string overrideName)
{
// Unblock the caller
await Task.Yield();
FileUtilities.CreateDirectory(m_engineCacheLocation);
string fileName = overrideName ?? GetFileName(fileType);
string path = Path.Combine(m_engineCacheLocation, fileName);
SerializationResult serializationResult = new SerializationResult()
{
Success = false,
FileType = fileType,
FullPath = path,
};
var fileEnvelope = GetFileEnvelope(fileType);
Contract.Assume(m_correlationId is FileEnvelopeId, "EngineSerializer must be initialized with a valid correlation id");
var correlationId = (FileEnvelopeId)m_correlationId;
try
{
// We must delete the existing file in case it was hardlinked from the cache. Opening a filestream
// that truncates the existing file will fail if it is a hardlink.
FileUtilities.DeleteFile(path, tempDirectoryCleaner: m_tempDirectoryCleaner);
using (
FileStream fileStream = FileUtilities.CreateFileStream(
path,
FileMode.Create,
FileAccess.Write,
FileShare.Delete,
// Do not write the file with SequentialScan since it will be reread in the subsequent build
FileOptions.None))
{
Stopwatch sw = Stopwatch.StartNew();
fileEnvelope.WriteHeader(fileStream, correlationId);
// Write whether the file is compressed or not.
fileStream.WriteByte(m_useCompression ? (byte)1 : (byte)0);
long uncompressedLength = 0;
if (m_useCompression)
{
using (var writer = new BuildXLWriter(m_debug, new TrackedStream(new BufferedStream(new DeflateStream(fileStream, CompressionLevel.Fastest, leaveOpen: true), bufferSize: 64 << 10)), false, false))
{
// TODO: We can improve performance significantly by parallelizing the compression.
// There's no setting to do that, but given you have the entire file content upfront in a memory stream,
// it shouldn't be particularly complicated to split the memory stream into reasonably sized chunks (say 100MB)
// and compress each of them into separate MemoryStream backed DeflateStreams in separate threads.
// Then just write those out to a file. Of course you'll need to write out the position of those streams
// into the header when you write out the actual file.
serializer(writer);
uncompressedLength = writer.BaseStream.Length;
}
}
else
{
using (var writer = new BuildXLWriter(m_debug, fileStream, leaveOpen: true, logStats: false))
{
serializer(writer);
uncompressedLength = writer.BaseStream.Length;
}
}
Interlocked.Add(ref m_bytesSavedDueToCompression, uncompressedLength - fileStream.Length);
fileEnvelope.FixUpHeader(fileStream, correlationId);
Tracing.Logger.Log.SerializedFile(LoggingContext, fileName, sw.ElapsedMilliseconds);
serializationResult.Success = true;
Interlocked.Add(ref m_bytesSerialized, fileStream.Position);
}
}
catch (BuildXLException ex)
{
Tracing.Logger.Log.FailedToSerializePipGraph(LoggingContext, ex.LogEventMessage);
}
catch (IOException ex)
{
Tracing.Logger.Log.FailedToSerializePipGraph(LoggingContext, ex.Message);
}
return(serializationResult);
}
/// <summary>
/// Attempts to downoad the file to disk.
/// </summary>
private async Task <int> TryDownloadFileToDiskAsync(DownloaderArgs arguments)
{
try
{
FileUtilities.CreateDirectory(arguments.DownloadDirectory);
FileUtilities.DeleteFile(arguments.DownloadPath, retryOnFailure: true);
}
catch (BuildXLException e)
{
Console.Error.WriteLine(e.GetLogEventMessage());
return(1);
}
// We have to download the file.
Console.WriteLine($"Starting download from url '{arguments.Url}' to '{arguments.DownloadPath}'.");
var stopwatch = Stopwatch.StartNew();
try
{
using (var httpClient = new HttpClient())
{
httpClient.Timeout = TimeSpan.FromMinutes(10);
var httpRequest = new HttpRequestMessage(HttpMethod.Get, arguments.Url);
// If the download URI is pointing to a VSTS feed and we get a valid auth token, make it part of the request
// We only want to send the token over HTTPS and to a VSTS domain to avoid security issues
if (IsVSTSPackageSecureURI(arguments.Url) &&
await TryGetAuthenticationHeaderAsync(arguments.Url) is var authHeader &&
authHeader != null)
{
httpRequest.Headers.Accept.Clear();
httpRequest.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
httpRequest.Headers.Authorization = authHeader;
}
var response = await httpClient.SendAsync(httpRequest);
response.EnsureSuccessStatusCode();
var stream = await response.Content.ReadAsStreamAsync();
using (var targetStream = new FileStream(arguments.DownloadPath, FileMode.Create, FileAccess.Write, FileShare.None))
{
await stream.CopyToAsync(targetStream);
}
Console.WriteLine($"Finished download from url '{arguments.Url}' in {stopwatch.ElapsedMilliseconds}ms with {new FileInfo(arguments.DownloadPath).Length} bytes.");
}
}
catch (HttpRequestException e)
{
var message = e.InnerException == null
? e.Message
: e.Message + " " + e.InnerException?.Message;
Console.Error.WriteLine($"Failed to download from url '{arguments.Url}': {message}.");
return(1);
}
catch (Exception e) when(e is IOException || e is UnauthorizedAccessException)
{
Console.Error.WriteLine($"Failed to download from url '{arguments.Url}': {e.Message}.");
return(1);
}
if (arguments.Hash.HasValue)
{
// If the hash is given in the download setting, use the corresponding hashType (hash algorithm) to get the content hash of the downloaded file.
// We don't record the file until we know it is the correct one and will be used in this build.
var downloadedHash = await GetContentHashAsync(arguments.DownloadPath, arguments.Hash.Value.HashType);
// Validate downloaded hash if specified
if (arguments.Hash != downloadedHash)
{
Console.Error.WriteLine($"Invalid content for url '{arguments.Url}'. The content hash was expected to be: '{arguments.Hash}' but the downloaded files hash was '{downloadedHash}'. " +
"This means that the data on the server has been altered and is not trusted.");
return(1);
}
}
return(0);
}
private async Task <Possible <ProjectGraphResult> > TryComputeBuildGraphAsync(IEnumerable <AbsolutePath> msBuildSearchLocations, IEnumerable <AbsolutePath> dotnetSearchLocations, IEnumerable <AbsolutePath> parsingEntryPoints, BuildParameters.IBuildParameters buildParameters)
{
// We create a unique output file on the obj folder associated with the current front end, and using a GUID as the file name
AbsolutePath outputDirectory = m_host.GetFolderForFrontEnd(Name);
AbsolutePath outputFile = outputDirectory.Combine(m_context.PathTable, Guid.NewGuid().ToString());
// We create a unique response file that will contain the tool arguments
AbsolutePath responseFile = outputDirectory.Combine(m_context.PathTable, Guid.NewGuid().ToString());
// Make sure the directories are there
FileUtilities.CreateDirectory(outputDirectory.ToString(m_context.PathTable));
Possible <ProjectGraphWithPredictionsResult <AbsolutePath> > maybeProjectGraphResult = await ComputeBuildGraphAsync(responseFile, parsingEntryPoints, outputFile, msBuildSearchLocations, dotnetSearchLocations, buildParameters);
if (!maybeProjectGraphResult.Succeeded)
{
// A more specific error has been logged already
return(maybeProjectGraphResult.Failure);
}
var projectGraphResult = maybeProjectGraphResult.Result;
if (m_resolverSettings.KeepProjectGraphFile != true)
{
DeleteGraphBuilderRelatedFiles(outputFile, responseFile);
}
else
{
// Graph-related files are requested to be left on disk. Let's print a message with their location.
Tracing.Logger.Log.GraphBuilderFilesAreNotRemoved(m_context.LoggingContext, outputFile.ToString(m_context.PathTable), responseFile.ToString(m_context.PathTable));
}
if (!projectGraphResult.Succeeded)
{
var failure = projectGraphResult.Failure;
Tracing.Logger.Log.ProjectGraphConstructionError(m_context.LoggingContext, failure.HasLocation ? failure.Location : m_resolverSettings.Location(m_context.PathTable), failure.Message);
return(new MsBuildGraphConstructionFailure(m_resolverSettings, m_context.PathTable));
}
ProjectGraphWithPredictions <AbsolutePath> projectGraph = projectGraphResult.Result;
// The module contains all project files that are part of the graph
var projectFiles = new HashSet <AbsolutePath>();
foreach (ProjectWithPredictions <AbsolutePath> node in projectGraph.ProjectNodes)
{
projectFiles.Add(node.FullPath);
}
var moduleDescriptor = ModuleDescriptor.CreateWithUniqueId(m_context.StringTable, m_resolverSettings.ModuleName, this);
var moduleDefinition = ModuleDefinition.CreateModuleDefinitionWithImplicitReferences(
moduleDescriptor,
m_resolverSettings.RootTraversal,
m_resolverSettings.File,
projectFiles,
allowedModuleDependencies: null, // no module policies
cyclicalFriendModules: null, // no allowlist of cycles
mounts: null);
return(new ProjectGraphResult(projectGraph, moduleDefinition, projectGraphResult.PathToMsBuild, projectGraphResult.PathToDotNetExe));
}
public static void LoadandProcessPolys()
{
using (DatabaseCommands commands = new DatabaseCommands())
{
#region Get Dwgs to Process
var dwgs = commands.LoadandProcessPolys();
foreach (var dwg in dwgs)
{
DwDrawingStacks.Add(dwg);
}
#endregion
GDwgPath = commands.GetGlobalDWGPath();
var doc =
Application.DocumentManager.MdiActiveDocument;
var ed = doc.Editor;
foreach (var dwg in DwDrawingStacks)
{
string gpath = Convert.ToString(GDwgPath);
if (gpath != null)
{
string path = Path.Combine(gpath, dwg.PolylineDwgName);
if (path == null)
{
throw new ArgumentNullException(nameof(path));
}
if (!File.Exists(path))
{
DatabaseLogs.FormatLogs("File Not Found", path);
return;
}
try
{
// We'll just suffix the selected filename with "-purged"
// for the output location. This file will be overwritten
// if the command is called multiple times
//var output =
// Path.GetDirectoryName(pfnr.StringResult) + "\\" +
// Path.GetFileNameWithoutExtension(pfnr.StringResult) +
// "-purged" +
// Path.GetExtension(pfnr.StringResult);
// Assume a post-R12 drawing
using (var db = new Autodesk.AutoCAD.DatabaseServices.Database(false, true))
{
// Read the DWG file into our Database object
db.ReadDwgFile(
path,
FileOpenMode.OpenForReadAndReadShare,
false,
""
);
// No graphical changes, so we can keep the preview
// bitmap
db.RetainOriginalThumbnailBitmap = true;
// We'll store the current working database, to reset
// after the purge operation
var wdb = HostApplicationServices.WorkingDatabase;
HostApplicationServices.WorkingDatabase = db;
// Purge unused DGN linestyles from the drawing
// (returns false if nothing is erased)
ObjectIdCollection collection = GetIdsByTypeTypeValue("POLYLINE", "LWPOLYLINE", "POLYLINE2D", "POLYLINE3d");
CopyPolylinesBetweenDatabases(db, collection);
// Still need to reset the working database
HostApplicationServices.WorkingDatabase = wdb;
wdb.Save();
string date = dwg.DateStamp.Value.ToFileTime().ToString();
string output = Path.Combine(commands.GetGlobalDestinationPath(dwg.DateStamp.Value), String.Format("HOLES-{0}", date));
FileUtilities.CreateDirectory(output);
PLineToLayers.ProcessLayers(collection, wdb);
output = Path.Combine(output, dwg.PolylineDwgName);
doc.Database.SaveAs(output, DwgVersion.Current);
// if (PurgeDgnLinetypesInDb(db, ed))
// {
// // Check the version of the drawing to save back to
// var ver =
// (db.LastSavedAsVersion == DwgVersion.MC0To0
// ? DwgVersion.Current
// : db.LastSavedAsVersion
// );
// // Now we can save
// string output = null;
// db.SaveAs(output, ver);
// ed.WriteMessage(
// "\nSaved purged file to \"{0}\".",
// output
// );
// }
// // Still need to reset the working database
// HostApplicationServices.WorkingDatabase = wdb;
}
}
catch (Autodesk.AutoCAD.Runtime.Exception ex)
{
ed.WriteMessage("\nException: {0}", ex.Message);
}
}
break;
}
}
}
public override async Task <bool> InitResolverAsync(IResolverSettings resolverSettings, object workspaceResolver)
{
Contract.Requires(resolverSettings != null);
Contract.Assert(m_resolverState == State.Created);
Contract.Assert(
resolverSettings is INugetResolverSettings,
I($"Wrong type for resolver settings, expected {nameof(INugetResolverSettings)} but got {nameof(resolverSettings.GetType)}"));
Name = resolverSettings.Name;
m_resolverState = State.ResolverInitializing;
m_nugetWorkspaceResolver = workspaceResolver as WorkspaceNugetModuleResolver;
Contract.Assert(m_nugetWorkspaceResolver != null, "Workspace module resolver is expected to be of source type");
// TODO: We could do something smarter in the future and just download/generate what is needed
// Use this result to populate the dictionaries that are used for package retrieval (m_packageDirectories, m_packages and m_owningModules)
var maybePackages = await m_nugetWorkspaceResolver.GetAllKnownPackagesAsync();
if (!maybePackages.Succeeded)
{
// Error should have been reported.
return(false);
}
m_owningModules = new Dictionary <ModuleId, Package>();
foreach (var package in maybePackages.Result.Values.SelectMany(v => v))
{
m_packages[package.Id] = package;
m_owningModules[package.ModuleId] = package;
}
if (Configuration.FrontEnd.GenerateCgManifestForNugets.IsValid ||
Configuration.FrontEnd.ValidateCgManifestForNugets.IsValid)
{
var cgManfiestGenerator = new NugetCgManifestGenerator(Context);
string generatedCgManifest = cgManfiestGenerator.GenerateCgManifestForPackages(maybePackages.Result);
string existingCgManifest = "INVALID";
if (!Configuration.FrontEnd.GenerateCgManifestForNugets.IsValid &&
Configuration.FrontEnd.ValidateCgManifestForNugets.IsValid)
{
// Validation of existing cgmainfest.json results in failure due to mismatch. Should fail the build in this case.
try
{
existingCgManifest = File.ReadAllText(Configuration.FrontEnd.ValidateCgManifestForNugets.ToString(Context.PathTable));
FrontEndHost.Engine.RecordFrontEndFile(
Configuration.FrontEnd.ValidateCgManifestForNugets,
CGManifestResolverName);
}
// CgManifest FileNotFound, log error and fail build
catch (DirectoryNotFoundException e)
{
Logger.ReportComponentGovernanceValidationError(Context.LoggingContext, "Cannot read Component Governance Manifest file from disk\n" + e.ToString());
return(false);
}
catch (FileNotFoundException e)
{
Logger.ReportComponentGovernanceValidationError(Context.LoggingContext, "Cannot read Component Governance Manifest file from disk\n" + e.ToString());
return(false);
}
if (!cgManfiestGenerator.CompareForEquality(generatedCgManifest, existingCgManifest))
{
Logger.ReportComponentGovernanceValidationError(Context.LoggingContext, @"Existing Component Governance Manifest file is outdated, please generate a new one using the argument /generateCgManifestForNugets:<path>");
return(false);
}
m_resolverState = State.ResolverInitialized;
return(true);
}
// GenerateCgManifestForNugets writes a new file when the old file does not match, hence it will always be valid and does not need validation
try
{
// We are calling FrontEndHost.Engine.RecordFrontEndFile towards the end of this function because we may update this file after the read below
// Updating the file will cause a hash mismatch and the build to fail if this file is read again downstream
existingCgManifest = File.ReadAllText(Configuration.FrontEnd.GenerateCgManifestForNugets.ToString(Context.PathTable));
}
// CgManifest FileNotFound, continue to write the new file
// No operations required as the empty existingCgManifest will not match with the newly generated cgManifest
catch (DirectoryNotFoundException) { }
catch (FileNotFoundException) { }
if (!cgManfiestGenerator.CompareForEquality(generatedCgManifest, existingCgManifest))
{
if (Configuration.FrontEnd.GenerateCgManifestForNugets.IsValid)
{
// Overwrite or create new cgmanifest.json file with updated nuget package and version info
string targetFilePath = Configuration.FrontEnd.GenerateCgManifestForNugets.ToString(Context.PathTable);
try
{
FileUtilities.CreateDirectory(Path.GetDirectoryName(targetFilePath));
await FileUtilities.WriteAllTextAsync(targetFilePath, generatedCgManifest, Encoding.UTF8);
}
catch (BuildXLException e)
{
Logger.ReportComponentGovernanceGenerationError(Context.LoggingContext, "Could not write Component Governance Manifest file to disk\n" + e.ToString());
return(false);
}
}
}
FrontEndHost.Engine.RecordFrontEndFile(
Configuration.FrontEnd.GenerateCgManifestForNugets,
CGManifestResolverName);
}
m_resolverState = State.ResolverInitialized;
return(true);
}
private static ServerDeploymentCacheCreated CreateServerDeployment(string destDir, AppDeployment clientApp)
{
Stopwatch st = Stopwatch.StartNew();
// Check if the main server process is in use before attempting to delete the deployment, this way we avoid partially deleting files
// due to access permission issues. This is not completely bullet proof (there can be a race) but it is highly unlikely the
// process starts to be in use right after this check
KillServer(destDir);
// Deletes the existing cache directory if it exists, so we avoid accumulating garbage.
if (Directory.Exists(destDir))
{
// Remove all files regardless of files being readonly
FileUtilities.DeleteDirectoryContents(destDir, true);
}
// Perform the deployment
AppDeployment serverDeployment = AppDeployment.ReadDeploymentManifest(clientApp.BaseDirectory, AppDeployment.ServerDeploymentManifestFileName);
HashSet <string> directories = new HashSet <string>();
List <KeyValuePair <string, string> > filesToCopy = new List <KeyValuePair <string, string> >();
foreach (string path in serverDeployment.GetRelevantRelativePaths(forServerDeployment: true).Concat(new string[] { AppDeployment.ServerDeploymentManifestFileName }))
{
string targetPath = Path.Combine(destDir, path);
string sourcePath = Path.Combine(clientApp.BaseDirectory, path);
string directory = Path.GetDirectoryName(targetPath);
if (directories.Add(directory))
{
FileUtilities.CreateDirectory(directory);
}
filesToCopy.Add(new KeyValuePair <string, string>(sourcePath, targetPath));
}
// Because some deployments use virtualized vpak, using a very parallelized copy is beneficial
Parallel.ForEach(
filesToCopy,
new ParallelOptions()
{
MaxDegreeOfParallelism = 50,
},
(fileToCopy) =>
{
if (File.Exists(fileToCopy.Key))
{
File.Copy(fileToCopy.Key, fileToCopy.Value);
}
});
#if NET_FRAMEWORK
var ngenExe = Path.Combine(System.Runtime.InteropServices.RuntimeEnvironment.GetRuntimeDirectory(), @"ngen.exe");
var destExe = Path.Combine(destDir, System.AppDomain.CurrentDomain.FriendlyName);
// queue:1 means it runs in the background
if (File.Exists(ngenExe))
{
var ngenArgs = "install " + destExe + " /queue:1";
ProcessStartInfo startInfo = new ProcessStartInfo(ngenExe, ngenArgs);
startInfo.UseShellExecute = false;
startInfo.CreateNoWindow = true;
Process.Start(startInfo);
}
#endif
ServerDeploymentCacheCreated cacheCreated = default(ServerDeploymentCacheCreated);
cacheCreated.TimeToCreateServerCacheMilliseconds = st.ElapsedMilliseconds;
return(cacheCreated);
}
