Esempio n. 1
0
        /// <summary>
        /// Removes a corrupt file and attempts to save it to the appropriate logs directory for future debugging.
        /// WARNING: This should only be called if it is ok to delete the corrupt file.
        /// </summary>
        /// <param name="corruptFile">
        /// The string path of the corrupt file to copy.
        /// </param>
        /// <param name="configuration">
        /// The BuildXL configuration being used to determine logs layout.
        /// </param>
        /// <param name="pathTable">
        /// PathTable for expanding the BuildXL logs path.
        /// </param>
        /// <param name="loggingContext">
        /// LoggingContext for verbose and warning messages.
        /// </param>
        /// <param name="removeFile">
        /// Whether or not the corrupt file should be removed.
        /// Conservatively defaults to false.
        /// </param>
        public static bool TryLogAndMaybeRemoveCorruptFile(string corruptFile, IConfiguration configuration, PathTable pathTable, LoggingContext loggingContext, bool removeFile = false)
        {
            // Build the destination path
            var destFolder       = configuration.Logging.EngineCacheCorruptFilesLogDirectory.ToString(pathTable);
            var possibleFileName = FileUtilities.GetFileName(corruptFile);
            var fileName         = possibleFileName.Succeeded ? possibleFileName.Result : Path.GetFileName(corruptFile);

            var destination = Path.Combine(destFolder, fileName);

            Logger.Log.MovingCorruptFile(loggingContext, corruptFile, destination);

            try
            {
                FileUtilities.CreateDirectory(destFolder);

                if (removeFile)
                {
                    FileUtilities.MoveFileAsync(corruptFile, destination).GetAwaiter().GetResult();
                }
                else
                {
                    FileUtilities.CopyFileAsync(corruptFile, destination).GetAwaiter().GetResult();
                }

                return(true);
            }
            catch (Exception moveException)
            {
                Logger.Log.FailedToMoveCorruptFile(loggingContext, corruptFile, destination, moveException.ToStringDemystified());
            }

            // If moving the file to logs for debugging failed, still attempt to delete the file since it's corrupt
            if (removeFile)
            {
                try
                {
                    FileUtilities.DeleteFile(corruptFile, waitUntilDeletionFinished: true);
                }
                catch (Exception deleteException)
                {
                    Logger.Log.FailedToDeleteCorruptFile(loggingContext, corruptFile, deleteException.ToStringDemystified());
                }
            }

            return(false);
        }
Esempio n. 2
0
        public void Test_CreateDirectory_AlreadyExists()
        {
            //---------------Set up test pack-------------------
            string path = GetTestPath("Existing");

            if (Directory.Exists(path))
            {
                Directory.Delete(path, true);
            }
            FileUtilities.CreateDirectory(path);
            //---------------Assert Precondition----------------
            AssertDirectoryExists(path);
            //---------------Execute Test ----------------------
            FileUtilities.CreateDirectory(path);
            //---------------Test Result -----------------------
            AssertDirectoryExists(path);
        }
Esempio n. 3
0
        private bool TryPrepareWorkingDirectory(SandboxedProcessInfo info)
        {
            if (!Directory.Exists(info.WorkingDirectory))
            {
                try
                {
                    FileUtilities.CreateDirectory(info.WorkingDirectory);
                }
                catch (BuildXLException e)
                {
                    m_logger.LogError($"Failed to prepare temporary folder '{info.WorkingDirectory}': {e.ToStringDemystified()}");
                    return(false);
                }
            }

            return(true);
        }
Esempio n. 4
0
        private Task <SandboxedProcessResult> RunNinjaGraphBuilderAsync(AbsolutePath outputFile)
        {
            AbsolutePath pathToTool = Configuration.Layout.BuildEngineDirectory.Combine(Context.PathTable, m_relativePathToGraphConstructionTool);
            string       rootString = ProjectRoot.ToString(Context.PathTable);

            AbsolutePath outputDirectory = outputFile.GetParent(Context.PathTable);

            FileUtilities.CreateDirectory(outputDirectory.ToString(Context.PathTable)); // Ensure it exists

            AbsolutePath argumentsFile = outputDirectory.Combine(Context.PathTable, Guid.NewGuid().ToString());

            SerializeToolArguments(outputFile, argumentsFile);

            // After running the tool we'd like to remove some files
            void CleanUpOnResult()
            {
                try
                {
                    var shouldKeepArgs = m_resolverSettings.KeepToolFiles ?? false;
                    if (!shouldKeepArgs)
                    {
                        FileUtilities.DeleteFile(argumentsFile.ToString(Context.PathTable));
                    }
                }
                catch (BuildXLException e)
                {
                    Tracing.Logger.Log.CouldNotDeleteToolArgumentsFile(
                        Context.LoggingContext,
                        m_resolverSettings.Location(Context.PathTable),
                        argumentsFile.ToString(Context.PathTable),
                        e.Message);
                }
            }

            return(FrontEndUtilities.RunSandboxedToolAsync(
                       Context,
                       pathToTool.ToString(Context.PathTable),
                       buildStorageDirectory: outputDirectory.ToString(Context.PathTable),
                       fileAccessManifest: GenerateFileAccessManifest(pathToTool.GetParent(Context.PathTable), outputFile),
                       arguments: I($@"""{argumentsFile.ToString(Context.PathTable)}"""),
                       workingDirectory: SpecFile.GetParent(Context.PathTable).ToString(Context.PathTable),
                       description: "Ninja graph builder",
                       BuildParameters.GetFactory().PopulateFromEnvironment(),
                       onResult: CleanUpOnResult));
        }
Esempio n. 5
0
        /// <summary>
        /// Serializes sandboxed process info to file.
        /// </summary>
        protected void SerializeSandboxedProcessInfoToFile()
        {
            string file = GetSandboxedProcessInfoFile();

            FileUtilities.CreateDirectory(Path.GetDirectoryName(file));

            try
            {
                using (FileStream stream = File.OpenWrite(file))
                {
                    SandboxedProcessInfo.Serialize(stream);
                }
            }
            catch (IOException ioException)
            {
                ThrowBuildXLException($"Failed to serialize sandboxed process info '{file}'", ioException);
            }
        }
Esempio n. 6
0
        /// <summary>
        /// Serializes sandboxed process input files.
        /// </summary>
        protected void SerializeSandboxedProcessInputFile(string path, Action <FileStream> serialize)
        {
            string file = SandboxedProcessInfoFile;

            FileUtilities.CreateDirectory(Path.GetDirectoryName(file));

            try
            {
                using (FileStream stream = File.OpenWrite(path))
                {
                    serialize(stream);
                }
            }
            catch (IOException ioException)
            {
                ThrowBuildXLException($"Failed to serialize sandboxed process input '{file}'", ioException);
            }
        }
Esempio n. 7
0
        public void ValidateResourcesAreReportedWithSharedCompilation()
        {
            string embeddedFilePath = Path.Combine(SourceRoot, "dummy.txt");

            FileUtilities.CreateDirectory(SourceRoot);
            File.WriteAllText(embeddedFilePath, "-");

            var collector = RunManagedCompilation(
                useSharedCompilation: true,
                out string _,
                out AbsolutePath _,
                additionalArgs: $"Resources='{embeddedFilePath}'");

            AbsolutePath dummyPath        = AbsolutePath.Create(PathTable, embeddedFilePath);
            var          resourceAccesses = collector.GetFileAccessPaths().Where(p => p.Equals(dummyPath));

            XAssert.IsTrue(resourceAccesses.Any());
        }
Esempio n. 8
0
        public void ValidateSharedCompilationWithRelativePaths()
        {
            // An empty manifest should be enough
            var manifestPath = Path.Combine(SourceRoot, "app.manifest");

            FileUtilities.CreateDirectory(SourceRoot);
            File.WriteAllText(manifestPath, string.Empty);

            // Let's pass the manifest as a relative path
            var collector = RunManagedCompilation(
                useSharedCompilation: true,
                absolutePathToReference: out _,
                usedTempDirectory: out _,
                additionalArgs: "Win32Manifest='app.manifest'");

            // Verify the access was made against the correct absolute path
            Assert.True(collector.GetAllFileAccessPaths().Select(path => path.ToUpperInvariant()).Any(input => input.Contains(manifestPath.ToUpperInvariant())));
        }
        private void SetupEnvironment(StreamWriter writer, Process pip, string outputFile)
        {
            var pipEnviornment = new PipEnvironment(LoggingContext);
            var environment    = pipEnviornment.GetEffectiveEnvironmentVariables(PathTable, pip).ToDictionary();

            writer.WriteLine($"{CommentPrefix} Environment Variables");
            if (string.IsNullOrEmpty(JsonEnvironmentScript))
            {
                writer.WriteLine($"{CommentPrefix} Clear Existing Environment Variables");

#if PLATFORM_WIN
                writer.WriteLine(@"for /f ""tokens=1* delims=="" %%a in ('set') do (");
                writer.WriteLine("    set %%a=");
                writer.WriteLine(")");
#else
                writer.WriteLine(@"for c in $(set | cut -d '=' -f 1); do unset $c 2> /dev/null; done");
#endif
                writer.WriteLine();

                writer.WriteLine($"{CommentPrefix} Setting PIP Environment Variables");
                foreach (var environmentVariable in environment)
                {
                    writer.WriteLine($"{ScriptVariableExportKeyword} {SanitizeEnvironmentVariableValue(environmentVariable.Key)}={SanitizeEnvironmentVariableValue(environmentVariable.Value)}");
                }
            }
            else
            {
                FileUtilities.CreateDirectory(Path.GetDirectoryName(outputFile));
                using (var jsonFile = File.Create($"{outputFile}.env.json"))
                    using (var jsonStream = new StreamWriter(jsonFile))
                        using (var json = new JsonTextWriter(jsonStream))
                        {
                            json.WriteStartObject();
                            foreach (var environmentVariable in environment)
                            {
                                json.WritePropertyName(environmentVariable.Key);
                                json.WriteValue(environmentVariable.Value);
                            }
                            json.WriteEndObject();
                        }
                writer.WriteLine($"{ExecuteScriptCommand} {JsonEnvironmentScript} {outputFile}.env.json");
            }
            writer.WriteLine();
        }
Esempio n. 10
0
        private Task <SandboxedProcessResult> RunNinjaGraphBuilderAsync(AbsolutePath outputFile)
        {
            AbsolutePath outputDirectory = outputFile.GetParent(m_context.PathTable);

            FileUtilities.CreateDirectory(outputDirectory.ToString(m_context.PathTable)); // Ensure it exists

            AbsolutePath argumentsFile = outputDirectory.Combine(m_context.PathTable, Guid.NewGuid().ToString());

            SerializeToolArguments(outputFile, argumentsFile);

            // After running the tool we'd like to remove some files
            void cleanUpOnResult()
            {
                try
                {
                    var shouldKeepArgs = m_resolverSettings.KeepProjectGraphFile ?? false;
                    if (!shouldKeepArgs)
                    {
                        FileUtilities.DeleteFile(argumentsFile.ToString(m_context.PathTable));
                    }
                }
                catch (BuildXLException e)
                {
                    Tracing.Logger.Log.CouldNotDeleteToolArgumentsFile(
                        m_context.LoggingContext,
                        m_resolverSettings.Location(m_context.PathTable),
                        argumentsFile.ToString(m_context.PathTable),
                        e.Message);
                }
            }

            return(FrontEndUtilities.RunSandboxedToolAsync(
                       m_context,
                       m_pathToTool.ToString(m_context.PathTable),
                       buildStorageDirectory: outputDirectory.ToString(m_context.PathTable),
                       fileAccessManifest: GenerateFileAccessManifest(m_pathToTool.GetParent(m_context.PathTable), outputFile),
                       arguments: I($@"""{argumentsFile.ToString(m_context.PathTable)}"""),
                       workingDirectory: SpecFile.GetParent(m_context.PathTable).ToString(m_context.PathTable),
                       description: "Ninja graph builder",
                       RetrieveBuildParameters(),
                       onResult: cleanUpOnResult));
        }
Esempio n. 11
0
        private void SerializeComputedGraph(GenericJavaScriptGraph <DeserializedJavaScriptProject, YarnConfiguration> graph)
        {
            AbsolutePath outputDirectory = m_host.GetFolderForFrontEnd(Name);
            AbsolutePath outputFile      = outputDirectory.Combine(m_context.PathTable, Guid.NewGuid().ToString());

            // Make sure the directories are there
            FileUtilities.CreateDirectory(outputDirectory.ToString(m_context.PathTable));

            try
            {
                File.WriteAllText(outputFile.ToString(m_context.PathTable), JObject.FromObject(graph, ConstructProjectGraphSerializer(JsonSerializerSettings)).ToString());
                // Graph-related files are requested to be left on disk. Let's print a message with their location.
                JavaScript.Tracing.Logger.Log.GraphBuilderFilesAreNotRemoved(m_context.LoggingContext, outputFile.ToString(m_context.PathTable));
            }
            catch (Exception ex)
            {
                // Serializing the graph is done on a best-effort basis. If there is any issues with it, just log it and move on.
                Tracing.Logger.Log.CannotSerializeGraphFile(m_context.LoggingContext, m_resolverSettings.Location(m_context.PathTable), outputFile.ToString(m_context.PathTable), ex.ToString());
            }
        }
Esempio n. 12
0
        private bool TryPrepareTemporaryFolders(SandboxedProcessInfo info)
        {
            Contract.Requires(info != null);

            if (info.RedirectedTempFolders != null)
            {
                foreach (var redirection in info.RedirectedTempFolders)
                {
                    try
                    {
                        FileUtilities.DeleteDirectoryContents(redirection.target, deleteRootDirectory: false);
                        FileUtilities.CreateDirectory(redirection.target);
                    }
                    catch (BuildXLException e)
                    {
                        m_logger.LogError($"Failed to prepare temporary folder '{redirection.target}': {e.ToStringDemystified()}");
                        return(false);
                    }
                }
            }

            foreach (var tmpEnvVar in BuildParameters.DisallowedTempVariables)
            {
                if (info.EnvironmentVariables.ContainsKey(tmpEnvVar))
                {
                    string tempPath = info.EnvironmentVariables[tmpEnvVar];

                    try
                    {
                        FileUtilities.CreateDirectory(tempPath);
                    }
                    catch (BuildXLException e)
                    {
                        m_logger.LogError($"Failed to prepare temporary folder '{tempPath}': {e.ToStringDemystified()}");
                        return(false);
                    }
                }
            }

            return(true);
        }
Esempio n. 13
0
        private async Task <bool> SaveCgManifetsFileAsync(string generatedCgManifest)
        {
            // Overwrite or create new cgmanifest.json file with updated nuget package and version info
            try
            {
                string targetFilePath = Configuration.FrontEnd.GenerateCgManifestForNugets.ToString(Context.PathTable);
                FileUtilities.CreateDirectory(Path.GetDirectoryName(targetFilePath));
                await FileUtilities.WriteAllTextAsync(targetFilePath, generatedCgManifest, Encoding.UTF8);

                FrontEndHost.Engine.RecordFrontEndFile(
                    Configuration.FrontEnd.GenerateCgManifestForNugets,
                    CGManifestResolverName);
            }
            catch (BuildXLException e)
            {
                Logger.ReportComponentGovernanceGenerationError(Context.LoggingContext, "Could not write Component Governance Manifest file to disk\n" + e.ToString());
                return(false);
            }

            return(true);
        }
Esempio n. 14
0
        private void SetupEnvironment(StreamWriter writer, Process pip, string outputFile)
        {
            var pipEnviornment = new PipEnvironment();
            var environment    = pipEnviornment.GetEffectiveEnvironmentVariables(PathTable, pip).ToDictionary();

            writer.WriteLine(":::: Environment Variables");
            if (string.IsNullOrEmpty(JsonEnvironmentScript))
            {
                writer.WriteLine(":: Clear Existing Environment Variables");
                writer.WriteLine(@"for /f ""tokens=1* delims=="" %%a in ('set') do (");
                writer.WriteLine("    set %%a=");
                writer.WriteLine(")");
                writer.WriteLine();

                writer.WriteLine(":: Setting PIP Environment Variables");
                foreach (var environmentVariable in environment)
                {
                    writer.WriteLine("set {0}={1}", SanitizeEnvironmentVariableValue(environmentVariable.Key), SanitizeEnvironmentVariableValue(environmentVariable.Value));
                }
            }
            else
            {
                FileUtilities.CreateDirectory(Path.GetDirectoryName(outputFile));
                using (var jsonFile = File.Create($"{outputFile}.env.json"))
                    using (var jsonStream = new StreamWriter(jsonFile))
                        using (var json = new JsonTextWriter(jsonStream))
                        {
                            json.WriteStartObject();
                            foreach (var environmentVariable in environment)
                            {
                                json.WritePropertyName(environmentVariable.Key);
                                json.WriteValue(environmentVariable.Value);
                            }
                            json.WriteEndObject();
                        }
                writer.WriteLine($"call {JsonEnvironmentScript} {outputFile}.env.json");
            }
            writer.WriteLine();
        }
Esempio n. 15
0
        public async Task <Possible <string, Failure> > ProduceFileAsync(
            CasHash hash,
            string filename,
            FileState fileState,
            UrgencyHint urgencyHint,
            Guid activityId)
        {
            Contract.Requires(!IsClosed);
            Contract.Requires(filename != null);

            using (var counter = m_counters.ProduceFileCounter())
            {
                using (var eventing = new ProduceFileActivity(BasicFilesystemCache.EventSource, activityId, this))
                {
                    eventing.Start(hash, filename, fileState, urgencyHint);

                    if (!m_pinnedToCas.ContainsKey(hash))
                    {
                        counter.Miss();
                        return(eventing.StopFailure(new UnpinnedCasEntryFailure(CacheId, hash)));
                    }

                    try
                    {
                        FileUtilities.CreateDirectory(Path.GetDirectoryName(filename));
                        await m_cache.CopyFromCasAsync(hash, filename);

                        counter.FileSize(new FileInfo(filename).Length);
                    }
                    catch (Exception e)
                    {
                        counter.Fail();
                        return(eventing.StopFailure(new ProduceFileFailure(CacheId, hash, filename, e)));
                    }

                    return(eventing.Returns(filename));
                }
            }
        }
Esempio n. 16
0
        public void Test_CreateDirectory_AlreadyExists_WithFiles()
        {
            //---------------Set up test pack-------------------
            string path = GetTestPath("Existing");

            if (Directory.Exists(path))
            {
                Directory.Delete(path, true);
            }
            FileUtilities.CreateDirectory(path);
            string filePath = Path.Combine(path, "TestFile.txt");

            System.IO.File.AppendAllText(filePath, "File Contents");
            //---------------Assert Precondition----------------
            AssertDirectoryExists(path);
            AssertFileExists(filePath);
            //---------------Execute Test ----------------------
            FileUtilities.CreateDirectory(path);
            //---------------Test Result -----------------------
            AssertDirectoryExists(path);
            AssertFileExists(filePath);
        }
        public void LongPathAccessControlTest()
        {
            var longPath = Enumerable.Range(0, NativeIOConstants.MaxDirectoryPath).Aggregate(TemporaryDirectory, (path, _) => Path.Combine(path, "dir"));
            var file     = Path.Combine(longPath, "fileWithWriteAccess.txt");

            FileUtilities.CreateDirectory(longPath);
            SafeFileHandle fileHandle;
            var            result = FileUtilities.TryCreateOrOpenFile(
                file,
                FileDesiredAccess.GenericWrite,
                FileShare.Delete,
                FileMode.Create,
                FileFlagsAndAttributes.FileAttributeNormal,
                out fileHandle);

            XAssert.IsTrue(result.Succeeded);

            FileUtilities.SetFileAccessControl(file, FileSystemRights.WriteAttributes, true);
            XAssert.IsTrue(FileUtilities.HasWritableAccessControl(file));

            //Delete the created directory
            fileHandle.Close();
            FileUtilities.DeleteDirectoryContents(longPath, deleteRootDirectory: true);
        }
Esempio n. 18
0
        private static ServerDeploymentCacheCreated CreateServerDeployment(string destDir, AppDeployment clientApp)
        {
            Stopwatch st = Stopwatch.StartNew();

            // Every time the server cache gets created, a file with the result of GetDeploymentHash is created, so we avoid computing this again
            // The assumption is that nobody but this process modifies the server cache folder
            string serverDeploymentHashFile = Path.Combine(destDir, ServerDeploymentHashFilename);

            // Deletes the existing cache directory if it exists, so we avoid accumulating garbage.
            if (Directory.Exists(destDir))
            {
                // Check if the main root process (likely bxl.exe) is in use before attempting to delete, so we avoid partially deleting files
                // Not completely bullet proof (there can be a race) but it is highly unlikely the process starts to be in use right after this check
                KillServer(destDir);

                // Delete the deployment hash file first to make sure the deployment cache cannot be used in case the
                // cleanup of the other files is interrupted.
                PoisonServerDeployment(destDir);

                // Remove all files regardless of files being readonly
                FileUtilities.DeleteDirectoryContents(destDir, true);
            }

            // Perform the deployment
            AppDeployment    serverDeployment = AppDeployment.ReadDeploymentManifest(clientApp.BaseDirectory, AppDeployment.ServerDeploymentManifestFileName);
            HashSet <string> directories      = new HashSet <string>();
            List <KeyValuePair <string, string> > filesToCopy = new List <KeyValuePair <string, string> >();

            foreach (string path in serverDeployment.GetRelevantRelativePaths(forServerDeployment: true).Concat(new string[] { AppDeployment.ServerDeploymentManifestFileName }))
            {
                string targetPath = Path.Combine(destDir, path);
                string sourcePath = Path.Combine(clientApp.BaseDirectory, path);
                string directory  = Path.GetDirectoryName(targetPath);
                if (directories.Add(directory))
                {
                    FileUtilities.CreateDirectory(directory);
                }

                filesToCopy.Add(new KeyValuePair <string, string>(sourcePath, targetPath));
            }

            // Because some deployments use virtualized vpak, using a very parallelized copy is beneficial
            Parallel.ForEach(
                filesToCopy,
                new ParallelOptions()
            {
                MaxDegreeOfParallelism = 50,
            },
                (fileToCopy) =>
            {
                if (File.Exists(fileToCopy.Key))
                {
                    File.Copy(fileToCopy.Key, fileToCopy.Value);
                }
            });

#if !FEATURE_CORECLR
            var ngenExe = Path.Combine(System.Runtime.InteropServices.RuntimeEnvironment.GetRuntimeDirectory(), @"ngen.exe");
            var destExe = Path.Combine(destDir, System.AppDomain.CurrentDomain.FriendlyName);

            // queue:1 means it runs in the background
            var  ngenArgs = "install " + destExe + " /queue:1";
            bool runNgen  = File.Exists(ngenExe);
            if (runNgen)
            {
                ProcessStartInfo startInfo = new ProcessStartInfo(ngenExe, ngenArgs);
                startInfo.UseShellExecute = false;
                startInfo.CreateNoWindow  = true;
                Process.Start(startInfo);
            }
#endif
            using (var file = new StreamWriter(File.OpenWrite(serverDeploymentHashFile)))
            {
                file.WriteLine(clientApp.TimestampBasedHash);

                // This isn't actually consumed. It is only used for debugging
                file.WriteLine("Debug info:");
#if !FEATURE_CORECLR
                if (runNgen)
                {
                    file.WriteLine("Ran Ngen: " + ngenExe + " " + ngenArgs);
                }
#endif
                file.WriteLine(clientApp.TimestampBasedHashDebug);
            }

            ServerDeploymentCacheCreated cacheCreated = default(ServerDeploymentCacheCreated);
            cacheCreated.TimeToCreateServerCacheMilliseconds = st.ElapsedMilliseconds;

            return(cacheCreated);
        }
Esempio n. 19
0
        private MergeResult TryCreateHardlinkForOutput(ExpandedAbsolutePath fileOutput, int rewriteCount, string sourcePath, Process process, PipExecutionContext pipExecutionContext, HashSet <AbsolutePath> createdDirectories)
        {
            if (!CanMerge(fileOutput.ExpandedPath, rewriteCount, process, pipExecutionContext, out bool isDisallowed, out bool shouldDelete))
            {
                if (isDisallowed)
                {
                    Tracing.Logger.Log.DisallowedDoubleWriteOnMerge(m_loggingContext, process.SemiStableHash, process.GetDescription(pipExecutionContext), fileOutput.ExpandedPath, sourcePath);
                    return(MergeResult.DisallowedDoubleWrite);
                }

                // The file cannot be merged, but no merging is allowed
                return(MergeResult.Success);
            }

            if (shouldDelete)
            {
                FileUtilities.DeleteFile(fileOutput.ExpandedPath, waitUntilDeletionFinished: true);
            }
            else
            {
                // If we need to delete the target file, the directory is already created. So only do this otherwise.

                // Make sure the destination directory exists before trying to create the hardlink
                var containingDirectory = fileOutput.Path.GetParent(m_pathTable);
                if (!createdDirectories.Contains(containingDirectory))
                {
                    string destinationDirectory = containingDirectory.ToString(m_pathTable);

                    if (!FileUtilities.DirectoryExistsNoFollow(destinationDirectory))
                    {
                        FileUtilities.CreateDirectory(destinationDirectory);
                    }

                    createdDirectories.Add(containingDirectory);
                }
            }

            var createHardlinkStatus = FileUtilities.TryCreateHardLink(fileOutput.ExpandedPath, sourcePath);

            if (createHardlinkStatus != CreateHardLinkStatus.Success)
            {
                // Maybe somebody else raced and successfully created the hardlink since the last time we checked,
                // so let's check again before failing
                if (!CanMerge(fileOutput.ExpandedPath, rewriteCount, process, pipExecutionContext, out isDisallowed, out bool _))
                {
                    if (isDisallowed)
                    {
                        Tracing.Logger.Log.DisallowedDoubleWriteOnMerge(m_loggingContext, process.SemiStableHash, process.GetDescription(pipExecutionContext), fileOutput.ExpandedPath, sourcePath);
                        return(MergeResult.DisallowedDoubleWrite);
                    }

                    // The file cannot be merged, but no merging is allowed
                    return(MergeResult.Success);
                }

                Tracing.Logger.Log.FailedToCreateHardlinkOnMerge(m_loggingContext, process.SemiStableHash, process.GetDescription(pipExecutionContext), fileOutput.ExpandedPath, sourcePath, createHardlinkStatus.ToString());
                return(MergeResult.Failure);
            }

            return(MergeResult.Success);
        }
Esempio n. 20
0
        private static Possible <KeyValueStoreAccessor> OpenInternal(
            string storeDirectory,
            int storeVersion,
            bool defaultColumnKeyTracked,
            IEnumerable <string> additionalColumns,
            IEnumerable <string> additionalKeyTrackedColumns,
            Action <Failure> failureHandler,
            bool openReadOnly,
            bool dropMismatchingColumns,
            bool createNew)
        {
            KeyValueStoreAccessor accessor = null;
            bool useVersioning             = storeVersion != VersionConstants.IgnoreStore;

            try
            {
                var persistedStoreVersion = -1;
                if (createNew)
                {
                    accessor?.Dispose();

                    if (FileUtilities.FileExistsNoFollow(storeDirectory))
                    {
                        FileUtilities.DeleteFile(storeDirectory);
                    }
                    else if (FileUtilities.DirectoryExistsNoFollow(storeDirectory))
                    {
                        FileUtilities.DeleteDirectoryContents(storeDirectory);
                    }

                    FileUtilities.CreateDirectory(storeDirectory);

                    if (useVersioning)
                    {
                        WriteVersionFile(storeDirectory, storeVersion);
                    }

                    persistedStoreVersion = storeVersion;
                }
                else
                {
                    var possibleStoreVersion = ReadStoreVersion(storeDirectory);
                    if (possibleStoreVersion.Succeeded)
                    {
                        persistedStoreVersion = possibleStoreVersion.Result;
                        // Even if the store does not use the built in versioning, checks for an invalid store will be done to ensure a corrupt store is not opened
                        if (persistedStoreVersion == VersionConstants.InvalidStore)
                        {
                            return(new Failure <string>("The existing store is invalid and and may not be safe to open."));
                        }

                        // First check for invalid (corrupt) stores before incompatible store format versions
                        if (useVersioning && persistedStoreVersion != storeVersion)
                        {
                            return(new Failure <string>($"The existing store format version is incompatible expected format version. Existing store version: {persistedStoreVersion}, expected format version: {storeVersion}."));
                        }
                    }
                    else
                    {
                        return(possibleStoreVersion.Failure);
                    }
                }

                accessor = new KeyValueStoreAccessor(
                    storeDirectory,
                    persistedStoreVersion,
                    defaultColumnKeyTracked,
                    additionalColumns,
                    additionalKeyTrackedColumns,
                    failureHandler,
                    openReadOnly,
                    dropMismatchingColumns,
                    createNew);
            }
            catch (Exception ex)
            {
                return(new Failure <Exception>(ex));
            }

            return(accessor);
        }
Esempio n. 21
0
        private bool TryExtractToDisk(ExtractorArgs arguments)
        {
            var archive = arguments.PathToFileToExtract;
            var target  = arguments.ExtractDirectory;

            try
            {
                FileUtilities.DeleteDirectoryContents(target, false);
                FileUtilities.CreateDirectory(target);
            }
            catch (BuildXLException e)
            {
                ErrorExtractingArchive(archive, target, e.Message);
                return(false);
            }

            switch (arguments.ArchiveType)
            {
            case DownloadArchiveType.Zip:
                try
                {
                    new FastZip().ExtractZip(archive, target, null);
                }
                catch (ZipException e)
                {
                    ErrorExtractingArchive(archive, target, e.Message);
                    return(false);
                }

                break;

            case DownloadArchiveType.Gzip:
                try
                {
                    var targetFile = Path.Combine(target, Path.GetFileNameWithoutExtension(arguments.PathToFileToExtract));

                    using (var reader = new StreamReader(arguments.PathToFileToExtract))
                        using (var gzipStream = new GZipInputStream(reader.BaseStream))
                            using (var output = FileUtilities.CreateFileStream(targetFile, FileMode.Create, FileAccess.Write, FileShare.Read))
                            {
                                byte[] buffer = new byte[4096];
                                StreamUtils.Copy(gzipStream, output, buffer);
                            }
                }
                catch (GZipException e)
                {
                    ErrorExtractingArchive(archive, target, e.Message);
                    return(false);
                }

                break;

            case DownloadArchiveType.Tar:
                try
                {
                    using (var reader = new StreamReader(arguments.PathToFileToExtract))
                        using (var tar = TarArchive.CreateInputTarArchive(reader.BaseStream, nameEncoding: null))
                        {
                            tar.ExtractContents(target);
                        }
                }
                catch (TarException e)
                {
                    ErrorExtractingArchive(archive, target, e.Message);
                    return(false);
                }

                break;

            case DownloadArchiveType.Tgz:
                try
                {
                    using (var reader = new StreamReader(arguments.PathToFileToExtract))
                        using (var gzipStream = new GZipInputStream(reader.BaseStream))
                            using (var tar = TarArchive.CreateInputTarArchive(gzipStream, nameEncoding: null))
                            {
                                tar.ExtractContents(target);
                            }
                }
                catch (GZipException e)
                {
                    ErrorExtractingArchive(archive, target, e.Message);
                    return(false);
                }
                catch (TarException e)
                {
                    ErrorExtractingArchive(archive, target, e.Message);
                    return(false);
                }

                break;

            case DownloadArchiveType.File:
                Console.WriteLine("Specified download archive type is 'File'. Nothing to extract.");
                return(true);

            default:
                throw Contract.AssertFailure($"Unexpected archive type '{arguments.ArchiveType}'");
            }

            try
            {
                if (!FileUtilities.DirectoryExistsNoFollow(target))
                {
                    ErrorNothingExtracted(archive, target);
                    return(false);
                }
            }
            catch (BuildXLException e)
            {
                ErrorExtractingArchive(archive, target, e.Message);
                return(false);
            }

            return(true);
        }
Esempio n. 22
0
        /// <summary>
        /// Extract files to disk
        /// </summary>
        /// <remarks>
        /// At the point of authoring (Jan 2019) the BCL does not implement tar-files or bz2.
        /// https://github.com/dotnet/corefx/issues/3253 has been discussed since Sept 2015
        /// Therefore we rely here on 3rd party library: https://github.com/icsharpcode/SharpZipLib
        /// </remarks>
        private bool TryExtractToDisk(DownloadData downloadData)
        {
            var archive = downloadData.DownloadedFilePath.ToString(m_context.PathTable);
            var target  = downloadData.ContentsFolder.Path.ToString(m_context.PathTable);

            try
            {
                FileUtilities.DeleteDirectoryContents(target, false);
                FileUtilities.CreateDirectory(target);
            }
            catch (BuildXLException e)
            {
                m_logger.ErrorExtractingArchive(m_context.LoggingContext, downloadData.Settings.ModuleName, archive, target, e.Message);
                return(false);
            }

            switch (downloadData.Settings.ArchiveType)
            {
            case DownloadArchiveType.Zip:
                try
                {
                    new FastZip().ExtractZip(archive, target, null);
                }
                catch (ZipException e)
                {
                    m_logger.ErrorExtractingArchive(m_context.LoggingContext, downloadData.Settings.ModuleName, archive, target, e.Message);
                    return(false);
                }

                break;

            case DownloadArchiveType.Gzip:
                try
                {
                    var targetFile = Path.Combine(
                        target,
                        downloadData.DownloadedFilePath.GetName(m_context.PathTable).RemoveExtension(m_context.StringTable)
                        .ToString(m_context.StringTable));

                    using (var reader = m_context.FileSystem.OpenText(downloadData.DownloadedFilePath))
                        using (var gzipStream = new GZipInputStream(reader.BaseStream))
                            using (var output = FileUtilities.CreateFileStream(targetFile, FileMode.Create, FileAccess.Write, FileShare.Read))
                            {
                                byte[] buffer = new byte[4096];
                                StreamUtils.Copy(gzipStream, output, buffer);
                            }
                }
                catch (GZipException e)
                {
                    m_logger.ErrorExtractingArchive(m_context.LoggingContext, downloadData.Settings.ModuleName, archive, target, e.Message);
                    return(false);
                }

                break;

            case DownloadArchiveType.Tar:
                try
                {
                    using (var reader = m_context.FileSystem.OpenText(downloadData.DownloadedFilePath))
                        using (var tar = TarArchive.CreateInputTarArchive(reader.BaseStream))
                        {
                            tar.ExtractContents(target);
                        }
                }
                catch (TarException e)
                {
                    m_logger.ErrorExtractingArchive(m_context.LoggingContext, downloadData.Settings.ModuleName, archive, target, e.Message);
                    return(false);
                }

                break;

            case DownloadArchiveType.Tgz:
                try
                {
                    using (var reader = m_context.FileSystem.OpenText(downloadData.DownloadedFilePath))
                        using (var gzipStream = new GZipInputStream(reader.BaseStream))
                            using (var tar = TarArchive.CreateInputTarArchive(gzipStream))
                            {
                                tar.ExtractContents(target);
                            }
                }
                catch (GZipException e)
                {
                    m_logger.ErrorExtractingArchive(m_context.LoggingContext, downloadData.Settings.ModuleName, archive, target, e.Message);
                    return(false);
                }
                catch (TarException e)
                {
                    m_logger.ErrorExtractingArchive(m_context.LoggingContext, downloadData.Settings.ModuleName, archive, target, e.Message);
                    return(false);
                }

                break;

            default:
                throw Contract.AssertFailure("Unexpected archive type");
            }

            try
            {
                if (!FileUtilities.DirectoryExistsNoFollow(target))
                {
                    m_logger.ErrorNothingExtracted(m_context.LoggingContext, downloadData.Settings.ModuleName, archive, target);
                    return(false);
                }
            }
            catch (BuildXLException e)
            {
                m_logger.ErrorExtractingArchive(m_context.LoggingContext, downloadData.Settings.ModuleName, archive, target, e.Message);
                return(false);
            }

            return(true);
        }
Esempio n. 23
0
        private static Possible <KeyValueStoreAccessor> OpenInternal(
            RocksDbStoreConfiguration storeConfiguration,
            int storeVersion,
            Action <RocksDbFailureEvent>?failureHandler,
            bool createNewStore,
            Action <Failure <Exception> >?invalidationHandler)
        {
            KeyValueStoreAccessor?accessor = null;
            bool useVersioning             = storeVersion != VersionConstants.IgnoreStore;

            try
            {
                var persistedStoreVersion = -1;
                if (createNewStore)
                {
                    if (FileUtilities.FileExistsNoFollow(storeConfiguration.StoreDirectory))
                    {
                        FileUtilities.DeleteFile(storeConfiguration.StoreDirectory);
                    }
                    else if (FileUtilities.DirectoryExistsNoFollow(storeConfiguration.StoreDirectory))
                    {
                        FileUtilities.DeleteDirectoryContents(storeConfiguration.StoreDirectory);
                    }

                    FileUtilities.CreateDirectory(storeConfiguration.StoreDirectory);

                    if (useVersioning)
                    {
                        WriteVersionFile(storeConfiguration.StoreDirectory, storeVersion);
                    }

                    persistedStoreVersion = storeVersion;
                }
                else
                {
                    var possibleStoreVersion = ReadStoreVersion(storeConfiguration.StoreDirectory);
                    if (possibleStoreVersion.Succeeded)
                    {
                        persistedStoreVersion = possibleStoreVersion.Result;
                        // Even if the store does not use the built in versioning, checks for an invalid store will be done to ensure a corrupt store is not opened
                        if (persistedStoreVersion == VersionConstants.InvalidStore)
                        {
                            return(new Failure <string>("The existing store is invalid and and may not be safe to open."));
                        }

                        // First check for invalid (corrupt) stores before incompatible store format versions
                        if (useVersioning && persistedStoreVersion != storeVersion)
                        {
                            return(new Failure <string>($"The existing store format version is incompatible expected format version. Existing store version: {persistedStoreVersion}, expected format version: {storeVersion}."));
                        }
                    }
                    else
                    {
                        return(possibleStoreVersion.Failure);
                    }
                }

                accessor = new KeyValueStoreAccessor(
                    storeConfiguration,
                    persistedStoreVersion,
                    failureHandler,
                    createNewStore,
                    invalidationHandler);
            }
            catch (Exception ex)
            {
                return(new Failure <Exception>(ex));
            }

            return(accessor);
        }
Esempio n. 24
0
        /// <summary>
        /// Attempts to downoad the file to disk.
        /// </summary>
        /// <returns>Returns EvaluationResult.Continue if we sucessfully downloaded and need to continue to store the incremental information, else the result will be what should be returned</returns>
        private async Task <EvaluationResult> TryDownloadFileToDiskAsync(DownloadData downloadData)
        {
            var downloadFilePathAsString = downloadData.DownloadedFilePath.ToString(m_context.PathTable);

            try
            {
                FileUtilities.CreateDirectory(Path.GetDirectoryName(downloadFilePathAsString));
                FileUtilities.DeleteFile(downloadFilePathAsString, waitUntilDeletionFinished: true);
            }
            catch (BuildXLException e)
            {
                m_logger.ErrorPreppingForDownload(m_context.LoggingContext, downloadData.Settings.ModuleName, e.Message);
                return(EvaluationResult.Error);
            }

            // We have to download the file.
            m_logger.StartDownload(m_context.LoggingContext, downloadData.Settings.ModuleName, downloadData.Settings.Url);
            var stopwatch = Stopwatch.StartNew();

            try
            {
                using (var httpClient = new HttpClient())
                {
                    httpClient.Timeout = TimeSpan.FromMinutes(10);
                    var response = await httpClient.GetAsync((Uri)downloadData.DownloadUri, m_context.CancellationToken);

                    response.EnsureSuccessStatusCode();
                    var stream = await response.Content.ReadAsStreamAsync();

                    using (var targetStream = new FileStream(downloadFilePathAsString, FileMode.Create, FileAccess.Write, FileShare.None))
                    {
                        await stream.CopyToAsync(targetStream);
                    }

                    m_logger.Downloaded(
                        m_context.LoggingContext,
                        downloadData.Settings.ModuleName,
                        downloadData.Settings.Url,
                        stopwatch.ElapsedMilliseconds,
                        new FileInfo(downloadFilePathAsString).Length);
                }
            }
            catch (TaskCanceledException e)
            {
                string message = m_context.CancellationToken.IsCancellationRequested ? "Download manually canceled." : e.Message;
                m_logger.DownloadFailed(m_context.LoggingContext, downloadData.Settings.ModuleName, downloadData.Settings.Url, message);
                return(EvaluationResult.Canceled);
            }
            catch (HttpRequestException e)
            {
                var message = e.InnerException == null
                    ? e.Message
                    : e.Message + " " + e.InnerException?.Message;
                m_logger.DownloadFailed(m_context.LoggingContext, downloadData.Settings.ModuleName, downloadData.Settings.Url, message);
                return(EvaluationResult.Error);
            }
            catch (IOException e)
            {
                m_logger.DownloadFailed(m_context.LoggingContext, downloadData.Settings.ModuleName, downloadData.Settings.Url, e.Message);
                return(EvaluationResult.Error);
            }
            catch (UnauthorizedAccessException e)
            {
                m_logger.DownloadFailed(m_context.LoggingContext, downloadData.Settings.ModuleName, downloadData.Settings.Url, e.Message);
                return(EvaluationResult.Error);
            }

            // Indicate we should continue to store the incremental information
            return(EvaluationResult.Continue);
        }
Esempio n. 25
0
        private async Task <SerializationResult> SerializeToFileInternalAsync(GraphCacheFile fileType, Action <BuildXLWriter> serializer, string overrideName)
        {
            // Unblock the caller
            await Task.Yield();

            FileUtilities.CreateDirectory(m_engineCacheLocation);

            string fileName = overrideName ?? GetFileName(fileType);
            string path     = Path.Combine(m_engineCacheLocation, fileName);
            SerializationResult serializationResult = new SerializationResult()
            {
                Success  = false,
                FileType = fileType,
                FullPath = path,
            };

            var fileEnvelope = GetFileEnvelope(fileType);

            Contract.Assume(m_correlationId is FileEnvelopeId, "EngineSerializer must be initialized with a valid correlation id");
            var correlationId = (FileEnvelopeId)m_correlationId;

            try
            {
                // We must delete the existing file in case it was hardlinked from the cache. Opening a filestream
                // that truncates the existing file will fail if it is a hardlink.
                FileUtilities.DeleteFile(path, tempDirectoryCleaner: m_tempDirectoryCleaner);
                using (
                    FileStream fileStream = FileUtilities.CreateFileStream(
                        path,
                        FileMode.Create,
                        FileAccess.Write,
                        FileShare.Delete,
                        // Do not write the file with SequentialScan since it will be reread in the subsequent build
                        FileOptions.None))
                {
                    Stopwatch sw = Stopwatch.StartNew();

                    fileEnvelope.WriteHeader(fileStream, correlationId);

                    // Write whether the file is compressed or not.
                    fileStream.WriteByte(m_useCompression ? (byte)1 : (byte)0);

                    long uncompressedLength = 0;

                    if (m_useCompression)
                    {
                        using (var writer = new BuildXLWriter(m_debug, new TrackedStream(new BufferedStream(new DeflateStream(fileStream, CompressionLevel.Fastest, leaveOpen: true), bufferSize: 64 << 10)), false, false))
                        {
                            // TODO: We can improve performance significantly by parallelizing the compression.
                            // There's no setting to do that, but given you have the entire file content upfront in a memory stream,
                            // it shouldn't be particularly complicated to split the memory stream into reasonably sized chunks (say 100MB)
                            // and compress each of them into separate MemoryStream backed DeflateStreams in separate threads.
                            // Then just write those out to a file. Of course you'll need to write out the position of those streams
                            // into the header when you write out the actual file.
                            serializer(writer);
                            uncompressedLength = writer.BaseStream.Length;
                        }
                    }
                    else
                    {
                        using (var writer = new BuildXLWriter(m_debug, fileStream, leaveOpen: true, logStats: false))
                        {
                            serializer(writer);
                            uncompressedLength = writer.BaseStream.Length;
                        }
                    }

                    Interlocked.Add(ref m_bytesSavedDueToCompression, uncompressedLength - fileStream.Length);

                    fileEnvelope.FixUpHeader(fileStream, correlationId);

                    Tracing.Logger.Log.SerializedFile(LoggingContext, fileName, sw.ElapsedMilliseconds);
                    serializationResult.Success = true;
                    Interlocked.Add(ref m_bytesSerialized, fileStream.Position);
                }
            }
            catch (BuildXLException ex)
            {
                Tracing.Logger.Log.FailedToSerializePipGraph(LoggingContext, ex.LogEventMessage);
            }
            catch (IOException ex)
            {
                Tracing.Logger.Log.FailedToSerializePipGraph(LoggingContext, ex.Message);
            }

            return(serializationResult);
        }
Esempio n. 26
0
        /// <summary>
        /// Attempts to downoad the file to disk.
        /// </summary>
        private async Task <int> TryDownloadFileToDiskAsync(DownloaderArgs arguments)
        {
            try
            {
                FileUtilities.CreateDirectory(arguments.DownloadDirectory);
                FileUtilities.DeleteFile(arguments.DownloadPath, retryOnFailure: true);
            }
            catch (BuildXLException e)
            {
                Console.Error.WriteLine(e.GetLogEventMessage());
                return(1);
            }

            // We have to download the file.
            Console.WriteLine($"Starting download from url '{arguments.Url}' to '{arguments.DownloadPath}'.");

            var stopwatch = Stopwatch.StartNew();

            try
            {
                using (var httpClient = new HttpClient())
                {
                    httpClient.Timeout = TimeSpan.FromMinutes(10);

                    var httpRequest = new HttpRequestMessage(HttpMethod.Get, arguments.Url);

                    // If the download URI is pointing to a VSTS feed and we get a valid auth token, make it part of the request
                    // We only want to send the token over HTTPS and to a VSTS domain to avoid security issues
                    if (IsVSTSPackageSecureURI(arguments.Url) &&
                        await TryGetAuthenticationHeaderAsync(arguments.Url) is var authHeader &&
                        authHeader != null)
                    {
                        httpRequest.Headers.Accept.Clear();
                        httpRequest.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
                        httpRequest.Headers.Authorization = authHeader;
                    }

                    var response = await httpClient.SendAsync(httpRequest);

                    response.EnsureSuccessStatusCode();
                    var stream = await response.Content.ReadAsStreamAsync();

                    using (var targetStream = new FileStream(arguments.DownloadPath, FileMode.Create, FileAccess.Write, FileShare.None))
                    {
                        await stream.CopyToAsync(targetStream);
                    }

                    Console.WriteLine($"Finished download from url '{arguments.Url}' in {stopwatch.ElapsedMilliseconds}ms with {new FileInfo(arguments.DownloadPath).Length} bytes.");
                }
            }
            catch (HttpRequestException e)
            {
                var message = e.InnerException == null
                    ? e.Message
                    : e.Message + " " + e.InnerException?.Message;
                Console.Error.WriteLine($"Failed to download from url '{arguments.Url}': {message}.");
                return(1);
            }
            catch (Exception e) when(e is IOException || e is UnauthorizedAccessException)
            {
                Console.Error.WriteLine($"Failed to download from url '{arguments.Url}': {e.Message}.");
                return(1);
            }

            if (arguments.Hash.HasValue)
            {
                // If the hash is given in the download setting, use the corresponding hashType (hash algorithm) to get the content hash of the downloaded file.
                // We don't record the file until we know it is the correct one and will be used in this build.
                var downloadedHash = await GetContentHashAsync(arguments.DownloadPath, arguments.Hash.Value.HashType);

                // Validate downloaded hash if specified
                if (arguments.Hash != downloadedHash)
                {
                    Console.Error.WriteLine($"Invalid content for url '{arguments.Url}'. The content hash was expected to be: '{arguments.Hash}' but the downloaded files hash was '{downloadedHash}'. " +
                                            "This means that the data on the server has been altered and is not trusted.");
                    return(1);
                }
            }

            return(0);
        }
        private async Task <Possible <ProjectGraphResult> > TryComputeBuildGraphAsync(IEnumerable <AbsolutePath> msBuildSearchLocations, IEnumerable <AbsolutePath> dotnetSearchLocations, IEnumerable <AbsolutePath> parsingEntryPoints, BuildParameters.IBuildParameters buildParameters)
        {
            // We create a unique output file on the obj folder associated with the current front end, and using a GUID as the file name
            AbsolutePath outputDirectory = m_host.GetFolderForFrontEnd(Name);
            AbsolutePath outputFile      = outputDirectory.Combine(m_context.PathTable, Guid.NewGuid().ToString());
            // We create a unique response file that will contain the tool arguments
            AbsolutePath responseFile = outputDirectory.Combine(m_context.PathTable, Guid.NewGuid().ToString());

            // Make sure the directories are there
            FileUtilities.CreateDirectory(outputDirectory.ToString(m_context.PathTable));

            Possible <ProjectGraphWithPredictionsResult <AbsolutePath> > maybeProjectGraphResult = await ComputeBuildGraphAsync(responseFile, parsingEntryPoints, outputFile, msBuildSearchLocations, dotnetSearchLocations, buildParameters);

            if (!maybeProjectGraphResult.Succeeded)
            {
                // A more specific error has been logged already
                return(maybeProjectGraphResult.Failure);
            }

            var projectGraphResult = maybeProjectGraphResult.Result;

            if (m_resolverSettings.KeepProjectGraphFile != true)
            {
                DeleteGraphBuilderRelatedFiles(outputFile, responseFile);
            }
            else
            {
                // Graph-related files are requested to be left on disk. Let's print a message with their location.
                Tracing.Logger.Log.GraphBuilderFilesAreNotRemoved(m_context.LoggingContext, outputFile.ToString(m_context.PathTable), responseFile.ToString(m_context.PathTable));
            }

            if (!projectGraphResult.Succeeded)
            {
                var failure = projectGraphResult.Failure;
                Tracing.Logger.Log.ProjectGraphConstructionError(m_context.LoggingContext, failure.HasLocation ? failure.Location : m_resolverSettings.Location(m_context.PathTable), failure.Message);

                return(new MsBuildGraphConstructionFailure(m_resolverSettings, m_context.PathTable));
            }

            ProjectGraphWithPredictions <AbsolutePath> projectGraph = projectGraphResult.Result;

            // The module contains all project files that are part of the graph
            var projectFiles = new HashSet <AbsolutePath>();

            foreach (ProjectWithPredictions <AbsolutePath> node in projectGraph.ProjectNodes)
            {
                projectFiles.Add(node.FullPath);
            }

            var moduleDescriptor = ModuleDescriptor.CreateWithUniqueId(m_context.StringTable, m_resolverSettings.ModuleName, this);
            var moduleDefinition = ModuleDefinition.CreateModuleDefinitionWithImplicitReferences(
                moduleDescriptor,
                m_resolverSettings.RootTraversal,
                m_resolverSettings.File,
                projectFiles,
                allowedModuleDependencies: null, // no module policies
                cyclicalFriendModules: null,     // no allowlist of cycles
                mounts: null);

            return(new ProjectGraphResult(projectGraph, moduleDefinition, projectGraphResult.PathToMsBuild, projectGraphResult.PathToDotNetExe));
        }
Esempio n. 28
0
        public static void LoadandProcessPolys()
        {
            using (DatabaseCommands commands = new DatabaseCommands())
            {
                #region Get Dwgs to Process

                var dwgs = commands.LoadandProcessPolys();

                foreach (var dwg in dwgs)
                {
                    DwDrawingStacks.Add(dwg);
                }

                #endregion

                GDwgPath = commands.GetGlobalDWGPath();

                var doc =
                    Application.DocumentManager.MdiActiveDocument;
                var ed = doc.Editor;

                foreach (var dwg in DwDrawingStacks)
                {
                    string gpath = Convert.ToString(GDwgPath);

                    if (gpath != null)
                    {
                        string path = Path.Combine(gpath, dwg.PolylineDwgName);

                        if (path == null)
                        {
                            throw new ArgumentNullException(nameof(path));
                        }


                        if (!File.Exists(path))
                        {
                            DatabaseLogs.FormatLogs("File Not Found", path);
                            return;
                        }

                        try
                        {
                            // We'll just suffix the selected filename with "-purged"
                            // for the output location. This file will be overwritten
                            // if the command is called multiple times

                            //var output =
                            //    Path.GetDirectoryName(pfnr.StringResult) + "\\" +
                            //    Path.GetFileNameWithoutExtension(pfnr.StringResult) +
                            //    "-purged" +
                            //    Path.GetExtension(pfnr.StringResult);

                            // Assume a post-R12 drawing

                            using (var db = new Autodesk.AutoCAD.DatabaseServices.Database(false, true))
                            {
                                // Read the DWG file into our Database object

                                db.ReadDwgFile(
                                    path,
                                    FileOpenMode.OpenForReadAndReadShare,
                                    false,
                                    ""
                                    );

                                // No graphical changes, so we can keep the preview
                                // bitmap

                                db.RetainOriginalThumbnailBitmap = true;

                                // We'll store the current working database, to reset
                                // after the purge operation

                                var wdb = HostApplicationServices.WorkingDatabase;
                                HostApplicationServices.WorkingDatabase = db;

                                // Purge unused DGN linestyles from the drawing
                                // (returns false if nothing is erased)
                                ObjectIdCollection collection = GetIdsByTypeTypeValue("POLYLINE", "LWPOLYLINE", "POLYLINE2D", "POLYLINE3d");

                                CopyPolylinesBetweenDatabases(db, collection);

                                // Still need to reset the working database

                                HostApplicationServices.WorkingDatabase = wdb;
                                wdb.Save();
                                string date   = dwg.DateStamp.Value.ToFileTime().ToString();
                                string output = Path.Combine(commands.GetGlobalDestinationPath(dwg.DateStamp.Value), String.Format("HOLES-{0}", date));
                                FileUtilities.CreateDirectory(output);
                                PLineToLayers.ProcessLayers(collection, wdb);
                                output = Path.Combine(output, dwg.PolylineDwgName);
                                doc.Database.SaveAs(output, DwgVersion.Current);

                                //    if (PurgeDgnLinetypesInDb(db, ed))
                                //    {
                                //        // Check the version of the drawing to save back to

                                //        var ver =
                                //            (db.LastSavedAsVersion == DwgVersion.MC0To0
                                //                ? DwgVersion.Current
                                //                : db.LastSavedAsVersion
                                //                );

                                //        // Now we can save

                                //        string output = null;
                                //        db.SaveAs(output, ver);

                                //        ed.WriteMessage(
                                //            "\nSaved purged file to \"{0}\".",
                                //            output
                                //            );
                                //    }

                                //    // Still need to reset the working database

                                //    HostApplicationServices.WorkingDatabase = wdb;
                            }
                        }
                        catch (Autodesk.AutoCAD.Runtime.Exception ex)
                        {
                            ed.WriteMessage("\nException: {0}", ex.Message);
                        }
                    }
                    break;
                }
            }
        }
Esempio n. 29
0
        public override async Task <bool> InitResolverAsync(IResolverSettings resolverSettings, object workspaceResolver)
        {
            Contract.Requires(resolverSettings != null);

            Contract.Assert(m_resolverState == State.Created);
            Contract.Assert(
                resolverSettings is INugetResolverSettings,
                I($"Wrong type for resolver settings, expected {nameof(INugetResolverSettings)} but got {nameof(resolverSettings.GetType)}"));

            Name            = resolverSettings.Name;
            m_resolverState = State.ResolverInitializing;

            m_nugetWorkspaceResolver = workspaceResolver as WorkspaceNugetModuleResolver;
            Contract.Assert(m_nugetWorkspaceResolver != null, "Workspace module resolver is expected to be of source type");

            // TODO: We could do something smarter in the future and just download/generate what is needed
            // Use this result to populate the dictionaries that are used for package retrieval (m_packageDirectories, m_packages and m_owningModules)
            var maybePackages = await m_nugetWorkspaceResolver.GetAllKnownPackagesAsync();

            if (!maybePackages.Succeeded)
            {
                // Error should have been reported.
                return(false);
            }

            m_owningModules = new Dictionary <ModuleId, Package>();

            foreach (var package in maybePackages.Result.Values.SelectMany(v => v))
            {
                m_packages[package.Id]            = package;
                m_owningModules[package.ModuleId] = package;
            }

            if (Configuration.FrontEnd.GenerateCgManifestForNugets.IsValid ||
                Configuration.FrontEnd.ValidateCgManifestForNugets.IsValid)
            {
                var    cgManfiestGenerator = new NugetCgManifestGenerator(Context);
                string generatedCgManifest = cgManfiestGenerator.GenerateCgManifestForPackages(maybePackages.Result);
                string existingCgManifest  = "INVALID";

                if (!Configuration.FrontEnd.GenerateCgManifestForNugets.IsValid &&
                    Configuration.FrontEnd.ValidateCgManifestForNugets.IsValid)
                {
                    // Validation of existing cgmainfest.json results in failure due to mismatch. Should fail the build in this case.
                    try
                    {
                        existingCgManifest = File.ReadAllText(Configuration.FrontEnd.ValidateCgManifestForNugets.ToString(Context.PathTable));
                        FrontEndHost.Engine.RecordFrontEndFile(
                            Configuration.FrontEnd.ValidateCgManifestForNugets,
                            CGManifestResolverName);
                    }
                    // CgManifest FileNotFound, log error and fail build
                    catch (DirectoryNotFoundException e)
                    {
                        Logger.ReportComponentGovernanceValidationError(Context.LoggingContext, "Cannot read Component Governance Manifest file from disk\n" + e.ToString());
                        return(false);
                    }
                    catch (FileNotFoundException e)
                    {
                        Logger.ReportComponentGovernanceValidationError(Context.LoggingContext, "Cannot read Component Governance Manifest file from disk\n" + e.ToString());
                        return(false);
                    }
                    if (!cgManfiestGenerator.CompareForEquality(generatedCgManifest, existingCgManifest))
                    {
                        Logger.ReportComponentGovernanceValidationError(Context.LoggingContext, @"Existing Component Governance Manifest file is outdated, please generate a new one using the argument /generateCgManifestForNugets:<path>");
                        return(false);
                    }

                    m_resolverState = State.ResolverInitialized;
                    return(true);
                }

                // GenerateCgManifestForNugets writes a new file when the old file does not match, hence it will always be valid and does not need validation

                try
                {
                    // We are calling FrontEndHost.Engine.RecordFrontEndFile towards the end of this function because we may update this file after the read below
                    // Updating the file will cause a hash mismatch and the build to fail if this file is read again downstream
                    existingCgManifest = File.ReadAllText(Configuration.FrontEnd.GenerateCgManifestForNugets.ToString(Context.PathTable));
                }
                // CgManifest FileNotFound, continue to write the new file
                // No operations required as the empty existingCgManifest will not match with the newly generated cgManifest
                catch (DirectoryNotFoundException) { }
                catch (FileNotFoundException) { }

                if (!cgManfiestGenerator.CompareForEquality(generatedCgManifest, existingCgManifest))
                {
                    if (Configuration.FrontEnd.GenerateCgManifestForNugets.IsValid)
                    {
                        // Overwrite or create new cgmanifest.json file with updated nuget package and version info
                        string targetFilePath = Configuration.FrontEnd.GenerateCgManifestForNugets.ToString(Context.PathTable);

                        try
                        {
                            FileUtilities.CreateDirectory(Path.GetDirectoryName(targetFilePath));
                            await FileUtilities.WriteAllTextAsync(targetFilePath, generatedCgManifest, Encoding.UTF8);
                        }
                        catch (BuildXLException e)
                        {
                            Logger.ReportComponentGovernanceGenerationError(Context.LoggingContext, "Could not write Component Governance Manifest file to disk\n" + e.ToString());
                            return(false);
                        }
                    }
                }

                FrontEndHost.Engine.RecordFrontEndFile(
                    Configuration.FrontEnd.GenerateCgManifestForNugets,
                    CGManifestResolverName);
            }

            m_resolverState = State.ResolverInitialized;

            return(true);
        }
Esempio n. 30
0
        private static ServerDeploymentCacheCreated CreateServerDeployment(string destDir, AppDeployment clientApp)
        {
            Stopwatch st = Stopwatch.StartNew();

            // Check if the main server process is in use before attempting to delete the deployment, this way we avoid partially deleting files
            // due to access permission issues. This is not completely bullet proof (there can be a race) but it is highly unlikely the
            // process starts to be in use right after this check
            KillServer(destDir);

            // Deletes the existing cache directory if it exists, so we avoid accumulating garbage.
            if (Directory.Exists(destDir))
            {
                // Remove all files regardless of files being readonly
                FileUtilities.DeleteDirectoryContents(destDir, true);
            }

            // Perform the deployment
            AppDeployment    serverDeployment = AppDeployment.ReadDeploymentManifest(clientApp.BaseDirectory, AppDeployment.ServerDeploymentManifestFileName);
            HashSet <string> directories      = new HashSet <string>();
            List <KeyValuePair <string, string> > filesToCopy = new List <KeyValuePair <string, string> >();

            foreach (string path in serverDeployment.GetRelevantRelativePaths(forServerDeployment: true).Concat(new string[] { AppDeployment.ServerDeploymentManifestFileName }))
            {
                string targetPath = Path.Combine(destDir, path);
                string sourcePath = Path.Combine(clientApp.BaseDirectory, path);
                string directory  = Path.GetDirectoryName(targetPath);
                if (directories.Add(directory))
                {
                    FileUtilities.CreateDirectory(directory);
                }

                filesToCopy.Add(new KeyValuePair <string, string>(sourcePath, targetPath));
            }

            // Because some deployments use virtualized vpak, using a very parallelized copy is beneficial
            Parallel.ForEach(
                filesToCopy,
                new ParallelOptions()
            {
                MaxDegreeOfParallelism = 50,
            },
                (fileToCopy) =>
            {
                if (File.Exists(fileToCopy.Key))
                {
                    File.Copy(fileToCopy.Key, fileToCopy.Value);
                }
            });

#if NET_FRAMEWORK
            var ngenExe = Path.Combine(System.Runtime.InteropServices.RuntimeEnvironment.GetRuntimeDirectory(), @"ngen.exe");
            var destExe = Path.Combine(destDir, System.AppDomain.CurrentDomain.FriendlyName);

            // queue:1 means it runs in the background
            if (File.Exists(ngenExe))
            {
                var ngenArgs = "install " + destExe + " /queue:1";
                ProcessStartInfo startInfo = new ProcessStartInfo(ngenExe, ngenArgs);
                startInfo.UseShellExecute = false;
                startInfo.CreateNoWindow  = true;
                Process.Start(startInfo);
            }
#endif

            ServerDeploymentCacheCreated cacheCreated = default(ServerDeploymentCacheCreated);
            cacheCreated.TimeToCreateServerCacheMilliseconds = st.ElapsedMilliseconds;

            return(cacheCreated);
        }