Inheritance: IFileUtilities
Beispiel #1
0
        public static CustomResult Create(CustomConfig config, string inputFileName, CustomResult result, IS3Client s3Client)
        {
            string inputBaseName   = inputFileName.TrimEndFromFirst(".tsv");
            string ngaFileName     = inputBaseName + SaCommon.GeneFileSuffix;
            string localNgaPath    = Path.Combine(Path.GetTempPath(), ngaFileName);
            string localSchemaPath = localNgaPath + SaCommon.JsonSchemaSuffix;
            string localLogPath    = Path.Combine(Path.GetTempPath(), LogFileName);

            HttpUtilities.ValidateUrl(LambdaUrlHelper.GetUgaUrl());
            var outputFiles = new List <string>();

            using (var aes = new AesCryptoServiceProvider())
            {
                FileMetadata ngaMetadata, schemaMetadata, logMetaData;
                using (var logStream = FileUtilities.GetCreateStream(localLogPath))
                    using (var logCryptoStream = new CryptoStream(logStream, aes.CreateEncryptor(), CryptoStreamMode.Write))
                        using (var logMd5Stream = new MD5Stream(logCryptoStream))
                            //
                            using (var customTsvStream = (PersistentStream)PersistentStreamUtils.GetReadStream(config.tsvUrl))
                                using (var parser = GetGeneAnnotationsParserFromCustomTsvStream(customTsvStream))
                                    //
                                    using (var ngaStream = FileUtilities.GetCreateStream(localNgaPath))
                                        using (var ngaCryptoStream = new CryptoStream(ngaStream, aes.CreateEncryptor(), CryptoStreamMode.Write))
                                            using (var ngaMd5Stream = new MD5Stream(ngaCryptoStream))
                                                //
                                                using (var schemaStream = FileUtilities.GetCreateStream(localSchemaPath))
                                                    using (var schemaCryptoStream = new CryptoStream(schemaStream, aes.CreateEncryptor(), CryptoStreamMode.Write))
                                                        using (var schemaMd5Stream = new MD5Stream(schemaCryptoStream))
                                                        {
                                                            using (var ngaWriter = CaUtilities.GetNgaWriter(ngaMd5Stream, parser, inputFileName))
                                                                using (var schemaWriter = new StreamWriter(schemaMd5Stream))
                                                                    using (var logWriter = new StreamWriter(logMd5Stream))
                                                                    {
                                                                        ngaWriter.Write(parser.GetItems(config.skipGeneIdValidation, logWriter));
                                                                        var unknownGenes = parser.GetUnknownGenes();
                                                                        if (!config.skipGeneIdValidation && unknownGenes.Count > 0)
                                                                        {
                                                                            throw new UserErrorException($"The following gene IDs were not recognized in Nirvana: {string.Join(',', unknownGenes)}");
                                                                        }

                                                                        schemaWriter.Write(parser.JsonSchema);
                                                                    }

                                                            //all the writers have to be disposed before GetFileMetaData is called
                                                            ngaMetadata    = ngaMd5Stream.GetFileMetadata();
                                                            schemaMetadata = schemaMd5Stream.GetFileMetadata();
                                                            logMetaData    = logMd5Stream.GetFileMetadata();
                                                        }

                if (config.skipGeneIdValidation)
                {
                    string logS3Key = string.Join('/', config.outputDir.path.Trim('/'), LogFileName);
                    Logger.WriteLine("uploading log file to " + logS3Key);
                    s3Client.DecryptUpload(config.outputDir.bucketName, logS3Key, localLogPath, aes, logMetaData);
                }

                string nsaS3Path    = string.Join('/', config.outputDir.path.Trim('/'), ngaFileName);
                string schemaS3Path = nsaS3Path + SaCommon.JsonSchemaSuffix;

                s3Client.DecryptUpload(config.outputDir.bucketName, nsaS3Path, localNgaPath, aes, ngaMetadata);
                s3Client.DecryptUpload(config.outputDir.bucketName, schemaS3Path, localSchemaPath, aes, schemaMetadata);


                outputFiles.Add(ngaFileName);
                outputFiles.Add(ngaFileName + SaCommon.JsonSchemaSuffix);

                LambdaUtilities.DeleteTempOutput();

                return(CustomAnnotationLambda.GetSuccessResult(config, result, outputFiles));
            }
        }
 /// <summary>
 /// Writes null to the file as one 0x00 byte
 /// </summary>
 /// <returns>Number of bytes used to store the null</returns>
 internal int WriteNull()
 {
     byteBuffer[0] = 0x00;
     return(FileUtilities.WriteWithLength(fileStream, byteBuffer, 1));
 }
 /// <summary>
 /// Writes a byte to the file
 /// </summary>
 /// <returns>Number of bytes used to store the byte</returns>
 internal int WriteByte(byte val)
 {
     byteBuffer[0] = 0x01; // length
     byteBuffer[1] = val;
     return(FileUtilities.WriteWithLength(fileStream, byteBuffer, 2));
 }
Beispiel #4
0
        private static void ExecuteRequest(object state)
        {
            NetworkStream stream = null;

            byte []         buffer;
            int             id      = counter++;
            string          tmpfile = null;
            TcpClient       client  = null;
            BinaryReader    reader  = null;
            BinaryWriter    writer  = null;
            string          remote_ip;
            WebServiceLogin login = new WebServiceLogin();

            try {
                buffer = new byte [1024];

                client              = (TcpClient)state;
                stream              = client.GetStream();
                stream.ReadTimeout  = (int)TimeSpan.FromMinutes(5).TotalMilliseconds;
                stream.WriteTimeout = stream.ReadTimeout;
                reader              = new BinaryReader(stream);
                writer              = new BinaryWriter(stream);
                remote_ip           = ((IPEndPoint)client.Client.RemoteEndPoint).Address.ToString();

                /* Format:
                 * Field                   Length in bytes    Description
                 * version                       1                  1
                 * name_length                   1
                 * name                       <name_length>
                 * password_length               1
                 * password                 <password_length>
                 * work_id                       4                 The Work.id field in the database
                 * file_count                    2
                 * reserved                      8
                 *
                 * < file #1>
                 * marker                       12               'MonkeyWrench'
                 * md5                          16
                 * flags                         1                  1 = compressed, 2 = hidden
                 * filename_length               1
                 * filename               <filename_length>
                 * <client waits for answer, either type 2 or 4, type 2: server already has data, type 4: server does not have data>
                 * [type: 4] compressed_mime_length        1
                 * [type: 4] compressed_mime   <compressed_mime_length>
                 * [type: 4] content_length      4
                 * [type: 4] content       <content_length>
                 * [type: 4] <client waits for response, type 2>
                 *
                 * < file #2 >
                 * ...
                 *
                 * The response data has this format:
                 * version                        1
                 * type                           1                  1 = everything OK, 2 = file received OK, 3 = error, 4 = send file
                 *  depending on type, here are the subsequent fields:
                 * [type: 3] message_length       2
                 * [type: 3] message          <message length>
                 *
                 */
                byte version     = reader.ReadByte();
                byte name_length = reader.ReadByte();
                login.User = ReadString(reader, buffer, name_length);
                byte password_length = reader.ReadByte();
                login.Password = ReadString(reader, buffer, password_length);
                login.Ip4      = remote_ip;
                int    work_id    = reader.ReadInt32();
                ushort file_count = reader.ReadUInt16();
                reader.ReadInt64();


                log.DebugFormat("Upload.ExecuteRequest (): {0} version: {1} work_id: {2} file count: {3} remote ip: {4}", id, version, work_id, file_count, client.Client.RemoteEndPoint.ToString());

                using (DB db = new DB()) {
                    Authentication.VerifyUserInRole(remote_ip, db, login, Roles.BuildBot, true);
                    for (ushort i = 0; i < file_count; i++)
                    {
                        byte [] md5 = new byte [16];
                        byte    flags;
                        bool    hidden;
                        byte    filename_length;
                        byte    compressed_mime_length;
                        int     content_length;
                        string  filename;
                        string  compressed_mime;
                        string  marker;

                        marker = ReadString(reader, buffer, 12);
                        if (marker != "MonkeyWrench")
                        {
                            throw new Exception(string.Format("Didn't get marker 'MonkeyWrench' at start of file, got '{0}'", marker));
                        }

                        reader.Read(md5, 0, 16);

                        flags           = reader.ReadByte();
                        filename_length = reader.ReadByte();

                        filename = ReadString(reader, buffer, filename_length);

                        hidden = (flags & 0x2) == 0x2;
                        // compressed = (flags & 0x1) == 0x1;

                        log.DebugFormat("Upload.ExecuteRequest (): {0} file #{1}: filename: '{2}' ", id, i + 1, filename);

                        DBFile file = DBFile_Extensions.Find(db, FileUtilities.MD5BytesToString(md5));
                        if (file == null)
                        {
                            log.DebugFormat("Upload.ExecuteRequest (): {0} file #{1} must be sent, sending 'send file' response", id, i + 1);
                            // Write 'send file'
                            writer.Write((byte)1);                               // version
                            writer.Write((byte)4);                               // type (4 = send file)
                            writer.Flush();

                            compressed_mime_length = reader.ReadByte();
                            compressed_mime        = ReadString(reader, buffer, compressed_mime_length);
                            content_length         = reader.ReadInt32();

                            log.DebugFormat("Upload.ExecuteRequest (): {0} file #{1} content_length: {2} compressed_mime: '{3}' reading...", id, i + 1, content_length, compressed_mime);

                            int bytes_left = content_length;
                            tmpfile = Path.GetTempFileName();
                            using (FileStream fs = new FileStream(tmpfile, FileMode.Open, FileAccess.Write, FileShare.Read)) {
                                while (bytes_left > 0)
                                {
                                    int to_read = Math.Min(bytes_left, buffer.Length);
                                    int read    = reader.Read(buffer, 0, to_read);
                                    if (read == 0)
                                    {
                                        throw new Exception(string.Format("Failed to read {0} bytes, {1} bytes left", content_length, bytes_left));
                                    }
                                    fs.Write(buffer, 0, read);
                                    bytes_left -= read;
                                }
                            }

                            log.DebugFormat("Upload.ExecuteRequest (): {0} file #{1} received, uploading '{2}' to database", id, i + 1, tmpfile);
                            file = db.Upload(FileUtilities.MD5BytesToString(md5), tmpfile, filename, Path.GetExtension(filename), hidden, compressed_mime);
                        }
                        else
                        {
                            log.DebugFormat("Upload.ExecuteRequest (): {0} file #{1} already in database, not uploading", id, i + 1);
                        }

                        DBWork work = DBWork_Extensions.Create(db, work_id);
                        work.AddFile(db, file, filename, hidden);

                        // Write 'file recieved OK'
                        writer.Write((byte)1);                           // version
                        writer.Write((byte)2);                           // type (2 = file received OK)
                        writer.Flush();
                        log.DebugFormat("Upload.ExecuteRequest (): {0} {1} uploaded successfully", id, filename);
                    }
                }

                // Write 'everything OK'
                writer.Write((byte)1);                   // version
                writer.Write((byte)1);                   // type (1 = everything OK)
                writer.Flush();
                log.DebugFormat("Upload.ExecuteRequest (): {0} completed", id);
            } catch (Exception ex) {
                log.ErrorFormat("Upload.ExecuteRequest (): {0} {1}", id, ex);
                try {
                    string  msg        = ex.ToString();
                    byte [] msg_buffer = Encoding.UTF8.GetBytes(msg);
                    writer.Write((byte)1);                                                     // version
                    writer.Write((byte)3);                                                     // type (3 = error)
                    writer.Write((ushort)Math.Min(msg_buffer.Length, ushort.MaxValue));        // message_length
                    writer.Write(msg_buffer, 0, Math.Min(msg_buffer.Length, ushort.MaxValue)); // message
                    stream.Flush();
                } catch (Exception ex2) {
                    log.ErrorFormat("Upload.ExecuteRequest (): {0} Failed to send exception to client: {1}", id, ex2);
                }
            } finally {
                if (tmpfile != null)
                {
                    FileUtilities.TryDeleteFile(tmpfile);
                }
                try {
                    client.Close();
                } catch (Exception ex) {
                    log.ErrorFormat("Error closing connection: {0}", ex);
                    // Ignore
                }
            }
        }
Beispiel #5
0
 public void AddMetadataReference(string referencePath, MetadataReferenceProperties properties)
 {
     referencePath = FileUtilities.NormalizeAbsolutePath(referencePath);
     AddMetadataReferenceAndTryConvertingToProjectReferenceIfPossible(referencePath, properties);
 }
Beispiel #6
0
        /// <summary>
        /// Actually do the work specified by the ZippedETLWriter constructors and other methods.
        /// </summary>
        public bool WriteArchive(CompressionLevel compressionLevel = CompressionLevel.Optimal)
        {
            List <string> pdbFileList = PrepForWrite();

            if (!Zip)
            {
                return(true);
            }

            bool success = false;
            var  sw      = Stopwatch.StartNew();

            if (ZipArchivePath == null)
            {
                ZipArchivePath = m_etlFilePath + ".zip";
            }
            var newFileName = ZipArchivePath + ".new";

            FileUtilities.ForceDelete(newFileName);
            try
            {
                Log.WriteLine("[Zipping ETL file {0}]", m_etlFilePath);
                using (var zipArchive = ZipFile.Open(newFileName, ZipArchiveMode.Create))
                {
                    zipArchive.CreateEntryFromFile(m_etlFilePath, Path.GetFileName(m_etlFilePath), compressionLevel);
                    if (pdbFileList != null)
                    {
                        Log.WriteLine("[Writing {0} PDBS to Zip file]", pdbFileList.Count);
                        // Add the Pdbs to the archive
                        foreach (var pdb in pdbFileList)
                        {
                            // If the path looks like a sym server cache path, grab that chunk, otherwise just copy the file name part of the path.
                            string relativePath;
                            var    m = Regex.Match(pdb, @"\\([^\\]+.pdb\\\w\w\w\w\w\w\w\w\w\w\w\w\w\w\w\w\w\w\w\w\w\w\w\w\w\w\w\w\w\w\w\w\d+\\[^\\]+)$");
                            if (m.Success)
                            {
                                relativePath = m.Groups[1].Value;
                            }
                            else
                            {
                                relativePath = Path.GetFileName(pdb);
                            }

                            var archivePath = Path.Combine("symbols", relativePath);

                            // log.WriteLine("Writing PDB {0} to archive.", archivePath);
                            zipArchive.CreateEntryFromFile(pdb, archivePath, compressionLevel);
                        }
                        Log.Flush();

                        if (m_additionalFiles != null)
                        {
                            foreach (Tuple <string, string> additionalFile in m_additionalFiles)
                            {
                                // We dont use CreatEntryFromFile because it will not open files thar are open for writting.
                                // Since a typical use of this is to write the log file, which will be open for writing, we
                                // use File.Open and allow this case explicitly.
                                using (Stream fs = File.Open(additionalFile.Item1, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
                                {
                                    // Item2 tells you the path in the archive.
                                    var entry = zipArchive.CreateEntry(additionalFile.Item2, compressionLevel);
                                    using (Stream es = entry.Open())
                                        fs.CopyTo(es);
                                }
                            }
                        }
                    }
                }
                FileUtilities.ForceMove(newFileName, ZipArchivePath);
                if (DeleteInputFile)
                {
                    Log.WriteLine("Deleting original ETL file {0}", m_etlFilePath);
                    FileUtilities.ForceDelete(m_etlFilePath);
                }

                // We make the ZIP the same time as the original file.
                File.SetLastWriteTimeUtc(ZipArchivePath, File.GetLastWriteTimeUtc(m_etlFilePath));
                success = true;
            }
            finally
            {
                FileUtilities.ForceDelete(newFileName);
            }
            sw.Stop();
            Log.WriteLine("ZIP generation took {0:f3} sec", sw.Elapsed.TotalSeconds);
            Log.WriteLine("ZIP output file {0}", ZipArchivePath);
            return(success);
        }
Beispiel #7
0
        /// <summary>
        /// Internal constructor
        /// </summary>
        /// <param name="ownerTask">The task that is using file tracker</param>
        /// <param name="tlogFiles">The .write. tlog files to interpret</param>
        /// <param name="skipMissingFiles">Ignore files that do not exist on disk</param>
        /// <param name="excludedInputPaths">The set of paths that contain files that are to be ignored during up to date check</param>
        private void InternalConstruct(ITask ownerTask, ITaskItem[] tlogFilesLocal, ITaskItem[] tlogFilesToIgnore, bool skipMissingFiles, DateTime missingFileTimeUtc, string[] excludedInputPaths)
        {
            if (ownerTask != null)
            {
                _log = new TaskLoggingHelper(ownerTask);
                _log.TaskResources     = AssemblyResources.PrimaryResources;
                _log.HelpKeywordPrefix = "MSBuild.";
            }

            ITaskItem[] expandedTlogFiles = TrackedDependencies.ExpandWildcards(tlogFilesLocal);

            if (tlogFilesToIgnore != null)
            {
                ITaskItem[] expandedTlogFilesToIgnore = TrackedDependencies.ExpandWildcards(tlogFilesToIgnore);

                if (expandedTlogFilesToIgnore.Length > 0)
                {
                    HashSet <string> ignore             = new HashSet <string>();
                    List <ITaskItem> remainingTlogFiles = new List <ITaskItem>();

                    foreach (ITaskItem tlogFileToIgnore in expandedTlogFilesToIgnore)
                    {
                        ignore.Add(tlogFileToIgnore.ItemSpec);
                    }

                    foreach (ITaskItem tlogFile in expandedTlogFiles)
                    {
                        if (!ignore.Contains(tlogFile.ItemSpec))
                        {
                            remainingTlogFiles.Add(tlogFile);
                        }
                    }

                    _tlogFiles = remainingTlogFiles.ToArray();
                }
                else
                {
                    _tlogFiles = expandedTlogFiles;
                }
            }
            else
            {
                _tlogFiles = expandedTlogFiles;
            }

            // We have no TLog files on disk, create a TLog marker from the
            // TLogFiles ItemSpec so we can fabricate one if we need to
            // This becomes our "first" tlog, since on the very first run, no tlogs
            // will exist, and if a compaction has been run (as part of the initial up-to-date check) then this
            // marker tlog will be created as empty.
            if (_tlogFiles == null || _tlogFiles.Length == 0)
            {
                _tlogMarker = tlogFilesLocal[0].ItemSpec.Replace("*", "1");
                _tlogMarker = _tlogMarker.Replace("?", "2");
            }

            if (excludedInputPaths != null)
            {
                // Assign our exclude paths to our lookup - and make sure that all recorded paths end in a slash so that
                // our "starts with" comparison doesn't pick up incomplete matches, such as C:\Foo matching C:\FooFile.txt
                foreach (string excludePath in excludedInputPaths)
                {
                    string fullexcludePath = FileUtilities.EnsureTrailingSlash(FileUtilities.NormalizePath(excludePath)).ToUpperInvariant();
                    _excludedInputPaths.Add(fullexcludePath);
                }
            }

            _tlogsAvailable     = TrackedDependencies.ItemsExist(_tlogFiles);
            _skipMissingFiles   = skipMissingFiles;
            _missingFileTimeUtc = missingFileTimeUtc.ToUniversalTime();
            if (_tlogFiles != null)
            {
                // Read the TLogs into our internal structures
                ConstructFileTable();
            }
        }
 public void ItShouldNotCopyAnyFiles()
 {
     FileUtilities.AssertWasNotCalled(
         f => f.FileCopy(null, null),
         o => o.IgnoreArguments());
 }
        /// <summary>
        /// Given the absolute location of a file, and a disc location, returns relative file path to that disk location.
        /// Throws UriFormatException.
        /// </summary>
        /// <param name="basePath">
        /// The base path we want to relativize to. Must be absolute.
        /// Should <i>not</i> include a filename as the last segment will be interpreted as a directory.
        /// </param>
        /// <param name="path">
        /// The path we need to make relative to basePath.  The path can be either absolute path or a relative path in which case it is relative to the base path.
        /// If the path cannot be made relative to the base path (for example, it is on another drive), it is returned verbatim.
        /// </param>
        /// <returns>relative path (can be the full path)</returns>
        internal static string MakeRelative(string basePath, string path)
        {
            string result = FileUtilities.MakeRelative(basePath, path);

            return(result);
        }
            protected void DecorateItemsWithMetadata(IEnumerable <ItemBatchingContext> itemBatchingContexts, ImmutableList <ProjectMetadataElement> metadata, bool?needToExpandMetadata = null)
            {
                if (metadata.Count > 0)
                {
                    ////////////////////////////////////////////////////
                    // UNDONE: Implement batching here.
                    //
                    // We want to allow built-in metadata in metadata values here.
                    // For example, so that an Idl file can specify that its Tlb output should be named %(Filename).tlb.
                    //
                    // In other words, we want batching. However, we won't need to go to the trouble of using the regular batching code!
                    // That's because that code is all about grouping into buckets of similar items. In this context, we're not
                    // invoking a task, and it's fine to process each item individually, which will always give the correct results.
                    //
                    // For the CTP, to make the minimal change, we will not do this quite correctly.
                    //
                    // We will do this:
                    // -- check whether any metadata values or their conditions contain any bare built-in metadata expressions,
                    //    or whether they contain any custom metadata && the Include involved an @(itemlist) expression.
                    // -- if either case is found, we go ahead and evaluate all the metadata separately for each item.
                    // -- otherwise we can do the old thing (evaluating all metadata once then applying to all items)
                    //
                    // This algorithm gives the correct results except when:
                    // -- batchable expressions exist on the include, exclude, or condition on the item element itself
                    //
                    // It means that 99% of cases still go through the old code, which is best for the CTP.
                    // When we ultimately implement this correctly, we should make sure we optimize for the case of very many items
                    // and little metadata, none of which varies between items.

                    // Do not expand properties as they have been already expanded by the lazy evaluator upon item operation construction.
                    // Prior to lazy evaluation ExpanderOptions.ExpandAll was used.
                    const ExpanderOptions metadataExpansionOptions = ExpanderOptions.ExpandAll;

                    needToExpandMetadata ??= NeedToExpandMetadataForEachItem(metadata, out _);

                    if (needToExpandMetadata.Value)
                    {
                        foreach (var itemContext in itemBatchingContexts)
                        {
                            _expander.Metadata = itemContext.GetMetadataTable();

                            foreach (var metadataElement in metadata)
                            {
                                if (!EvaluateCondition(metadataElement.Condition, metadataElement, metadataExpansionOptions, ParserOptions.AllowAll, _expander, _lazyEvaluator))
                                {
                                    continue;
                                }

                                string evaluatedValue = _expander.ExpandIntoStringLeaveEscaped(metadataElement.Value, metadataExpansionOptions, metadataElement.Location);

                                itemContext.OperationItem.SetMetadata(metadataElement, FileUtilities.MaybeAdjustFilePath(evaluatedValue, metadataElement.ContainingProject.DirectoryPath));
                            }
                        }

                        // End of legal area for metadata expressions.
                        _expander.Metadata = null;
                    }

                    // End of pseudo batching
                    ////////////////////////////////////////////////////
                    // Start of old code
                    else
                    {
                        // Metadata expressions are allowed here.
                        // Temporarily gather and expand these in a table so they can reference other metadata elements above.
                        EvaluatorMetadataTable metadataTable = new EvaluatorMetadataTable(_itemType);
                        _expander.Metadata = metadataTable;

                        // Also keep a list of everything so we can get the predecessor objects correct.
                        List <Pair <ProjectMetadataElement, string> > metadataList = new List <Pair <ProjectMetadataElement, string> >(metadata.Count);

                        foreach (var metadataElement in metadata)
                        {
                            // Because of the checking above, it should be safe to expand metadata in conditions; the condition
                            // will be true for either all the items or none
                            if (
                                !EvaluateCondition(
                                    metadataElement.Condition,
                                    metadataElement,
                                    metadataExpansionOptions,
                                    ParserOptions.AllowAll,
                                    _expander,
                                    _lazyEvaluator
                                    ))
                            {
                                continue;
                            }

                            string evaluatedValue = _expander.ExpandIntoStringLeaveEscaped(metadataElement.Value, metadataExpansionOptions, metadataElement.Location);
                            evaluatedValue = FileUtilities.MaybeAdjustFilePath(evaluatedValue, metadataElement.ContainingProject.DirectoryPath);

                            metadataTable.SetValue(metadataElement, evaluatedValue);
                            metadataList.Add(new Pair <ProjectMetadataElement, string>(metadataElement, evaluatedValue));
                        }

                        // Apply those metadata to each item
                        // Note that several items could share the same metadata objects

                        // Set all the items at once to make a potential copy-on-write optimization possible.
                        // This is valuable in the case where one item element evaluates to
                        // many items (either by semicolon or wildcards)
                        // and that item also has the same piece/s of metadata for each item.
                        _itemFactory.SetMetadata(metadataList, itemBatchingContexts.Select(i => i.OperationItem));

                        // End of legal area for metadata expressions.
                        _expander.Metadata = null;
                    }
                }
            }
Beispiel #11
0
        /// <inheritdoc />
        public Task <FileExistenceResult> CheckFileExistsAsync(AbsolutePath path, TimeSpan timeout, CancellationToken cancellationToken)
        {
            var resultCode = FileUtilities.Exists(path.Path) ? FileExistenceResult.ResultCode.FileExists : FileExistenceResult.ResultCode.FileNotFound;

            return(Task.FromResult(new FileExistenceResult(resultCode)));
        }
Beispiel #12
0
        [PlatformSpecific(TestPlatforms.Windows)] // "Under Unix all filenames are valid and this test is not useful"
        public void SomeInputsFailToCreate()
        {
            string temp    = Path.GetTempPath();
            string file    = Path.Combine(temp, "2A333ED756AF4dc392E728D0F864A38e");
            string dir     = Path.Combine(temp, "2A333ED756AF4dc392E728D0F864A38f");
            string invalid = "!@#$%^&*()|";
            string dir2    = Path.Combine(temp, "2A333ED756AF4dc392E728D0F864A390");

            try
            {
                FileStream fs = File.Create(file);
                fs.Dispose(); //we're gonna try to delete it

                MakeDir    t      = new MakeDir();
                MockEngine engine = new MockEngine();
                t.BuildEngine = engine;

                t.Directories = new ITaskItem[]
                {
                    new TaskItem(dir),
                    new TaskItem(file),
                    new TaskItem(invalid),
                    new TaskItem(dir2)
                };

                bool success = t.Execute();

                if (NativeMethodsShared.IsWindows)
                {
                    Assert.False(success);
                    Assert.Equal(2, t.DirectoriesCreated.Length);
                    Assert.Equal(dir2, t.DirectoriesCreated[1].ItemSpec);
                }
                else
                {
                    // Since Unix pretty much does not have invalid characters,
                    // the invalid name is not really invalid
                    Assert.True(success);
                    Assert.Equal(3, t.DirectoriesCreated.Length);
                    Assert.Equal(dir2, t.DirectoriesCreated[2].ItemSpec);
                }

                Assert.Equal(dir, t.DirectoriesCreated[0].ItemSpec);
                Assert.True
                (
                    engine.Log.Contains
                    (
                        String.Format(AssemblyResources.GetString("MakeDir.Comment"), dir)
                    )
                );
            }
            finally
            {
                FileUtilities.DeleteWithoutTrailingBackslash(dir);
                File.Delete(file);
                if (!NativeMethodsShared.IsWindows)
                {
                    File.Delete(invalid);
                }
                FileUtilities.DeleteWithoutTrailingBackslash(dir2);
            }
        }
Beispiel #13
0
        /// <summary>
        /// Builds a patch using given paths for the target and upgrade packages.
        /// </summary>
        /// <param name="targetPath">The path to the target MSI.</param>
        /// <param name="upgradePath">The path to the upgrade MSI.</param>
        /// <param name="name">The name of the patch to build.</param>
        /// <param name="version">Optional version for the bundle.</param>
        /// <returns>The path to the patch.</returns>
        private string BuildPatch(string targetPath, string upgradePath, string name, string version)
        {
            // Get the name of the calling method.
            StackTrace stack  = new StackTrace();
            string     caller = stack.GetFrame(1).GetMethod().Name;

            // Create paths.
            string source        = Path.Combine(this.TestContext.TestDataDirectory, String.Concat(name, ".wxs"));
            string rootDirectory = FileUtilities.GetUniqueFileName();
            string objDirectory  = Path.Combine(rootDirectory, Settings.WixobjFolder);
            string msiDirectory  = Path.Combine(rootDirectory, Settings.MsiFolder);
            string wixmst        = Path.Combine(objDirectory, String.Concat(name, ".wixmst"));
            string wixmsp        = Path.Combine(objDirectory, String.Concat(name, ".wixmsp"));
            string package       = Path.Combine(msiDirectory, String.Concat(name, ".msp"));

            // Add the root directory to be cleaned up.
            this.TestArtifacts.Add(new DirectoryInfo(rootDirectory));

            // Compile.
            Candle candle = new Candle();

            candle.Extensions.AddRange(DependencyExtensionTests.Extensions);
            candle.OtherArguments = String.Concat("-dTestName=", caller);
            if (!String.IsNullOrEmpty(version))
            {
                candle.OtherArguments = String.Concat(candle.OtherArguments, " -dVersion=", version);
            }
            candle.OutputFile = String.Concat(objDirectory, @"\");
            candle.SourceFiles.Add(source);
            candle.WorkingDirectory = this.TestContext.TestDataDirectory;
            candle.Run();

            // Make sure the output directory is cleaned up.
            this.TestArtifacts.Add(new DirectoryInfo(objDirectory));

            // Link.
            Light light = new Light();

            light.Extensions.AddRange(DependencyExtensionTests.Extensions);
            light.ObjectFiles = candle.ExpectedOutputFiles;
            light.OutputFile  = wixmsp;
            light.SuppressMSIAndMSMValidation = true;
            light.WorkingDirectory            = this.TestContext.TestDataDirectory;
            light.Run();

            // Make sure the output directory is cleaned up.
            this.TestArtifacts.Add(new DirectoryInfo(msiDirectory));

            // Torch.
            Torch torch = new Torch();

            torch.TargetInput        = Path.ChangeExtension(targetPath, "wixpdb");
            torch.UpdatedInput       = Path.ChangeExtension(upgradePath, "wixpdb");
            torch.PreserveUnmodified = true;
            torch.XmlInput           = true;
            torch.OutputFile         = wixmst;
            torch.WorkingDirectory   = this.TestContext.TestDataDirectory;
            torch.Run();

            // Pyro.
            Pyro pyro = new Pyro();

            pyro.Baselines.Add(torch.OutputFile, name);
            pyro.InputFile        = light.OutputFile;
            pyro.OutputFile       = package;
            pyro.WorkingDirectory = this.TestContext.TestDataDirectory;
            pyro.SuppressWarnings.Add("1079");
            pyro.Run();

            return(pyro.OutputFile);
        }
        // internal for testing
        internal ImmutableArray <CompletionItem> GetItems(string directoryPath, CancellationToken cancellationToken)
        {
            if (!PathUtilities.IsUnixLikePlatform && directoryPath.Length == 1 && directoryPath[0] == '\\')
            {
                // The user has typed only "\".  In this case, we want to add "\\" to the list.
                return(ImmutableArray.Create(CreateNetworkRoot()));
            }

            var result = ArrayBuilder <CompletionItem> .GetInstance();

            var pathKind = PathUtilities.GetPathKind(directoryPath);

            switch (pathKind)
            {
            case PathKind.Empty:
                // base directory
                if (_baseDirectoryOpt != null)
                {
                    result.AddRange(GetItemsInDirectory(_baseDirectoryOpt, cancellationToken));
                }

                // roots
                if (PathUtilities.IsUnixLikePlatform)
                {
                    result.AddRange(CreateUnixRoot());
                }
                else
                {
                    foreach (var drive in GetLogicalDrives())
                    {
                        result.Add(CreateLogicalDriveItem(drive.TrimEnd(s_windowsDirectorySeparator)));
                    }

                    result.Add(CreateNetworkRoot());
                }

                // entries on search paths
                foreach (var searchPath in _searchPaths)
                {
                    result.AddRange(GetItemsInDirectory(searchPath, cancellationToken));
                }

                break;

            case PathKind.Absolute:
            case PathKind.RelativeToCurrentDirectory:
            case PathKind.RelativeToCurrentParent:
            case PathKind.RelativeToCurrentRoot:
                var fullDirectoryPath = FileUtilities.ResolveRelativePath(directoryPath, basePath: null, baseDirectory: _baseDirectoryOpt);
                if (fullDirectoryPath != null)
                {
                    result.AddRange(GetItemsInDirectory(fullDirectoryPath, cancellationToken));
                }
                else
                {
                    // invalid path
                    result.Clear();
                }

                break;

            case PathKind.Relative:

                // base directory:
                if (_baseDirectoryOpt != null)
                {
                    result.AddRange(GetItemsInDirectory(PathUtilities.CombineAbsoluteAndRelativePaths(_baseDirectoryOpt, directoryPath), cancellationToken));
                }

                // search paths:
                foreach (var searchPath in _searchPaths)
                {
                    result.AddRange(GetItemsInDirectory(PathUtilities.CombineAbsoluteAndRelativePaths(searchPath, directoryPath), cancellationToken));
                }

                break;

            case PathKind.RelativeToDriveDirectory:
                // Paths "C:dir" are not supported, but when the path doesn't include any directory, i.e. "C:",
                // we return the drive itself.
                if (directoryPath.Length == 2)
                {
                    result.Add(CreateLogicalDriveItem(directoryPath));
                }

                break;

            default:
                throw ExceptionUtilities.UnexpectedValue(pathKind);
            }

            return(result.ToImmutableAndFree());
        }
Beispiel #15
0
        private List <string> PrepForWrite()
        {
            // If the user did not specify a place to put log messages, make one for them.
            if (Log == null)
            {
                Log = new StringWriter();
            }

            Stopwatch sw = Stopwatch.StartNew();

            // Compute input & temp files.
            var dir = Path.GetDirectoryName(m_etlFilePath);

            if (dir.Length == 0)
            {
                dir = ".";
            }
            var           baseName    = Path.GetFileNameWithoutExtension(m_etlFilePath);
            List <string> mergeInputs = new List <string>();

            mergeInputs.Add(m_etlFilePath);
            mergeInputs.AddRange(Directory.GetFiles(dir, baseName + ".kernel*.etl"));
            mergeInputs.AddRange(Directory.GetFiles(dir, baseName + ".clr*.etl"));
            mergeInputs.AddRange(Directory.GetFiles(dir, baseName + ".user*.etl"));
            string        tempName    = Path.ChangeExtension(m_etlFilePath, ".etl.new");
            List <string> pdbFileList = null;

            try
            {
                // Do the merge and NGEN pdb lookup in parallel
                Task mergeWorker = Task.Factory.StartNew(delegate
                {
                    if (Merge)
                    {
                        var startTime = DateTime.UtcNow;
                        Log.WriteLine("Starting Merging of {0}", m_etlFilePath);

                        TraceEventMergeOptions options = TraceEventMergeOptions.None;
                        if (CompressETL)
                        {
                            options |= TraceEventMergeOptions.Compress;
                        }

                        // Do the merge
                        TraceEventSession.Merge(mergeInputs.ToArray(), tempName, options);
                        Log.WriteLine("Merging took {0:f1} sec", (DateTime.UtcNow - startTime).TotalSeconds);
                    }
                    else
                    {
                        Log.WriteLine("Merge == false, skipping Merge operation.");
                    }
                });

                Task pdbWorker = Task.Factory.StartNew(delegate
                {
                    if (NGenSymbolFiles)
                    {
                        var startTime = DateTime.UtcNow;
                        Log.WriteLine("Starting Generating NGEN pdbs for {0}", m_etlFilePath);
                        var symbolReader = SymbolReader;
                        if (symbolReader == null)
                        {
                            symbolReader = new SymbolReader(Log);
                        }
                        pdbFileList = GetNGenPdbs(m_etlFilePath, symbolReader, Log);
                        Log.WriteLine("Generating NGEN Pdbs took {0:f1} sec", (DateTime.UtcNow - startTime).TotalSeconds);
                    }
                    else
                    {
                        Log.WriteLine("NGenSymbolFiles == false, skipping NGEN pdb generation");
                    }
                });
                Task.WaitAll(mergeWorker, pdbWorker);

                if (File.Exists(tempName))
                {
                    // Delete/move the original files after the two worker threads finished execution to avoid races.
                    foreach (var mergeInput in mergeInputs)
                    {
                        FileUtilities.ForceDelete(mergeInput);
                    }

                    Log.WriteLine("Moving {0} to {1}", tempName, m_etlFilePath);
                    // Place the output in its final resting place.
                    FileUtilities.ForceMove(tempName, m_etlFilePath);
                }
            }
            finally
            {
                Log.WriteLine("Deleting temp file");
                if (File.Exists(tempName))
                {
                    File.Delete(tempName);
                }
            }

            sw.Stop();
            Log.WriteLine("Merge took {0:f3} sec.", sw.Elapsed.TotalSeconds);
            Log.WriteLine("Merge output file {0}", m_etlFilePath);

            return(pdbFileList);
        }
 /// <summary>
 /// Searches upward for a directory containing the specified file, beginning in the specified directory.
 /// </summary>
 /// <param name="startingDirectory">The directory to start the search in.</param>
 /// <param name="fileName">The name of the file to search for.</param>
 /// <param name="fileSystem">The file system abstraction to use that implements file system operations</param>
 /// <returns>The full path of the directory containing the file if it is found, otherwise an empty string. </returns>
 internal static string GetDirectoryNameOfFileAbove(string startingDirectory, string fileName, IFileSystem fileSystem)
 {
     return(FileUtilities.GetDirectoryNameOfFileAbove(startingDirectory, fileName, fileSystem));
 }
Beispiel #17
0
        /// <summary>
        /// After setting any properties to override default behavior, calling this method
        /// will actually do the unpacking.
        /// </summary>
        public void UnpackAchive()
        {
            if (Log == null)
            {
                Log = new StringWriter();
            }

            if (EtlFileName == null)
            {
                if (m_zipFilePath.EndsWith(".etl.zip", StringComparison.OrdinalIgnoreCase))
                {
                    EtlFileName = m_zipFilePath.Substring(0, m_zipFilePath.Length - 4);
                }
                else
                {
                    EtlFileName = Path.ChangeExtension(m_zipFilePath, ".etl");
                }
            }

            if (SymbolDirectory == null)
            {
                SymbolDirectory = new SymbolPath(SymbolPath.SymbolPathFromEnvironment).DefaultSymbolCache();
            }

            Stopwatch sw = Stopwatch.StartNew();

            Log.WriteLine("[Decompressing {0}]", m_zipFilePath);
            Log.WriteLine("Generating output file {0}", EtlFileName);
            using (var zipArchive = ZipFile.OpenRead(m_zipFilePath))
            {
                bool seenEtlFile = false;
                foreach (var entry in zipArchive.Entries)
                {
                    if (entry.Length == 0)  // Skip directories.
                    {
                        continue;
                    }

                    var archivePath = entry.FullName;
                    if (archivePath.EndsWith(".pdb", StringComparison.OrdinalIgnoreCase))
                    {
                        archivePath = archivePath.Replace('/', '\\');     // normalize separator convention
                        string pdbRelativePath = null;
                        if (archivePath.StartsWith(@"symbols\", StringComparison.OrdinalIgnoreCase))
                        {
                            pdbRelativePath = archivePath.Substring(8);
                        }
                        else if (archivePath.StartsWith(@"ngenpdbs\", StringComparison.OrdinalIgnoreCase))
                        {
                            pdbRelativePath = archivePath.Substring(9);
                        }
                        else
                        {
                            var m = Regex.Match(archivePath, @"^[^\\]+\.ngenpdbs?\\(.*)", RegexOptions.IgnoreCase);
                            if (m.Success)
                            {
                                pdbRelativePath = m.Groups[1].Value;
                            }
                            else
                            {
                                // .diagsession files (created by the Visual Studio Diagnostic Hub) put PDBs in a path like
                                // 194BAE98-C4ED-470E-9204-1F9389FC9DC1\symcache\xyz.pdb
                                m = Regex.Match(archivePath, @"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\\symcache\\(.*)", RegexOptions.IgnoreCase);
                                if (m.Success)
                                {
                                    pdbRelativePath = m.Groups[1].Value;
                                }
                                else
                                {
                                    Log.WriteLine("WARNING: found PDB file that was not in a symbol server style directory, skipping extraction");
                                    Log.WriteLine("         Unzip this ETL and PDB by hand to use this PDB.");
                                    continue;
                                }
                            }
                        }

                        var pdbTargetPath = Path.Combine(SymbolDirectory, pdbRelativePath);
                        var pdbTargetName = Path.GetFileName(pdbTargetPath);
                        if (File.Exists(pdbTargetPath) && (new System.IO.FileInfo(pdbTargetPath).Length == entry.Length))
                        {
                            Log.WriteLine("PDB {0} exists, skipping", pdbRelativePath);
                            continue;
                        }

                        // There is a possibilty that you want to put symbol file using symbol server conventions
                        // (in which case it is X.pdb\NNNN\X.pdb, but you already have a file named X.pdb)  detect
                        // this and delete the file if necessary.
                        var firstNameInRelativePath = pdbRelativePath;
                        var sepIdx = firstNameInRelativePath.IndexOf('\\');
                        if (sepIdx >= 0)
                        {
                            firstNameInRelativePath = firstNameInRelativePath.Substring(0, sepIdx);
                        }
                        var firstNamePath = Path.Combine(SymbolDirectory, firstNameInRelativePath);
                        if (File.Exists(firstNamePath))
                        {
                            Log.WriteLine("Deleting pdb file that is in the way {0}", firstNamePath);
                            FileUtilities.ForceDelete(firstNamePath);
                        }

                        Log.WriteLine("Extracting PDB {0}", pdbRelativePath);
                        AtomicExtract(entry, pdbTargetPath);
                    }
                    else if (archivePath.EndsWith(".etl", StringComparison.OrdinalIgnoreCase))
                    {
                        if (seenEtlFile)
                        {
                            throw new ApplicationException("The ZIP file does not have exactly 1 ETL file in it, can't auto-extract.");
                        }
                        seenEtlFile = true;
                        AtomicExtract(entry, EtlFileName);
                        Log.WriteLine("Extracting {0} Zipped size = {1:f3} MB Unzipped = {2:f3} MB", EtlFileName,
                                      entry.CompressedLength / 1000000.0, entry.Length / 1000000.0);
                    }
                    else if (archivePath == "LogFile.txt")
                    {
                        string logFilePath = Path.ChangeExtension(EtlFileName, ".LogFile.txt");
                        Log.WriteLine("Extracting LogFile.txt to {0}", logFilePath);
                        AtomicExtract(entry, logFilePath);
                    }
                    else
                    {
                        Log.WriteLine("Skipping unknown file {0}", archivePath);
                        // TODO do something with these?
                    }
                }
                if (!seenEtlFile)
                {
                    throw new ApplicationException("The ZIP file does not have any ETL files in it!");
                }
                Log.WriteLine("Finished decompression, took {0:f0} sec", sw.Elapsed.TotalSeconds);
            }
        }
 /// <summary>
 /// Searches upward for the specified file, beginning in the specified <see cref="IElementLocation"/>.
 /// </summary>
 /// <param name="file">The name of the file to search for.</param>
 /// <param name="startingDirectory">An optional directory to start the search in.  The default location is the directory
 /// <param name="fileSystem">The file system abstraction to use that implements file system operations</param>
 /// of the file containing the property function.</param>
 /// <returns>The full path of the file if it is found, otherwise an empty string.</returns>
 internal static string GetPathOfFileAbove(string file, string startingDirectory, IFileSystem fileSystem)
 {
     return(FileUtilities.GetPathOfFileAbove(file, startingDirectory, fileSystem));
 }
Beispiel #19
0
        /// <summary>
        /// Execute the task.
        /// </summary>
        public override bool Execute()
        {
            if (SuggestedRedirects == null || SuggestedRedirects.Length == 0)
            {
                Log.LogMessageFromResources("GenerateBindingRedirects.NoSuggestedRedirects");
                OutputAppConfigFile = null;
                return(true);
            }

            var redirects = ParseSuggestedRedirects();

            var doc = LoadAppConfig(AppConfigFile);

            if (doc == null)
            {
                return(false);
            }

            XElement runtimeNode = doc.Root
                                   .Nodes()
                                   .OfType <XElement>()
                                   .FirstOrDefault(e => e.Name.LocalName == "runtime");

            if (runtimeNode == null)
            {
                runtimeNode = new XElement("runtime");
                doc.Root.Add(runtimeNode);
            }
            else
            {
                UpdateExistingBindingRedirects(runtimeNode, redirects);
            }

            var ns = XNamespace.Get("urn:schemas-microsoft-com:asm.v1");

            var redirectNodes = from redirect in redirects
                                select new XElement(
                ns + "assemblyBinding",
                new XElement(
                    ns + "dependentAssembly",
                    new XElement(
                        ns + "assemblyIdentity",
                        new XAttribute("name", redirect.Key.Name),
                        new XAttribute("publicKeyToken", ResolveAssemblyReference.ByteArrayToString(redirect.Key.GetPublicKeyToken())),
                        new XAttribute("culture", String.IsNullOrEmpty(redirect.Key.CultureName) ? "neutral" : redirect.Key.CultureName)),
                    new XElement(
                        ns + "bindingRedirect",
                        new XAttribute("oldVersion", "0.0.0.0-" + redirect.Value),
                        new XAttribute("newVersion", redirect.Value))));

            runtimeNode.Add(redirectNodes);

            var writeOutput = true;

            if (FileSystems.Default.FileExists(OutputAppConfigFile.ItemSpec))
            {
                try
                {
                    var outputDoc = LoadAppConfig(OutputAppConfigFile);
                    if (outputDoc.ToString() == doc.ToString())
                    {
                        writeOutput = false;
                    }
                }
                catch (System.Xml.XmlException)
                {
                    writeOutput = true;
                }
            }

            if (AppConfigFile != null)
            {
                AppConfigFile.CopyMetadataTo(OutputAppConfigFile);
            }
            else
            {
                OutputAppConfigFile.SetMetadata(ItemMetadataNames.targetPath, TargetName);
            }

            if (writeOutput)
            {
                using (var stream = FileUtilities.OpenWrite(OutputAppConfigFile.ItemSpec, false))
                {
                    doc.Save(stream);
                }
            }

            return(!Log.HasLoggedErrors);
        }
 /// <summary>
 /// If the given path doesn't have a trailing slash then add one.
 /// If the path is an empty string, does not modify it.
 /// </summary>
 /// <param name="path">The path to check.</param>
 /// <returns>The specified path with a trailing slash.</returns>
 internal static string EnsureTrailingSlash(string path)
 {
     return(FileUtilities.EnsureTrailingSlash(path));
 }
Beispiel #21
0
        public static void FinalizeTLogs(bool trackedOperationsSucceeded, ITaskItem[] readTLogNames, ITaskItem[] writeTLogNames, ITaskItem[] trackedFilesToRemoveFromTLogs)
        {
            // Read the input table, skipping missing files
            FlatTrackingData inputs = new FlatTrackingData(readTLogNames, true);

            // Read the output table, skipping missing files
            FlatTrackingData outputs = new FlatTrackingData(writeTLogNames, true);


            // If we failed we need to clean the Tlogs
            if (!trackedOperationsSucceeded)
            {
                // If the tool errors in some way, we assume that any and all inputs and outputs it wrote during
                // execution are wrong. So we compact the read and write tlogs to remove the entries for the
                // set of sources being compiled - the next incremental build will find no entries
                // and correctly cause the sources to be compiled
                // Remove all from inputs tlog
                inputs.DependencyTable.Clear();
                inputs.SaveTlog();

                // Remove all from outputs tlog
                outputs.DependencyTable.Clear();
                outputs.SaveTlog();
            }
            else
            {
                // If all went well with the tool execution, then compact the tlogs
                // to remove any files that are no longer on disk.
                // This removes any temporary files from the dependency graph

                // In addition to temporary file removal, an optional set of files to remove may be been supplied

                if (trackedFilesToRemoveFromTLogs != null && trackedFilesToRemoveFromTLogs.Length > 0)
                {
                    IDictionary <string, ITaskItem> trackedFilesToRemove = new Dictionary <string, ITaskItem>(StringComparer.OrdinalIgnoreCase);

                    foreach (ITaskItem removeFile in trackedFilesToRemoveFromTLogs)
                    {
                        trackedFilesToRemove.Add(FileUtilities.NormalizePath(removeFile.ItemSpec), removeFile);
                    }

                    // UNDONE: If necessary we could have two independent sets of "ignore" files, one for inputs and one for outputs
                    // Use an anonymous method to encapsulate the contains check for the output tlogs
                    outputs.SaveTlog(delegate(string fullTrackedPath)
                    {
                        // We need to answer the question "should fullTrackedPath be included in the TLog?"
                        return(!trackedFilesToRemove.ContainsKey(fullTrackedPath));
                    }
                                     );

                    // Use an anonymous method to encapsulate the contains check for the input tlogs
                    inputs.SaveTlog(delegate(string fullTrackedPath)
                    {
                        // We need to answer the question "should fullTrackedPath be included in the TLog?"
                        return(!trackedFilesToRemove.ContainsKey(fullTrackedPath));
                    }
                                    );
                }
                else
                {
                    // Compact the write tlog
                    outputs.SaveTlog();

                    // Compact the read tlog
                    inputs.SaveTlog();
                }
            }
        }
 /// <summary>
 /// Gets the canonicalized full path of the provided path and ensures it contains the correct directory separator characters for the current operating system.
 /// </summary>
 /// <param name="path">One or more paths to combine and normalize.</param>
 /// <returns>A canonicalized full path with the correct directory separators.</returns>
 internal static string NormalizePath(params string[] path)
 {
     return(FileUtilities.NormalizePath(path));
 }
Beispiel #23
0
 public new void RemoveMetadataReference(string referencePath)
 {
     referencePath = FileUtilities.NormalizeAbsolutePath(referencePath);
     base.RemoveMetadataReference(referencePath);
 }
Beispiel #24
0
        // Resolves any conflicts for the given output path.
        // Returns a non-conflicting output path.
        // May return the same value if there are no conflicts.
        // null means cancel.
        public string ResolveOutputPathConflicts(string initialOutputPath, HashSet <string> excludedPaths, bool isBatch)
        {
            HashSet <string> queuedFiles = excludedPaths;
            bool?            conflict    = Utilities.FileExists(initialOutputPath, queuedFiles);

            if (conflict == null)
            {
                return(initialOutputPath);
            }

            WhenFileExists preference;

            if (isBatch)
            {
                preference = CustomConfig.WhenFileExistsBatch;
            }
            else
            {
                preference = CustomConfig.WhenFileExists;
            }

            switch (preference)
            {
            case WhenFileExists.Prompt:
                break;

            case WhenFileExists.Overwrite:
                return(initialOutputPath);

            case WhenFileExists.AutoRename:
                return(FileUtilities.CreateUniqueFileName(initialOutputPath, queuedFiles));

            default:
                throw new ArgumentOutOfRangeException();
            }

            // Continue and prompt user for resolution
            string dialogMessageTemplate;

            if ((bool)conflict)
            {
                dialogMessageTemplate = MiscRes.FileConflictWarning;
            }
            else
            {
                dialogMessageTemplate = MiscRes.QueueFileConflictWarning;
            }

            string dialogMessage  = string.Format(dialogMessageTemplate, initialOutputPath);
            var    conflictDialog = new CustomMessageDialogViewModel <FileConflictResolution>(
                MiscRes.FileConflictDialogTitle,
                dialogMessage,
                new List <CustomDialogButton <FileConflictResolution> >
            {
                new CustomDialogButton <FileConflictResolution>(FileConflictResolution.Overwrite, MiscRes.OverwriteButton, ButtonType.Default),
                new CustomDialogButton <FileConflictResolution>(FileConflictResolution.AutoRename, MiscRes.AutoRenameButton),
                new CustomDialogButton <FileConflictResolution>(FileConflictResolution.Cancel, CommonRes.Cancel, ButtonType.Cancel),
            });

            Ioc.Get <IWindowManager>().OpenDialog(conflictDialog);

            switch (conflictDialog.Result)
            {
            case FileConflictResolution.Cancel:
                return(null);

            case FileConflictResolution.Overwrite:
                return(initialOutputPath);

            case FileConflictResolution.AutoRename:
                return(FileUtilities.CreateUniqueFileName(initialOutputPath, queuedFiles));

            default:
                throw new ArgumentOutOfRangeException();
            }
        }
        private static void LoadWorldDirect(bool loadFromCloud)
        {
            WorldFile.IsWorldOnCloud = loadFromCloud;
            Main.checkXMas();
            Main.checkHalloween();
            bool flag = loadFromCloud && SocialAPI.Cloud != null;

            if (!FileUtilities.Exists(Main.worldPathName, flag) && Main.autoGen)
            {
                if (!flag)
                {
                    for (int i = Main.worldPathName.Length - 1; i >= 0; i--)
                    {
                        if (Main.worldPathName.Substring(i, 1) == (Path.DirectorySeparatorChar.ToString() ?? ""))
                        {
                            Directory.CreateDirectory(Main.worldPathName.Substring(0, i));
                            break;
                        }
                    }
                }
                WorldGen.clearWorld();
                Main.ActiveWorldFileData = WorldFile.CreateMetadata((Main.worldName == "") ? "World" : Main.worldName, flag, Main.expertMode);
                string text = (Main.AutogenSeedName ?? "").Trim();
                if (text.Length == 0)
                {
                    Main.ActiveWorldFileData.SetSeedToRandom();
                }
                else
                {
                    Main.ActiveWorldFileData.SetSeed(text);
                }
                WorldGen.generateWorld(Main.ActiveWorldFileData.Seed, Main.AutogenProgress);
                WorldFile.saveWorld();
            }
            using (MemoryStream memoryStream = new MemoryStream(FileUtilities.ReadAllBytes(Main.worldPathName, flag)))
            {
                using (BinaryReader binaryReader = new BinaryReader(memoryStream))
                {
                    try
                    {
                        WorldGen.loadFailed  = false;
                        WorldGen.loadSuccess = false;
                        int num = WorldFile.versionNumber = binaryReader.ReadInt32();
                        int num2;
                        if (num <= 87)
                        {
                            // Not supported
                            num2 = WorldFile.LoadWorld_Version1(binaryReader);
                        }
                        else
                        {
                            num2 = LoadWorld_Version2(binaryReader);
                        }
                        if (num < 141)
                        {
                            if (!loadFromCloud)
                            {
                                Main.ActiveWorldFileData.CreationTime = File.GetCreationTime(Main.worldPathName);
                            }
                            else
                            {
                                Main.ActiveWorldFileData.CreationTime = DateTime.Now;
                            }
                        }
                        binaryReader.Close();
                        memoryStream.Close();
                        if (num2 != 0)
                        {
                            WorldGen.loadFailed = true;
                        }
                        else
                        {
                            WorldGen.loadSuccess = true;
                        }
                        if (WorldGen.loadFailed || !WorldGen.loadSuccess)
                        {
                            return;
                        }
                        WorldGen.gen       = true;
                        WorldGen.waterLine = Main.maxTilesY;
                        Liquid.QuickWater(2, -1, -1);
                        WorldGen.WaterCheck();
                        int num3 = 0;
                        Liquid.quickSettle = true;
                        int   num4 = Liquid.numLiquid + LiquidBuffer.numLiquidBuffer;
                        float num5 = 0f;
                        while (Liquid.numLiquid > 0 && num3 < 100000)
                        {
                            num3++;
                            float num6 = (float)(num4 - (Liquid.numLiquid + LiquidBuffer.numLiquidBuffer)) / (float)num4;
                            if (Liquid.numLiquid + LiquidBuffer.numLiquidBuffer > num4)
                            {
                                num4 = Liquid.numLiquid + LiquidBuffer.numLiquidBuffer;
                            }
                            if (num6 > num5)
                            {
                                num5 = num6;
                            }
                            else
                            {
                                num6 = num5;
                            }

                            SetStatusText(string.Concat(new object[]
                            {
                                Lang.gen[27].Value,
                                " ",
                                (int)(num6 * 100f / 2f + 50f),
                                "%"
                            }));
                            Liquid.UpdateLiquid();
                        }
                        Liquid.quickSettle  = false;
                        Main.weatherCounter = WorldGen.genRand.Next(3600, 18000);
                        Cloud.resetClouds();
                        WorldGen.WaterCheck();
                        WorldGen.gen = false;
                        NPC.setFireFlyChance();
                        Main.InitLifeBytes();
                        if (Main.slimeRainTime > 0.0)
                        {
                            Main.StartSlimeRain(false);
                        }
                        NPC.setWorldMonsters();
                    }
                    catch (Exception value)
                    {
                        WorldGen.loadFailed  = true;
                        WorldGen.loadSuccess = false;
                        System.Console.WriteLine(value);
                        try
                        {
                            binaryReader.Close();
                            memoryStream.Close();
                        }
                        catch
                        {
                        }
                        return;
                    }
                }
            }

            EventInfo eventOnWorldLoad = typeof(WorldFile).GetEvent("OnWorldLoad", BindingFlags.Public | BindingFlags.Static);

            eventOnWorldLoad.GetRaiseMethod()?.Invoke(null, new object[] { });
            //if (WorldFile.OnWorldLoad != null)
            //WorldFile.OnWorldLoad();
        }
Beispiel #26
0
        private bool RemoveExtraneousFilesAndDirectories(
            Func <string, bool> isPathInBuild,
            List <string> pathsToScrub,
            HashSet <string> blockedPaths,
            HashSet <string> nonDeletableRootDirectories,
            MountPathExpander mountPathExpander,
            bool logRemovedFiles)
        {
            int directoriesEncountered        = 0;
            int filesEncountered              = 0;
            int filesRemoved                  = 0;
            int directoriesRemovedRecursively = 0;

            using (var pm = PerformanceMeasurement.Start(
                       m_loggingContext,
                       Category,
                       // The start of the scrubbing is logged before calling this function, since there are two sources of scrubbing (regular scrubbing and shared opaque scrubbing)
                       // with particular messages
                       (_ => {}),
                       loggingContext =>
            {
                Tracing.Logger.Log.ScrubbingFinished(loggingContext, directoriesEncountered, filesEncountered, filesRemoved, directoriesRemovedRecursively);
                Logger.Log.BulkStatistic(
                    loggingContext,
                    new Dictionary <string, long>
                {
                    [I($"{Category}.DirectoriesEncountered")] = directoriesEncountered,
                    [I($"{Category}.FilesEncountered")] = filesEncountered,
                    [I($"{Category}.FilesRemoved")] = filesRemoved,
                    [I($"{Category}.DirectoriesRemovedRecursively")] = directoriesRemovedRecursively,
                });
            }))
                using (var timer = new Timer(
                           o =>
                {
                    // We don't have a good proxy for how much scrubbing is left. Instead we use the file counters to at least show progress
                    Tracing.Logger.Log.ScrubbingStatus(m_loggingContext, filesEncountered);
                },
                           null,
                           dueTime: BuildXLEngine.GetTimerUpdatePeriodInMs(m_loggingConfiguration),
                           period: BuildXLEngine.GetTimerUpdatePeriodInMs(m_loggingConfiguration)))
                {
                    var deletableDirectoryCandidates = new ConcurrentDictionary <string, bool>(StringComparer.OrdinalIgnoreCase);
                    var nondeletableDirectories      = new ConcurrentDictionary <string, bool>(StringComparer.OrdinalIgnoreCase);
                    var directoriesToEnumerate       = new BlockingCollection <string>();

                    foreach (var path in pathsToScrub)
                    {
                        SemanticPathInfo foundSemanticPathInfo;

                        if (blockedPaths.Contains(path))
                        {
                            continue;
                        }

                        if (ValidateDirectory(mountPathExpander, path, out foundSemanticPathInfo))
                        {
                            if (!isPathInBuild(path))
                            {
                                directoriesToEnumerate.Add(path);
                            }
                            else
                            {
                                nondeletableDirectories.TryAdd(path, true);
                            }
                        }
                        else
                        {
                            string mountName = "Invalid";
                            string mountPath = "Invalid";

                            if (mountPathExpander != null && foundSemanticPathInfo.IsValid)
                            {
                                mountName = foundSemanticPathInfo.RootName.ToString(mountPathExpander.PathTable.StringTable);
                                mountPath = foundSemanticPathInfo.Root.ToString(mountPathExpander.PathTable);
                            }

                            Tracing.Logger.Log.ScrubbingFailedBecauseDirectoryIsNotScrubbable(pm.LoggingContext, path, mountName, mountPath);
                        }
                    }

                    var cleaningThreads = new Thread[m_maxDegreeParallelism];
                    int pending         = directoriesToEnumerate.Count;

                    if (directoriesToEnumerate.Count == 0)
                    {
                        directoriesToEnumerate.CompleteAdding();
                    }

                    for (int i = 0; i < m_maxDegreeParallelism; i++)
                    {
                        var t = new Thread(() =>
                        {
                            while (!directoriesToEnumerate.IsCompleted && !m_cancellationToken.IsCancellationRequested)
                            {
                                string currentDirectory;
                                if (directoriesToEnumerate.TryTake(out currentDirectory, Timeout.Infinite))
                                {
                                    Interlocked.Increment(ref directoriesEncountered);
                                    bool shouldDeleteCurrentDirectory = true;

                                    var result = FileUtilities.EnumerateDirectoryEntries(
                                        currentDirectory,
                                        false,
                                        (dir, fileName, attributes) =>
                                    {
                                        string fullPath = Path.Combine(dir, fileName);

                                        // Skip specifically blocked paths.
                                        if (blockedPaths.Contains(fullPath))
                                        {
                                            shouldDeleteCurrentDirectory = false;
                                            return;
                                        }

                                        // important to not follow directory symlinks because that can cause
                                        // re-enumerating and scrubbing the same physical folder multiple times
                                        if (FileUtilities.IsDirectoryNoFollow(attributes))
                                        {
                                            if (nondeletableDirectories.ContainsKey(fullPath))
                                            {
                                                shouldDeleteCurrentDirectory = false;
                                            }

                                            if (!isPathInBuild(fullPath))
                                            {
                                                // Current directory is not in the build, then recurse to its members.
                                                Interlocked.Increment(ref pending);
                                                directoriesToEnumerate.Add(fullPath);

                                                if (!nonDeletableRootDirectories.Contains(fullPath))
                                                {
                                                    // Current directory can be deleted, then it is a candidate to be deleted.
                                                    deletableDirectoryCandidates.TryAdd(fullPath, true);
                                                }
                                                else
                                                {
                                                    // Current directory can't be deleted (e.g., the root of a mount), then don't delete it.
                                                    // However, note that we recurse to its members to find all extraneous directories and files.
                                                    shouldDeleteCurrentDirectory = false;
                                                }
                                            }
                                            else
                                            {
                                                // Current directory is in the build, i.e., directory is an output directory.
                                                // Stop recursive directory traversal because none of its members should be deleted.
                                                shouldDeleteCurrentDirectory = false;
                                            }
                                        }
                                        else
                                        {
                                            Interlocked.Increment(ref filesEncountered);

                                            if (!isPathInBuild(fullPath))
                                            {
                                                // File is not in the build, delete it.
                                                if (TryDeleteFile(pm.LoggingContext, fullPath, logRemovedFiles))
                                                {
                                                    Interlocked.Increment(ref filesRemoved);
                                                }
                                            }
                                            else
                                            {
                                                // File is in the build, then don't delete it, but mark the current directory that
                                                // it should not be deleted.
                                                shouldDeleteCurrentDirectory = false;
                                            }
                                        }
                                    });

                                    if (!result.Succeeded)
                                    {
                                        // Different trace levels based on result.
                                        if (result.Status != EnumerateDirectoryStatus.SearchDirectoryNotFound)
                                        {
                                            Tracing.Logger.Log.ScrubbingFailedToEnumerateDirectory(
                                                pm.LoggingContext,
                                                currentDirectory,
                                                result.Status.ToString());
                                        }
                                    }

                                    if (!shouldDeleteCurrentDirectory)
                                    {
                                        // If directory should not be deleted, then all of its parents should not be deleted.
                                        int index;
                                        string preservedDirectory = currentDirectory;
                                        bool added;

                                        do
                                        {
                                            added = nondeletableDirectories.TryAdd(preservedDirectory, true);
                                        }while (added &&
                                                (index = preservedDirectory.LastIndexOf(Path.DirectorySeparatorChar)) != -1 &&
                                                !string.IsNullOrEmpty(preservedDirectory = preservedDirectory.Substring(0, index)));
                                    }

                                    Interlocked.Decrement(ref pending);
                                }

                                if (Volatile.Read(ref pending) == 0)
                                {
                                    directoriesToEnumerate.CompleteAdding();
                                }
                            }
                        });
                        t.Start();
                        cleaningThreads[i] = t;
                    }

                    foreach (var t in cleaningThreads)
                    {
                        t.Join();
                    }

                    // Collect all directories that need to be deleted.
                    var deleteableDirectories = new HashSet <string>(deletableDirectoryCandidates.Keys, StringComparer.OrdinalIgnoreCase);
                    deleteableDirectories.ExceptWith(nondeletableDirectories.Keys);

                    // Delete directories by considering only the top-most ones.
                    try
                    {
                        Parallel.ForEach(
                            CollapsePaths(deleteableDirectories).ToList(),
                            new ParallelOptions
                        {
                            MaxDegreeOfParallelism = m_maxDegreeParallelism,
                            CancellationToken      = m_cancellationToken,
                        },
                            directory =>
                        {
                            try
                            {
                                FileUtilities.DeleteDirectoryContents(directory, deleteRootDirectory: true, tempDirectoryCleaner: m_tempDirectoryCleaner);
                                Interlocked.Increment(ref directoriesRemovedRecursively);
                            }
                            catch (BuildXLException ex)
                            {
                                Tracing.Logger.Log.ScrubbingExternalFileOrDirectoryFailed(
                                    pm.LoggingContext,
                                    directory,
                                    ex.LogEventMessage);
                            }
                        });
                    }
                    catch (OperationCanceledException) { }
                    return(true);
                }
        }
 /// <summary>
 /// Writes a bool to the file
 /// </summary>
 /// <returns>Number of bytes used to store the bool</returns>
 internal int WriteBoolean(bool val)
 {
     byteBuffer[0] = 0x01; // length
     byteBuffer[1] = (byte)(val ? 0x01 : 0x00);
     return(FileUtilities.WriteWithLength(fileStream, byteBuffer, 2));
 }
Beispiel #28
0
 /// <summary>
 /// Generates a random directory name for the profile.
 /// </summary>
 /// <returns>A random directory name for the profile.</returns>
 private static string GenerateProfileDirectoryName()
 {
     return(FileUtilities.GenerateRandomTempDirectoryName("anonymous.{0}.webdriver-profile"));
 }
Beispiel #29
0
 public void Save()
 {
     FileUtilities.WriteAllBytes(this.Path, Encoding.ASCII.GetBytes(JsonConvert.SerializeObject((object)this._data, (Formatting)1)), this.IsCloudSave);
 }
Beispiel #30
0
 public TransientTestFolder()
 {
     FolderPath = FileUtilities.GetTemporaryDirectory();
 }