/// <summary> /// Actually sign all of the described files. /// </summary> private bool SignFiles() { // Generate the list of signed files in a deterministic order. Makes it easier to track down // bugs if repeated runs use the same ordering. var toSignList = _batchData.FilesToSign.ToList(); var round = 0; var signedSet = new HashSet <ImmutableArray <byte> >(ByteSequenceComparer.Instance); bool signFiles(IEnumerable <FileSignInfo> files) { var filesToSign = files.Where(fileInfo => fileInfo.SignInfo.ShouldSign).ToArray(); _log.LogMessage(MessageImportance.High, $"Signing Round {round}: {filesToSign.Length} files to sign."); if (filesToSign.Length == 0) { return(true); } foreach (var file in filesToSign) { _log.LogMessage(MessageImportance.Low, file.ToString()); } return(_signTool.Sign(_buildEngine, round, filesToSign)); } void repackFiles(IEnumerable <FileSignInfo> files) { foreach (var file in files) { if (file.IsZipContainer()) { _log.LogMessage($"Repacking container: '{file.FileName}'"); _batchData.ZipDataMap[file.ContentHash].Repack(_log); } } } // Is this file ready to be signed? That is are all of the items that it depends on already // signed? bool isReadyToSign(FileSignInfo file) { if (!file.IsZipContainer()) { return(true); } var zipData = _batchData.ZipDataMap[file.ContentHash]; return(zipData.NestedParts.All(x => !x.FileSignInfo.SignInfo.ShouldSign || signedSet.Contains(x.FileSignInfo.ContentHash))); } // Extract the next set of files that should be signed. This is the set of files for which all of the // dependencies have been signed. List <FileSignInfo> extractNextGroup() { var list = new List <FileSignInfo>(); var i = 0; while (i < toSignList.Count) { var current = toSignList[i]; if (isReadyToSign(current)) { list.Add(current); toSignList.RemoveAt(i); } else { i++; } } return(list); } while (toSignList.Count > 0) { var list = extractNextGroup(); if (list.Count == 0) { throw new InvalidOperationException("No progress made on signing which indicates a bug"); } repackFiles(list); if (!signFiles(list)) { return(false); } round++; list.ForEach(x => signedSet.Add(x.ContentHash)); } return(true); }
/// <summary> /// Actually sign all of the described files. /// </summary> private bool SignFiles() { // Generate the list of signed files in a deterministic order. Makes it easier to track down // bugs if repeated runs use the same ordering. var toProcessList = _batchData.FilesToSign.ToList(); var toRepackList = _batchData.FilesToSign.Where(x => x.ShouldRepack)?.Select(x => x.FullPath)?.ToList(); var round = 0; var trackedSet = new HashSet <SignedFileContentKey>(); bool signFiles(IEnumerable <FileSignInfo> files, out int totalFilesSigned) { var filesToSign = files.Where(fileInfo => fileInfo.SignInfo.ShouldSign).ToArray(); totalFilesSigned = filesToSign.Length; _log.LogMessage(MessageImportance.High, $"Signing Round {round}: {filesToSign.Length} files to sign."); if (filesToSign.Length == 0) { return(true); } foreach (var file in filesToSign) { _log.LogMessage(MessageImportance.Low, file.ToString()); } return(_signTool.Sign(_buildEngine, round, filesToSign)); } bool signEngines(IEnumerable <FileSignInfo> files, out int totalFilesSigned) { var enginesToSign = files.Where(fileInfo => fileInfo.SignInfo.ShouldSign && fileInfo.IsWixContainer() && Path.GetExtension(fileInfo.FullPath) == ".exe").ToArray(); totalFilesSigned = enginesToSign.Length; if (enginesToSign.Length == 0) { return(true); } Dictionary <string, FileSignInfo> engines = new Dictionary <string, FileSignInfo>(); var workingDirectory = Path.Combine(_signTool.TempDir, "engines"); // extract engines foreach (var file in enginesToSign) { string engineFileName = $"{Path.Combine(workingDirectory, file.FileName)}{SignToolConstants.MsiEngineExtension}"; _log.LogMessage(MessageImportance.Normal, $"Extracting engine from {file.FullPath}"); int exitCode = RunWixTool("insignia.exe", $"-ib {file.FullPath} -o {engineFileName}", workingDirectory, _signTool.WixToolsPath); if (exitCode != 0) { _log.LogError($"Failed to extract engine from {file.FullPath}"); return(false); } engines.Add(engineFileName, file); } // sign engines bool signResult = _signTool.Sign(_buildEngine, round, engines.Select(engine => new FileSignInfo(engine.Key, engine.Value.ContentHash, engine.Value.SignInfo))); if (!signResult) { _log.LogError($"Failed to sign engines"); return(signResult); } // attach engines foreach (var engine in engines) { _log.LogMessage(MessageImportance.Normal, $"Attaching engine {engine.Key} to {engine.Value.FullPath}"); int exitCode = RunWixTool("insignia.exe", $"-ab {engine.Key} {engine.Value.FullPath} -o {engine.Value.FullPath}", workingDirectory, _signTool.WixToolsPath); // cleanup engines (they fail signing verification if they stay in the drop File.Delete(engine.Key); if (exitCode != 0) { _log.LogError($"Failed to attach engine to {engine.Value.FullPath}"); return(false); } } return(true); } void repackFiles(IEnumerable <FileSignInfo> files) { foreach (var file in files) { if (file.IsZipContainer()) { _log.LogMessage($"Repacking container: '{file.FileName}'"); _batchData.ZipDataMap[file.FileContentKey].Repack(_log); toRepackList.Remove(file.FullPath); } else if (file.IsWixContainer()) { _log.LogMessage($"Packing wix container: '{file.FileName}'"); _batchData.ZipDataMap[file.FileContentKey].Repack(_log, _signTool.TempDir, _signTool.WixToolsPath); toRepackList.Remove(file.FullPath); } } } // Is this file ready to be signed? That is are all of the items that it depends on already // signed? bool isReady(FileSignInfo file) { if (file.IsContainer()) { var zipData = _batchData.ZipDataMap[file.FileContentKey]; return(zipData.NestedParts.All(x => (!x.FileSignInfo.SignInfo.ShouldSign || trackedSet.Contains(x.FileSignInfo.FileContentKey)) && !toRepackList.Contains(x.FileSignInfo.FullPath) )); } return(true); } // Extract the next set of files that should be signed. This is the set of files for which all of the // dependencies have been signed. List <FileSignInfo> extractNextGroup() { var list = new List <FileSignInfo>(); var i = 0; while (i < toProcessList.Count) { var current = toProcessList[i]; if (isReady(current)) { list.Add(current); toProcessList.RemoveAt(i); } else { i++; } } return(list); } while (toProcessList.Count > 0) { var trackList = extractNextGroup(); if (trackList.Count == 0) { throw new InvalidOperationException("No progress made on signing which indicates a bug"); } var repackList = trackList.Where(w => toRepackList.Contains(w.FullPath)); repackFiles(repackList); int totalFilesSigned; if (!signEngines(trackList, out totalFilesSigned)) { return(false); } if (totalFilesSigned > 0) { round++; } if (!signFiles(trackList, out totalFilesSigned)) { return(false); } if (totalFilesSigned > 0) { round++; } trackList.ForEach(x => trackedSet.Add(x.FileContentKey)); } return(true); }
/// <summary> /// Actually sign all of the described files. /// </summary> private bool SignFiles() { // Generate the list of signed files in a deterministic order. Makes it easier to track down // bugs if repeated runs use the same ordering. var toProcessList = _batchData.FilesToSign.ToList(); var toRepackSet = _batchData.FilesToSign.Where(x => x.ShouldRepack)?.Select(x => x.FullPath)?.ToHashSet(); var round = 0; var trackedSet = new HashSet <SignedFileContentKey>(); // Given a list of files that need signing, sign them in a batch. bool signGroup(IEnumerable <FileSignInfo> files, out int signedCount) { var filesToSign = files.Where(fileInfo => fileInfo.SignInfo.ShouldSign).ToArray(); signedCount = filesToSign.Length; if (filesToSign.Length == 0) { return(true); } _log.LogMessage(MessageImportance.High, $"Round {round}: Signing {filesToSign.Length} files."); foreach (var file in filesToSign) { string collisionIdInfo = string.Empty; if (_hashToCollisionIdMap != null) { if (_hashToCollisionIdMap.TryGetValue(file.FileContentKey, out string collisionPriorityId)) { collisionIdInfo = $"Collision Id='{collisionPriorityId}'"; } } _log.LogMessage(MessageImportance.Low, $"{file} {collisionIdInfo}"); } return(_signTool.Sign(_buildEngine, round, filesToSign)); } // Given a list of files that need signing, sign the installer engines // of those that are wix containers. bool signEngines(IEnumerable <FileSignInfo> files, out int signedCount) { var enginesToSign = files.Where(fileInfo => fileInfo.SignInfo.ShouldSign && fileInfo.IsWixContainer() && Path.GetExtension(fileInfo.FullPath) == ".exe").ToArray(); signedCount = enginesToSign.Length; if (enginesToSign.Length == 0) { return(true); } _log.LogMessage(MessageImportance.High, $"Round {round}: Signing {enginesToSign.Length} engines."); Dictionary <SignedFileContentKey, FileSignInfo> engines = new Dictionary <SignedFileContentKey, FileSignInfo>(); var workingDirectory = Path.Combine(_signTool.TempDir, "engines"); int engineContainer = 0; // extract engines foreach (var file in enginesToSign) { string engineFileName = $"{Path.Combine(workingDirectory, $"{engineContainer}", file.FileName)}{SignToolConstants.MsiEngineExtension}"; _log.LogMessage(MessageImportance.Normal, $"Extracting engine from {file.FullPath}"); if (!RunWixTool("insignia.exe", $"-ib {file.FullPath} -o {engineFileName}", workingDirectory, _signTool.WixToolsPath, _log)) { _log.LogError($"Failed to extract engine from {file.FullPath}"); return(false); } var fileUniqueKey = new SignedFileContentKey(file.ContentHash, engineFileName); engines.Add(fileUniqueKey, file); engineContainer++; } // sign engines bool signResult = _signTool.Sign(_buildEngine, round, engines.Select(engine => new FileSignInfo(new PathWithHash(engine.Key.FileName, engine.Value.ContentHash), engine.Value.SignInfo))); if (!signResult) { _log.LogError($"Failed to sign engines"); return(signResult); } // attach engines foreach (var engine in engines) { _log.LogMessage(MessageImportance.Normal, $"Attaching engine {engine.Key.FileName} to {engine.Value.FullPath}"); try { if (!RunWixTool("insignia.exe", $"-ab {engine.Key.FileName} {engine.Value.FullPath} -o {engine.Value.FullPath}", workingDirectory, _signTool.WixToolsPath, _log)) { _log.LogError($"Failed to attach engine to {engine.Value.FullPath}"); return(false); } } finally { // cleanup engines (they fail signing verification if they stay in the drop File.Delete(engine.Key.FileName); } } return(true); } // Given a group of file that are ready for processing, // repack those files that are containers. void repackGroup(IEnumerable <FileSignInfo> files, out int repackCount) { var repackList = files.Where(w => toRepackSet.Contains(w.FullPath)).ToList(); repackCount = repackList.Count(); if (repackCount == 0) { return; } _log.LogMessage(MessageImportance.High, $"Repacking {repackCount} containers."); ParallelOptions parallelOptions = new ParallelOptions(); parallelOptions.MaxDegreeOfParallelism = 16; Parallel.ForEach(repackList, parallelOptions, file => { if (file.IsZipContainer()) { _log.LogMessage($"Repacking container: '{file.FileName}'"); _batchData.ZipDataMap[file.FileContentKey].Repack(_log); } else if (file.IsWixContainer()) { _log.LogMessage($"Packing wix container: '{file.FileName}'"); _batchData.ZipDataMap[file.FileContentKey].Repack(_log, _signTool.TempDir, _signTool.WixToolsPath); } else { _log.LogError($"Don't know how to repack file '{file.FullPath}'"); } toRepackSet.Remove(file.FullPath); }); } // Is this file ready to be signed or repackaged? That is are all of the items that it depends on already // signed, don't need signing, and are repacked. bool isReady(FileSignInfo file) { if (file.IsContainer()) { var zipData = _batchData.ZipDataMap[file.FileContentKey]; return(zipData.NestedParts.Values.All(x => (!x.FileSignInfo.SignInfo.ShouldSign || trackedSet.Contains(x.FileSignInfo.FileContentKey)) && !toRepackSet.Contains(x.FileSignInfo.FullPath) )); } return(true); } // Identify the next set of files that should be signed or repacked. // This is the set of files for which all of the dependencies have been signed, // are already signed, are repacked, etc. List <FileSignInfo> identifyNextGroup() { var list = new List <FileSignInfo>(); var i = 0; while (i < toProcessList.Count) { var current = toProcessList[i]; if (isReady(current)) { list.Add(current); toProcessList.RemoveAt(i); } else { i++; } } return(list); } // Telemetry data double telemetryTotalFilesSigned = 0; double telemetryTotalFilesRepacked = 0; Stopwatch telemetrySignedTime = new Stopwatch(); Stopwatch telemetryRepackedTime = new Stopwatch(); try { // Core algorithm of batch signing. // While there are files left to process, // Identify which files are ready for processing (ready to repack or sign) // Repack those of that set that are containers // Sign any of those files that need signing, along with their engines. while (toProcessList.Count > 0) { var trackList = identifyNextGroup(); if (trackList.Count == 0) { throw new InvalidOperationException("No progress made on signing which indicates a bug"); } int fileModifiedCount; telemetryRepackedTime.Start(); repackGroup(trackList, out fileModifiedCount); telemetryRepackedTime.Stop(); telemetryTotalFilesRepacked += fileModifiedCount; try { telemetrySignedTime.Start(); if (!signEngines(trackList, out fileModifiedCount)) { return(false); } if (fileModifiedCount > 0) { round++; telemetryTotalFilesSigned += fileModifiedCount; } if (!signGroup(trackList, out fileModifiedCount)) { return(false); } if (fileModifiedCount > 0) { round++; telemetryTotalFilesSigned += fileModifiedCount; } } finally { telemetrySignedTime.Stop(); } trackList.ForEach(x => trackedSet.Add(x.FileContentKey)); } } finally { if (_telemetry != null) { _telemetry.AddMetric("Signed file count", telemetryTotalFilesSigned); _telemetry.AddMetric("Repacked file count", telemetryTotalFilesRepacked); _telemetry.AddMetric("Signing duration (s)", telemetrySignedTime.ElapsedMilliseconds / 1000); _telemetry.AddMetric("Repacking duration (s)", telemetryRepackedTime.ElapsedMilliseconds / 1000); } } return(true); }
/// <summary> /// Actually sign all of the described files. /// </summary> private bool SignFiles(ContentMap contentMap, Dictionary <FileName, ZipData> zipDataMap) { // Generate the list of signed files in a deterministic order. Makes it easier to track down // bugs if repeated runs use the same ordering. var toSignList = _batchData.FileNames.ToList(); var round = 0; var signedSet = new HashSet <FileName>(); bool signFiles(IEnumerable <FileName> files) { var filesToSign = files.Select(fileName => _batchData.FileSignInfoMap[fileName]).Where(info => !info.IsEmpty).ToArray(); _log.LogMessage(MessageImportance.High, $"Signing Round {round}: {filesToSign.Length} files to sign."); foreach (var file in filesToSign) { _log.LogMessage(MessageImportance.Low, $"File '{file.FileName}' Certificate='{file.Certificate}'" + (file.StrongName != null ? $" StrongName='{file.StrongName}'" : "")); } return(_signTool.Sign(_buildEngine, round, filesToSign)); } void repackFiles(IEnumerable <FileName> files) { foreach (var file in files) { if (file.IsZipContainer) { _log.LogMessage(MessageImportance.Low, $"Repacking container: '{file}'"); Repack(zipDataMap[file]); } } } // Is this file ready to be signed? That is are all of the items that it depends on already // signed? bool isReadyToSign(FileName fileName) { if (!fileName.IsZipContainer) { return(true); } var zipData = zipDataMap[fileName]; return(zipData.NestedParts.All(x => signedSet.Contains(x.FileName))); } // Extract the next set of files that should be signed. This is the set of files for which all of the // dependencies have been signed. List <FileName> extractNextGroup() { var list = new List <FileName>(); var i = 0; while (i < toSignList.Count) { var current = toSignList[i]; if (isReadyToSign(current)) { list.Add(current); toSignList.RemoveAt(i); } else { i++; } } return(list); } while (toSignList.Count > 0) { var list = extractNextGroup(); if (list.Count == 0) { throw new InvalidOperationException("No progress made on signing which indicates a bug"); } repackFiles(list); if (!signFiles(list)) { return(false); } round++; list.ForEach(x => signedSet.Add(x)); } return(true); }