/// <summary>
/// Actually sign all of the described files.
/// </summary>
private bool SignFiles()
{
// Generate the list of signed files in a deterministic order. Makes it easier to track down
// bugs if repeated runs use the same ordering.
var toSignList = _batchData.FilesToSign.ToList();
var round = 0;
var signedSet = new HashSet <ImmutableArray <byte> >(ByteSequenceComparer.Instance);
bool signFiles(IEnumerable <FileSignInfo> files)
{
var filesToSign = files.Where(fileInfo => fileInfo.SignInfo.ShouldSign).ToArray();
_log.LogMessage(MessageImportance.High, $"Signing Round {round}: {filesToSign.Length} files to sign.");
if (filesToSign.Length == 0)
{
return(true);
}
foreach (var file in filesToSign)
{
_log.LogMessage(MessageImportance.Low, file.ToString());
}
return(_signTool.Sign(_buildEngine, round, filesToSign));
}
void repackFiles(IEnumerable <FileSignInfo> files)
{
foreach (var file in files)
{
if (file.IsZipContainer())
{
_log.LogMessage($"Repacking container: '{file.FileName}'");
_batchData.ZipDataMap[file.ContentHash].Repack(_log);
}
}
}
// Is this file ready to be signed? That is are all of the items that it depends on already
// signed?
bool isReadyToSign(FileSignInfo file)
{
if (!file.IsZipContainer())
{
return(true);
}
var zipData = _batchData.ZipDataMap[file.ContentHash];
return(zipData.NestedParts.All(x => !x.FileSignInfo.SignInfo.ShouldSign || signedSet.Contains(x.FileSignInfo.ContentHash)));
}
// Extract the next set of files that should be signed. This is the set of files for which all of the
// dependencies have been signed.
List <FileSignInfo> extractNextGroup()
{
var list = new List <FileSignInfo>();
var i = 0;
while (i < toSignList.Count)
{
var current = toSignList[i];
if (isReadyToSign(current))
{
list.Add(current);
toSignList.RemoveAt(i);
}
else
{
i++;
}
}
return(list);
}
while (toSignList.Count > 0)
{
var list = extractNextGroup();
if (list.Count == 0)
{
throw new InvalidOperationException("No progress made on signing which indicates a bug");
}
repackFiles(list);
if (!signFiles(list))
{
return(false);
}
round++;
list.ForEach(x => signedSet.Add(x.ContentHash));
}
return(true);
}
/// <summary>
/// Actually sign all of the described files.
/// </summary>
private bool SignFiles()
{
// Generate the list of signed files in a deterministic order. Makes it easier to track down
// bugs if repeated runs use the same ordering.
var toProcessList = _batchData.FilesToSign.ToList();
var toRepackList = _batchData.FilesToSign.Where(x => x.ShouldRepack)?.Select(x => x.FullPath)?.ToList();
var round = 0;
var trackedSet = new HashSet <SignedFileContentKey>();
bool signFiles(IEnumerable <FileSignInfo> files, out int totalFilesSigned)
{
var filesToSign = files.Where(fileInfo => fileInfo.SignInfo.ShouldSign).ToArray();
totalFilesSigned = filesToSign.Length;
_log.LogMessage(MessageImportance.High, $"Signing Round {round}: {filesToSign.Length} files to sign.");
if (filesToSign.Length == 0)
{
return(true);
}
foreach (var file in filesToSign)
{
_log.LogMessage(MessageImportance.Low, file.ToString());
}
return(_signTool.Sign(_buildEngine, round, filesToSign));
}
bool signEngines(IEnumerable <FileSignInfo> files, out int totalFilesSigned)
{
var enginesToSign = files.Where(fileInfo => fileInfo.SignInfo.ShouldSign &&
fileInfo.IsWixContainer() &&
Path.GetExtension(fileInfo.FullPath) == ".exe").ToArray();
totalFilesSigned = enginesToSign.Length;
if (enginesToSign.Length == 0)
{
return(true);
}
Dictionary <string, FileSignInfo> engines = new Dictionary <string, FileSignInfo>();
var workingDirectory = Path.Combine(_signTool.TempDir, "engines");
// extract engines
foreach (var file in enginesToSign)
{
string engineFileName = $"{Path.Combine(workingDirectory, file.FileName)}{SignToolConstants.MsiEngineExtension}";
_log.LogMessage(MessageImportance.Normal, $"Extracting engine from {file.FullPath}");
int exitCode = RunWixTool("insignia.exe", $"-ib {file.FullPath} -o {engineFileName}", workingDirectory, _signTool.WixToolsPath);
if (exitCode != 0)
{
_log.LogError($"Failed to extract engine from {file.FullPath}");
return(false);
}
engines.Add(engineFileName, file);
}
// sign engines
bool signResult = _signTool.Sign(_buildEngine, round, engines.Select(engine => new FileSignInfo(engine.Key, engine.Value.ContentHash, engine.Value.SignInfo)));
if (!signResult)
{
_log.LogError($"Failed to sign engines");
return(signResult);
}
// attach engines
foreach (var engine in engines)
{
_log.LogMessage(MessageImportance.Normal, $"Attaching engine {engine.Key} to {engine.Value.FullPath}");
int exitCode = RunWixTool("insignia.exe", $"-ab {engine.Key} {engine.Value.FullPath} -o {engine.Value.FullPath}", workingDirectory, _signTool.WixToolsPath);
// cleanup engines (they fail signing verification if they stay in the drop
File.Delete(engine.Key);
if (exitCode != 0)
{
_log.LogError($"Failed to attach engine to {engine.Value.FullPath}");
return(false);
}
}
return(true);
}
void repackFiles(IEnumerable <FileSignInfo> files)
{
foreach (var file in files)
{
if (file.IsZipContainer())
{
_log.LogMessage($"Repacking container: '{file.FileName}'");
_batchData.ZipDataMap[file.FileContentKey].Repack(_log);
toRepackList.Remove(file.FullPath);
}
else if (file.IsWixContainer())
{
_log.LogMessage($"Packing wix container: '{file.FileName}'");
_batchData.ZipDataMap[file.FileContentKey].Repack(_log, _signTool.TempDir, _signTool.WixToolsPath);
toRepackList.Remove(file.FullPath);
}
}
}
// Is this file ready to be signed? That is are all of the items that it depends on already
// signed?
bool isReady(FileSignInfo file)
{
if (file.IsContainer())
{
var zipData = _batchData.ZipDataMap[file.FileContentKey];
return(zipData.NestedParts.All(x => (!x.FileSignInfo.SignInfo.ShouldSign ||
trackedSet.Contains(x.FileSignInfo.FileContentKey)) && !toRepackList.Contains(x.FileSignInfo.FullPath)
));
}
return(true);
}
// Extract the next set of files that should be signed. This is the set of files for which all of the
// dependencies have been signed.
List <FileSignInfo> extractNextGroup()
{
var list = new List <FileSignInfo>();
var i = 0;
while (i < toProcessList.Count)
{
var current = toProcessList[i];
if (isReady(current))
{
list.Add(current);
toProcessList.RemoveAt(i);
}
else
{
i++;
}
}
return(list);
}
while (toProcessList.Count > 0)
{
var trackList = extractNextGroup();
if (trackList.Count == 0)
{
throw new InvalidOperationException("No progress made on signing which indicates a bug");
}
var repackList = trackList.Where(w => toRepackList.Contains(w.FullPath));
repackFiles(repackList);
int totalFilesSigned;
if (!signEngines(trackList, out totalFilesSigned))
{
return(false);
}
if (totalFilesSigned > 0)
{
round++;
}
if (!signFiles(trackList, out totalFilesSigned))
{
return(false);
}
if (totalFilesSigned > 0)
{
round++;
}
trackList.ForEach(x => trackedSet.Add(x.FileContentKey));
}
return(true);
}
/// <summary>
/// Actually sign all of the described files.
/// </summary>
private bool SignFiles()
{
// Generate the list of signed files in a deterministic order. Makes it easier to track down
// bugs if repeated runs use the same ordering.
var toProcessList = _batchData.FilesToSign.ToList();
var toRepackSet = _batchData.FilesToSign.Where(x => x.ShouldRepack)?.Select(x => x.FullPath)?.ToHashSet();
var round = 0;
var trackedSet = new HashSet <SignedFileContentKey>();
// Given a list of files that need signing, sign them in a batch.
bool signGroup(IEnumerable <FileSignInfo> files, out int signedCount)
{
var filesToSign = files.Where(fileInfo => fileInfo.SignInfo.ShouldSign).ToArray();
signedCount = filesToSign.Length;
if (filesToSign.Length == 0)
{
return(true);
}
_log.LogMessage(MessageImportance.High, $"Round {round}: Signing {filesToSign.Length} files.");
foreach (var file in filesToSign)
{
string collisionIdInfo = string.Empty;
if (_hashToCollisionIdMap != null)
{
if (_hashToCollisionIdMap.TryGetValue(file.FileContentKey, out string collisionPriorityId))
{
collisionIdInfo = $"Collision Id='{collisionPriorityId}'";
}
}
_log.LogMessage(MessageImportance.Low, $"{file} {collisionIdInfo}");
}
return(_signTool.Sign(_buildEngine, round, filesToSign));
}
// Given a list of files that need signing, sign the installer engines
// of those that are wix containers.
bool signEngines(IEnumerable <FileSignInfo> files, out int signedCount)
{
var enginesToSign = files.Where(fileInfo => fileInfo.SignInfo.ShouldSign &&
fileInfo.IsWixContainer() &&
Path.GetExtension(fileInfo.FullPath) == ".exe").ToArray();
signedCount = enginesToSign.Length;
if (enginesToSign.Length == 0)
{
return(true);
}
_log.LogMessage(MessageImportance.High, $"Round {round}: Signing {enginesToSign.Length} engines.");
Dictionary <SignedFileContentKey, FileSignInfo> engines = new Dictionary <SignedFileContentKey, FileSignInfo>();
var workingDirectory = Path.Combine(_signTool.TempDir, "engines");
int engineContainer = 0;
// extract engines
foreach (var file in enginesToSign)
{
string engineFileName = $"{Path.Combine(workingDirectory, $"{engineContainer}", file.FileName)}{SignToolConstants.MsiEngineExtension}";
_log.LogMessage(MessageImportance.Normal, $"Extracting engine from {file.FullPath}");
if (!RunWixTool("insignia.exe", $"-ib {file.FullPath} -o {engineFileName}",
workingDirectory, _signTool.WixToolsPath, _log))
{
_log.LogError($"Failed to extract engine from {file.FullPath}");
return(false);
}
var fileUniqueKey = new SignedFileContentKey(file.ContentHash, engineFileName);
engines.Add(fileUniqueKey, file);
engineContainer++;
}
// sign engines
bool signResult = _signTool.Sign(_buildEngine, round, engines.Select(engine =>
new FileSignInfo(new PathWithHash(engine.Key.FileName, engine.Value.ContentHash), engine.Value.SignInfo)));
if (!signResult)
{
_log.LogError($"Failed to sign engines");
return(signResult);
}
// attach engines
foreach (var engine in engines)
{
_log.LogMessage(MessageImportance.Normal, $"Attaching engine {engine.Key.FileName} to {engine.Value.FullPath}");
try
{
if (!RunWixTool("insignia.exe",
$"-ab {engine.Key.FileName} {engine.Value.FullPath} -o {engine.Value.FullPath}", workingDirectory,
_signTool.WixToolsPath, _log))
{
_log.LogError($"Failed to attach engine to {engine.Value.FullPath}");
return(false);
}
}
finally
{
// cleanup engines (they fail signing verification if they stay in the drop
File.Delete(engine.Key.FileName);
}
}
return(true);
}
// Given a group of file that are ready for processing,
// repack those files that are containers.
void repackGroup(IEnumerable <FileSignInfo> files, out int repackCount)
{
var repackList = files.Where(w => toRepackSet.Contains(w.FullPath)).ToList();
repackCount = repackList.Count();
if (repackCount == 0)
{
return;
}
_log.LogMessage(MessageImportance.High, $"Repacking {repackCount} containers.");
ParallelOptions parallelOptions = new ParallelOptions();
parallelOptions.MaxDegreeOfParallelism = 16;
Parallel.ForEach(repackList, parallelOptions, file =>
{
if (file.IsZipContainer())
{
_log.LogMessage($"Repacking container: '{file.FileName}'");
_batchData.ZipDataMap[file.FileContentKey].Repack(_log);
}
else if (file.IsWixContainer())
{
_log.LogMessage($"Packing wix container: '{file.FileName}'");
_batchData.ZipDataMap[file.FileContentKey].Repack(_log, _signTool.TempDir, _signTool.WixToolsPath);
}
else
{
_log.LogError($"Don't know how to repack file '{file.FullPath}'");
}
toRepackSet.Remove(file.FullPath);
});
}
// Is this file ready to be signed or repackaged? That is are all of the items that it depends on already
// signed, don't need signing, and are repacked.
bool isReady(FileSignInfo file)
{
if (file.IsContainer())
{
var zipData = _batchData.ZipDataMap[file.FileContentKey];
return(zipData.NestedParts.Values.All(x => (!x.FileSignInfo.SignInfo.ShouldSign ||
trackedSet.Contains(x.FileSignInfo.FileContentKey)) && !toRepackSet.Contains(x.FileSignInfo.FullPath)
));
}
return(true);
}
// Identify the next set of files that should be signed or repacked.
// This is the set of files for which all of the dependencies have been signed,
// are already signed, are repacked, etc.
List <FileSignInfo> identifyNextGroup()
{
var list = new List <FileSignInfo>();
var i = 0;
while (i < toProcessList.Count)
{
var current = toProcessList[i];
if (isReady(current))
{
list.Add(current);
toProcessList.RemoveAt(i);
}
else
{
i++;
}
}
return(list);
}
// Telemetry data
double telemetryTotalFilesSigned = 0;
double telemetryTotalFilesRepacked = 0;
Stopwatch telemetrySignedTime = new Stopwatch();
Stopwatch telemetryRepackedTime = new Stopwatch();
try
{
// Core algorithm of batch signing.
// While there are files left to process,
// Identify which files are ready for processing (ready to repack or sign)
// Repack those of that set that are containers
// Sign any of those files that need signing, along with their engines.
while (toProcessList.Count > 0)
{
var trackList = identifyNextGroup();
if (trackList.Count == 0)
{
throw new InvalidOperationException("No progress made on signing which indicates a bug");
}
int fileModifiedCount;
telemetryRepackedTime.Start();
repackGroup(trackList, out fileModifiedCount);
telemetryRepackedTime.Stop();
telemetryTotalFilesRepacked += fileModifiedCount;
try
{
telemetrySignedTime.Start();
if (!signEngines(trackList, out fileModifiedCount))
{
return(false);
}
if (fileModifiedCount > 0)
{
round++;
telemetryTotalFilesSigned += fileModifiedCount;
}
if (!signGroup(trackList, out fileModifiedCount))
{
return(false);
}
if (fileModifiedCount > 0)
{
round++;
telemetryTotalFilesSigned += fileModifiedCount;
}
}
finally
{
telemetrySignedTime.Stop();
}
trackList.ForEach(x => trackedSet.Add(x.FileContentKey));
}
}
finally
{
if (_telemetry != null)
{
_telemetry.AddMetric("Signed file count", telemetryTotalFilesSigned);
_telemetry.AddMetric("Repacked file count", telemetryTotalFilesRepacked);
_telemetry.AddMetric("Signing duration (s)", telemetrySignedTime.ElapsedMilliseconds / 1000);
_telemetry.AddMetric("Repacking duration (s)", telemetryRepackedTime.ElapsedMilliseconds / 1000);
}
}
return(true);
}
/// <summary>
/// Actually sign all of the described files.
/// </summary>
private bool SignFiles(ContentMap contentMap, Dictionary <FileName, ZipData> zipDataMap)
{
// Generate the list of signed files in a deterministic order. Makes it easier to track down
// bugs if repeated runs use the same ordering.
var toSignList = _batchData.FileNames.ToList();
var round = 0;
var signedSet = new HashSet <FileName>();
bool signFiles(IEnumerable <FileName> files)
{
var filesToSign = files.Select(fileName => _batchData.FileSignInfoMap[fileName]).Where(info => !info.IsEmpty).ToArray();
_log.LogMessage(MessageImportance.High, $"Signing Round {round}: {filesToSign.Length} files to sign.");
foreach (var file in filesToSign)
{
_log.LogMessage(MessageImportance.Low,
$"File '{file.FileName}' Certificate='{file.Certificate}'" +
(file.StrongName != null ? $" StrongName='{file.StrongName}'" : ""));
}
return(_signTool.Sign(_buildEngine, round, filesToSign));
}
void repackFiles(IEnumerable <FileName> files)
{
foreach (var file in files)
{
if (file.IsZipContainer)
{
_log.LogMessage(MessageImportance.Low, $"Repacking container: '{file}'");
Repack(zipDataMap[file]);
}
}
}
// Is this file ready to be signed? That is are all of the items that it depends on already
// signed?
bool isReadyToSign(FileName fileName)
{
if (!fileName.IsZipContainer)
{
return(true);
}
var zipData = zipDataMap[fileName];
return(zipData.NestedParts.All(x => signedSet.Contains(x.FileName)));
}
// Extract the next set of files that should be signed. This is the set of files for which all of the
// dependencies have been signed.
List <FileName> extractNextGroup()
{
var list = new List <FileName>();
var i = 0;
while (i < toSignList.Count)
{
var current = toSignList[i];
if (isReadyToSign(current))
{
list.Add(current);
toSignList.RemoveAt(i);
}
else
{
i++;
}
}
return(list);
}
while (toSignList.Count > 0)
{
var list = extractNextGroup();
if (list.Count == 0)
{
throw new InvalidOperationException("No progress made on signing which indicates a bug");
}
repackFiles(list);
if (!signFiles(list))
{
return(false);
}
round++;
list.ForEach(x => signedSet.Add(x));
}
return(true);
}
