private void AddSources(RuntimeFile runtimeFile, RuntimeSpec runtime, List <string> sources)
{
sources.RemoveAll(name => name == runtime.Name);
sources.Add(runtime.Name);
foreach (var import in runtime.Import)
{
AddSources(runtimeFile, runtimeFile.Runtimes[import], sources);
}
}
private void ExecuteCore()
{
var target = new TargetInfo(TargetFramework, RuntimeIdentifier, string.Empty, isPortable: false);
var runtimeFiles = new List <RuntimeFile>();
var nativeFiles = new List <RuntimeFile>();
var resourceAssemblies = new List <ResourceAssembly>();
var platformManifest = new List <string>();
foreach (var reference in References)
{
var filePath = reference.ItemSpec;
var fileName = Path.GetFileName(filePath);
var fileVersion = FileUtilities.GetFileVersion(filePath)?.ToString() ?? string.Empty;
var assemblyVersion = FileUtilities.TryGetAssemblyVersion(filePath);
if (assemblyVersion == null)
{
var nativeFile = new RuntimeFile(fileName, null, fileVersion);
nativeFiles.Add(nativeFile);
platformManifest.Add($"{fileName}|{FrameworkName}||{fileVersion}");
}
else
{
var runtimeFile = new RuntimeFile(fileName,
fileVersion: fileVersion,
assemblyVersion: assemblyVersion.ToString());
runtimeFiles.Add(runtimeFile);
platformManifest.Add($"{fileName}|{FrameworkName}|{assemblyVersion}|{fileVersion}");
}
}
var runtimeLibrary = new RuntimeLibrary("package",
RuntimePackageName,
FrameworkVersion,
hash: string.Empty,
runtimeAssemblyGroups: new[] { new RuntimeAssetGroup(string.Empty, runtimeFiles) },
nativeLibraryGroups: new[] { new RuntimeAssetGroup(string.Empty, nativeFiles) },
Enumerable.Empty <ResourceAssembly>(),
Array.Empty <Dependency>(),
hashPath: null,
path: $"{RuntimePackageName.ToLowerInvariant()}/{FrameworkVersion}",
serviceable: true);
var context = new DependencyContext(target,
CompilationOptions.Default,
Enumerable.Empty <CompilationLibrary>(),
new[] { runtimeLibrary },
Enumerable.Empty <RuntimeFallbacks>());
Directory.CreateDirectory(Path.GetDirectoryName(DepsFilePath));
Directory.CreateDirectory(Path.GetDirectoryName(PlatformManifestOutputPath));
File.WriteAllText(
PlatformManifestOutputPath,
string.Join("\n", platformManifest.OrderBy(n => n)),
Encoding.UTF8);
try
{
using (var depsStream = File.Create(DepsFilePath))
{
new DependencyContextWriter().Write(context, depsStream);
}
}
catch (Exception ex)
{
// If there is a problem, ensure we don't write a partially complete version to disk.
if (File.Exists(DepsFilePath))
{
File.Delete(DepsFilePath);
}
Log.LogErrorFromException(ex);
}
}
private async Task <bool> RestoreForProject(string projectJsonPath, string rootDirectory, string packagesDirectory, IList <IWalkProvider> remoteProviders, SummaryContext summary)
{
var success = true;
Reports.Information.WriteLine(string.Format("Restoring packages for {0}", projectJsonPath.Bold()));
var sw = new Stopwatch();
sw.Start();
var projectFolder = Path.GetDirectoryName(projectJsonPath);
var projectLockFilePath = Path.Combine(projectFolder, LockFileFormat.LockFileName);
Runtime.Project project;
var diagnostics = new List <DiagnosticMessage>();
if (!Runtime.Project.TryGetProject(projectJsonPath, out project, diagnostics))
{
var errorMessages = diagnostics
.Where(x => x.Severity == DiagnosticMessageSeverity.Error)
.Select(x => x.Message);
throw new InvalidOperationException(errorMessages.Any() ?
$"Errors occurred when while parsing project.json:{Environment.NewLine}{string.Join(Environment.NewLine, errorMessages)}" :
"Invalid project.json");
}
if (diagnostics.HasErrors())
{
var errorMessages = diagnostics
.Where(x => x.Severity == DiagnosticMessageSeverity.Error)
.Select(x => x.Message);
summary.ErrorMessages.GetOrAdd(projectJsonPath, _ => new List <string>()).AddRange(errorMessages);
}
var lockFile = await ReadLockFile(projectLockFilePath);
var useLockFile = false;
if (Lock == false &&
Unlock == false &&
lockFile != null &&
lockFile.Islocked)
{
useLockFile = true;
}
if (useLockFile && !lockFile.IsValidForProject(project))
{
// Exhibit the same behavior as if it has been run with "dnu restore --lock"
Reports.Information.WriteLine("Updating the invalid lock file with {0}",
"dnu restore --lock".Yellow().Bold());
useLockFile = false;
Lock = true;
}
Func <string, string> getVariable = key =>
{
return(null);
};
if (!SkipRestoreEvents)
{
if (!ScriptExecutor.Execute(project, "prerestore", getVariable))
{
summary.ErrorMessages.GetOrAdd("prerestore", _ => new List <string>()).Add(ScriptExecutor.ErrorMessage);
Reports.Error.WriteLine(ScriptExecutor.ErrorMessage);
return(false);
}
}
var projectDirectory = project.ProjectDirectory;
var projectResolver = new ProjectResolver(projectDirectory, rootDirectory);
var packageRepository = new PackageRepository(packagesDirectory)
{
CheckHashFile = CheckHashFile
};
var restoreOperations = new RestoreOperations(Reports.Verbose);
var projectProviders = new List <IWalkProvider>();
var localProviders = new List <IWalkProvider>();
var contexts = new List <RestoreContext>();
var cache = new Dictionary <LibraryRange, Task <WalkProviderMatch> >();
projectProviders.Add(
new LocalWalkProvider(
new ProjectReferenceDependencyProvider(
projectResolver)));
localProviders.Add(
new LocalWalkProvider(
new NuGetDependencyResolver(packageRepository)));
var tasks = new List <Task <TargetContext> >();
if (useLockFile)
{
Reports.Information.WriteLine(string.Format("Following lock file {0}", projectLockFilePath.White().Bold()));
var context = new RestoreContext
{
FrameworkName = FallbackFramework,
ProjectLibraryProviders = projectProviders,
LocalLibraryProviders = localProviders,
RemoteLibraryProviders = remoteProviders,
MatchCache = cache
};
contexts.Add(context);
foreach (var lockFileLibrary in lockFile.PackageLibraries)
{
var projectLibrary = new LibraryRange(lockFileLibrary.Name, frameworkReference: false)
{
VersionRange = new SemanticVersionRange
{
MinVersion = lockFileLibrary.Version,
MaxVersion = lockFileLibrary.Version,
IsMaxInclusive = true,
VersionFloatBehavior = SemanticVersionFloatBehavior.None,
}
};
tasks.Add(CreateGraphNode(restoreOperations, context, projectLibrary, _ => false));
}
}
else
{
var frameworks = TargetFrameworks.Count == 0 ? project.GetTargetFrameworks().Select(f => f.FrameworkName) : TargetFrameworks;
foreach (var frameworkName in frameworks)
{
var context = new RestoreContext
{
FrameworkName = frameworkName,
ProjectLibraryProviders = projectProviders,
LocalLibraryProviders = localProviders,
RemoteLibraryProviders = remoteProviders,
MatchCache = cache
};
contexts.Add(context);
}
if (!contexts.Any())
{
contexts.Add(new RestoreContext
{
FrameworkName = FallbackFramework,
ProjectLibraryProviders = projectProviders,
LocalLibraryProviders = localProviders,
RemoteLibraryProviders = remoteProviders,
MatchCache = cache
});
}
foreach (var context in contexts)
{
var projectLibrary = new LibraryRange(project.Name, frameworkReference: false)
{
VersionRange = new SemanticVersionRange(project.Version)
};
tasks.Add(CreateGraphNode(restoreOperations, context, projectLibrary, _ => true));
}
}
var targetContexts = await Task.WhenAll(tasks);
foreach (var targetContext in targetContexts)
{
Reduce(targetContext.Root);
}
if (!useLockFile)
{
var projectRuntimeFile = RuntimeFile.ParseFromProject(project);
var restoreRuntimes = GetRestoreRuntimes(projectRuntimeFile.Runtimes.Keys).ToList();
if (restoreRuntimes.Any())
{
var runtimeTasks = new List <Task <TargetContext> >();
foreach (var pair in contexts.Zip(targetContexts, (context, graph) => new { context, graph }))
{
var runtimeFileTasks = new List <Task <RuntimeFile> >();
ForEach(pair.graph.Root, node =>
{
var match = node?.Item?.Match;
if (match == null)
{
return;
}
runtimeFileTasks.Add(match.Provider.GetRuntimes(node.Item.Match, pair.context.FrameworkName));
});
var libraryRuntimeFiles = await Task.WhenAll(runtimeFileTasks);
var runtimeFiles = new List <RuntimeFile> {
projectRuntimeFile
};
runtimeFiles.AddRange(libraryRuntimeFiles.Where(file => file != null));
foreach (var runtimeName in restoreRuntimes)
{
Reports.WriteVerbose($"Restoring packages for {pair.context.FrameworkName} on {runtimeName}...");
var runtimeDependencies = new Dictionary <string, DependencySpec>();
var runtimeNames = new HashSet <string>();
var runtimeStopwatch = Stopwatch.StartNew();
FindRuntimeDependencies(
runtimeName,
runtimeFiles,
runtimeDependencies,
runtimeNames);
runtimeStopwatch.Stop();
Reports.WriteVerbose($" Scanned Runtime graph in {runtimeStopwatch.ElapsedMilliseconds:0.00}ms");
// If there are no runtime specs in the graph, we still want to restore for the specified runtime, so synthesize one
if (!runtimeNames.Any(r => r.Equals(runtimeName)))
{
runtimeNames.Add(runtimeName);
}
var runtimeContext = new RestoreContext
{
FrameworkName = pair.context.FrameworkName,
ProjectLibraryProviders = pair.context.ProjectLibraryProviders,
LocalLibraryProviders = pair.context.LocalLibraryProviders,
RemoteLibraryProviders = pair.context.RemoteLibraryProviders,
RuntimeName = runtimeName,
AllRuntimeNames = runtimeNames,
RuntimeDependencies = runtimeDependencies,
MatchCache = cache
};
var projectLibrary = new LibraryRange(project.Name, frameworkReference: false)
{
VersionRange = new SemanticVersionRange(project.Version)
};
runtimeTasks.Add(CreateGraphNode(restoreOperations, runtimeContext, projectLibrary, _ => true));
}
}
var runtimeTragetContexts = await Task.WhenAll(runtimeTasks);
foreach (var runtimeTargetContext in runtimeTragetContexts)
{
Reduce(runtimeTargetContext.Root);
}
targetContexts = targetContexts.Concat(runtimeTragetContexts).ToArray();
}
}
var graphItems = new List <GraphItem>();
var installItems = new List <GraphItem>();
var missingItems = new HashSet <LibraryRange>();
foreach (var context in targetContexts)
{
ForEach(context.Root, node =>
{
if (node == null || node.LibraryRange == null)
{
return;
}
if (node.Item == null || node.Item.Match == null)
{
// This is a workaround for #1322. Since we use restore to generate the lock file
// after publish, it's possible to fail restore after copying the closure
if (!IgnoreMissingDependencies)
{
if (!node.LibraryRange.IsGacOrFrameworkReference &&
missingItems.Add(node.LibraryRange))
{
var versionString = node.LibraryRange.VersionRange == null ?
string.Empty :
(" " + node.LibraryRange.VersionRange.ToString());
var errorMessage =
$"Unable to locate {DependencyTargets.GetDisplayForTarget(node.LibraryRange.Target)} " +
$"{node.LibraryRange.Name.Red().Bold()}{versionString}";
summary.ErrorMessages.GetOrAdd(projectJsonPath, _ => new List <string>()).Add(errorMessage);
Reports.Error.WriteLine(errorMessage);
success = false;
}
}
return;
}
if (!string.Equals(node.Item.Match.Library.Name, node.LibraryRange.Name, StringComparison.Ordinal))
{
// Fix casing of the library name to be installed
node.Item.Match.Library = node.Item.Match.Library.ChangeName(node.LibraryRange.Name);
}
var isRemote = remoteProviders.Contains(node.Item.Match.Provider);
var isInstallItem = installItems.Any(item => item.Match.Library == node.Item.Match.Library);
if (!isInstallItem && isRemote)
{
// It's ok to download rejected nodes so we avoid downloading them in the future
// The trade off is that subsequent restores avoid going to any remotes
installItems.Add(node.Item);
}
// Don't add rejected nodes since we only want to write reduced nodes
// to the lock file
if (node.Disposition != GraphNode.DispositionType.Rejected)
{
var isGraphItem = graphItems.Any(item => item.Match.Library == node.Item.Match.Library);
if (!isGraphItem)
{
graphItems.Add(node.Item);
}
context.Matches.Add(node.Item.Match);
}
});
}
if (!SkipInstall)
{
await InstallPackages(installItems, packagesDirectory);
summary.InstallCount += installItems.Count;
}
if (!useLockFile)
{
Reports.Information.WriteLine(string.Format("Writing lock file {0}", projectLockFilePath.White().Bold()));
var repository = new PackageRepository(packagesDirectory);
WriteLockFile(lockFile,
projectLockFilePath,
project,
graphItems,
repository,
projectResolver,
targetContexts);
}
if (!SkipRestoreEvents)
{
if (!ScriptExecutor.Execute(project, "postrestore", getVariable))
{
summary.ErrorMessages.GetOrAdd("postrestore", _ => new List <string>()).Add(ScriptExecutor.ErrorMessage);
Reports.Error.WriteLine(ScriptExecutor.ErrorMessage);
return(false);
}
if (!ScriptExecutor.Execute(project, "prepare", getVariable))
{
summary.ErrorMessages.GetOrAdd("prepare", _ => new List <string>()).Add(ScriptExecutor.ErrorMessage);
Reports.Error.WriteLine(ScriptExecutor.ErrorMessage);
return(false);
}
}
Reports.Information.WriteLine(string.Format("{0}, {1}ms elapsed", "Restore complete".Green().Bold(), sw.ElapsedMilliseconds));
return(success);
}
public override bool Execute()
{
var target = new TargetInfo(TargetFrameworkMoniker, RuntimeIdentifier, string.Empty, isPortable: false);
var runtimeFiles = new List <RuntimeFile>();
var nativeFiles = new List <RuntimeFile>();
var resourceAssemblies = new List <ResourceAssembly>();
foreach (var file in Files)
{
if (!string.IsNullOrEmpty(file.GetMetadata("GeneratedBuildFile")))
{
continue;
}
string filePath = file.ItemSpec;
string fileName = Path.GetFileName(filePath);
string fileVersion = FileUtilities.GetFileVersion(filePath)?.ToString() ?? string.Empty;
Version assemblyVersion = FileUtilities.GetAssemblyName(filePath)?.Version;
string cultureMaybe = file.GetMetadata("Culture");
if (!string.IsNullOrEmpty(cultureMaybe))
{
resourceAssemblies.Add(new ResourceAssembly(Path.Combine(cultureMaybe, fileName), cultureMaybe));
}
else if (assemblyVersion == null)
{
var nativeFile = new RuntimeFile(fileName, null, fileVersion);
nativeFiles.Add(nativeFile);
}
else
{
var runtimeFile = new RuntimeFile(fileName,
fileVersion: fileVersion,
assemblyVersion: assemblyVersion.ToString());
runtimeFiles.Add(runtimeFile);
}
}
var runtimeLibrary = new RuntimeLibrary("package",
SharedFrameworkPackName,
Version,
hash: string.Empty,
runtimeAssemblyGroups: new[] { new RuntimeAssetGroup(string.Empty, runtimeFiles) },
nativeLibraryGroups: new[] { new RuntimeAssetGroup(string.Empty, nativeFiles) },
resourceAssemblies,
Array.Empty <Dependency>(),
hashPath: null,
path: $"{SharedFrameworkPackName.ToLowerInvariant()}/{Version}",
serviceable: true);
IEnumerable <RuntimeFallbacks> runtimeFallbackGraph = Array.Empty <RuntimeFallbacks>();
if (IncludeFallbacksInDepsFile)
{
RuntimeGraph runtimeGraph = JsonRuntimeFormat.ReadRuntimeGraph(RuntimeIdentifierGraph);
runtimeFallbackGraph = runtimeGraph.Runtimes
.Select(runtimeDict => runtimeGraph.ExpandRuntime(runtimeDict.Key))
.Where(expansion => expansion.Contains(RuntimeIdentifier))
.Select(expansion => new RuntimeFallbacks(expansion.First(), expansion.Skip(1))); // ExpandRuntime return runtime itself as first item.
}
var context = new DependencyContext(target,
CompilationOptions.Default,
Enumerable.Empty <CompilationLibrary>(),
new[] { runtimeLibrary },
runtimeFallbackGraph);
var depsFileName = string.IsNullOrEmpty(SharedFrameworkDepsNameOverride) ? $"{SharedFrameworkName}.deps.json" : $"{SharedFrameworkDepsNameOverride}.deps.json";
var depsFilePath = Path.Combine(IntermediateOutputPath, depsFileName);
try
{
using var depsStream = File.Create(depsFilePath);
new DependencyContextWriter().Write(context, depsStream);
GeneratedDepsFile = new TaskItem(depsFilePath);
}
catch (Exception ex)
{
// If there is a problem, ensure we don't write a partially complete version to disk.
if (File.Exists(depsFilePath))
{
File.Delete(depsFilePath);
}
Log.LogErrorFromException(ex, false);
return(false);
}
return(true);
}
public void Verify_DepsJsonChanges()
{
string depsJsonFileName = "Microsoft.Azure.WebJobs.Script.WebHost.deps.json";
string oldDepsJson = Path.GetFullPath(depsJsonFileName);
var newDepsJson = Directory.GetFiles(Path.GetFullPath(@"..\..\..\..\..\src\WebJobs.Script.WebHost\bin\"), depsJsonFileName, SearchOption.AllDirectories).FirstOrDefault();
Assert.True(File.Exists(oldDepsJson), $"{oldDepsJson} not found.");
Assert.True(File.Exists(newDepsJson), $"{newDepsJson} not found.");
IEnumerable <RuntimeFile> oldAssets = GetRuntimeFiles(oldDepsJson);
IEnumerable <RuntimeFile> newAssets = GetRuntimeFiles(newDepsJson);
var comparer = new RuntimeFileComparer();
var removed = oldAssets.Except(newAssets, comparer).ToList();
var added = newAssets.Except(oldAssets, comparer).ToList();
bool succeed = removed.Count == 0 && added.Count == 0;
if (succeed)
{
return;
}
IList <RuntimeFile> changed = new List <RuntimeFile>();
StringBuilder sb = new StringBuilder();
sb.AppendLine("IMPORTANT: The dependencies in WebHost have changed and MUST be reviewed before proceeding. Please follow up with brettsam, fabiocav or mathewc for approval.");
sb.AppendLine();
sb.AppendLine($"Previous file: {oldDepsJson}");
sb.AppendLine($"New file: {newDepsJson}");
sb.AppendLine();
sb.AppendLine(" Changed:");
foreach (RuntimeFile oldFile in oldAssets)
{
string fileName = Path.GetFileName(oldFile.Path);
if (_excludedList.Contains(fileName))
{
continue;
}
RuntimeFile newFile = newAssets.SingleOrDefault(p =>
{
return(Path.GetFileName(p.Path) == fileName &&
p.FileVersion != oldFile.FileVersion &&
p.AssemblyVersion != oldFile.AssemblyVersion);
});
if (newFile != null)
{
sb.AppendLine($" - {fileName}: {oldFile.AssemblyVersion}/{oldFile.FileVersion} -> {newFile.AssemblyVersion}/{newFile.FileVersion}");
changed.Add(oldFile);
changed.Add(newFile);
}
}
sb.AppendLine();
sb.AppendLine(" Removed:");
foreach (RuntimeFile f in removed.Except(changed))
{
sb.AppendLine($" - {Path.GetFileName(f.Path)}: {f.AssemblyVersion}/{f.FileVersion}");
}
sb.AppendLine();
sb.AppendLine(" Added:");
foreach (RuntimeFile f in added.Except(changed))
{
sb.AppendLine($" - {Path.GetFileName(f.Path)}: {f.AssemblyVersion}/{f.FileVersion}");
}
Assert.True(succeed, sb.ToString());
}
