RegisterProfile(
TimeProfile profile)
{
if (null == profile)
{
throw new Exception("Timing profile is invalid");
}
Profiles.Add(profile);
}
LoadPackageAssembly()
{
var assemblyLoadProfile = new TimeProfile(ETimingProfiles.LoadAssembly);
assemblyLoadProfile.StartProfile();
System.Reflection.Assembly scriptAssembly = null;
try
{
// this code works from an untrusted location, and debugging IS available when
// the pdb (.NET)/mdb (Mono) resides beside the assembly
byte[] asmBytes = System.IO.File.ReadAllBytes(Graph.Instance.ScriptAssemblyPathname);
if (Graph.Instance.CompileWithDebugSymbols)
{
var debugInfoFilename = Graph.Instance.ProcessState.RunningMono ?
Graph.Instance.ScriptAssemblyPathname + ".mdb" :
System.IO.Path.ChangeExtension(Graph.Instance.ScriptAssemblyPathname, ".pdb");
if (System.IO.File.Exists(debugInfoFilename))
{
byte[] pdbBytes = System.IO.File.ReadAllBytes(debugInfoFilename);
scriptAssembly = System.Reflection.Assembly.Load(asmBytes, pdbBytes);
}
}
if (null == scriptAssembly)
{
scriptAssembly = System.Reflection.Assembly.Load(asmBytes);
}
}
catch (System.IO.FileNotFoundException exception)
{
Log.ErrorMessage("Could not find assembly '{0}'", Graph.Instance.ScriptAssemblyPathname);
throw exception;
}
catch (System.Exception exception)
{
throw exception;
}
Graph.Instance.ScriptAssembly = scriptAssembly;
assemblyLoadProfile.StopProfile();
}
CompilePackageAssembly(
bool enforceBamAssemblyVersions = true,
bool enableClean = true)
{
// validate build root
if (null == Graph.Instance.BuildRoot)
{
throw new Exception("Build root has not been specified");
}
var gatherSourceProfile = new TimeProfile(ETimingProfiles.GatherSource);
gatherSourceProfile.StartProfile();
IdentifyAllPackages(enforceBamAssemblyVersions: enforceBamAssemblyVersions);
var cleanFirst = CommandLineProcessor.Evaluate(new Options.CleanFirst());
if (enableClean && cleanFirst && System.IO.Directory.Exists(Graph.Instance.BuildRoot))
{
Log.Info("Deleting build root '{0}'", Graph.Instance.BuildRoot);
try
{
// make sure no files are read-only, which may have happened as part of collation preserving file attributes
var dirInfo = new System.IO.DirectoryInfo(Graph.Instance.BuildRoot);
foreach (var file in dirInfo.EnumerateFiles("*", System.IO.SearchOption.AllDirectories))
{
file.Attributes &= ~System.IO.FileAttributes.ReadOnly;
}
System.IO.Directory.Delete(Graph.Instance.BuildRoot, true);
}
catch (System.IO.IOException ex)
{
Log.Info("Failed to delete build root, because {0}. Continuing", ex.Message);
}
}
BuildModeUtilities.ValidateBuildModePackage();
var definitions = new StringArray();
// gather source files
var sourceCode = new StringArray();
int packageIndex = 0;
foreach (var package in Graph.Instance.Packages)
{
Log.DebugMessage("{0}: '{1}' @ '{2}'", packageIndex, package.Version, (package.PackageRepositories.Count > 0) ? package.PackageRepositories[0] : "Not in a repository");
// to compile with debug information, you must compile the files
// to compile without, we need to file contents to hash the source
if (Graph.Instance.CompileWithDebugSymbols)
{
var scripts = package.GetScriptFiles();
sourceCode.AddRange(scripts);
Log.DebugMessage(scripts.ToString("\n\t"));
}
else
{
foreach (var scriptFile in package.GetScriptFiles())
{
using (var reader = new System.IO.StreamReader(scriptFile))
{
sourceCode.Add(reader.ReadToEnd());
}
Log.DebugMessage("\t'{0}'", scriptFile);
}
}
foreach (var define in package.Definitions)
{
if (!definitions.Contains(define))
{
definitions.Add(define);
}
}
++packageIndex;
}
// add/remove other definitions
definitions.Add(VersionDefineForCompiler);
definitions.Add(HostPlatformDefineForCompiler);
definitions.Sort();
gatherSourceProfile.StopProfile();
var assemblyCompileProfile = new TimeProfile(ETimingProfiles.AssemblyCompilation);
assemblyCompileProfile.StartProfile();
// assembly is written to the build root
var cachedAssemblyPathname = System.IO.Path.Combine(Graph.Instance.BuildRoot, ".CachedPackageAssembly");
cachedAssemblyPathname = System.IO.Path.Combine(cachedAssemblyPathname, Graph.Instance.MasterPackage.Name) + ".dll";
var hashPathName = System.IO.Path.ChangeExtension(cachedAssemblyPathname, "hash");
string thisHashCode = null;
var cacheAssembly = !CommandLineProcessor.Evaluate(new Options.DisableCacheAssembly());
string compileReason = null;
if (Graph.Instance.CompileWithDebugSymbols)
{
compileReason = "debug symbols were enabled";
}
else
{
// can an existing assembly be reused?
thisHashCode = GetPackageHash(sourceCode, definitions, Graph.Instance.MasterPackage.BamAssemblies);
if (cacheAssembly)
{
if (System.IO.File.Exists(hashPathName))
{
using (var reader = new System.IO.StreamReader(hashPathName))
{
var diskHashCode = reader.ReadLine();
if (diskHashCode.Equals(thisHashCode))
{
Log.DebugMessage("Cached assembly used '{0}', with hash {1}", cachedAssemblyPathname, diskHashCode);
Log.Detail("Re-using existing package assembly");
Graph.Instance.ScriptAssemblyPathname = cachedAssemblyPathname;
assemblyCompileProfile.StopProfile();
return;
}
else
{
compileReason = "package source has changed since the last compile";
}
}
}
else
{
compileReason = "no previously compiled package assembly exists";
}
}
else
{
compileReason = "user has disabled package assembly caching";
}
}
// use the compiler in the current runtime version to build the assembly of packages
var clrVersion = System.Environment.Version;
var compilerVersion = System.String.Format("v{0}.{1}", clrVersion.Major, clrVersion.Minor);
Log.Detail("Compiling package assembly (C# compiler {0}{1}), because {2}.",
compilerVersion,
Graph.Instance.ProcessState.TargetFrameworkVersion != null ? (", targetting " + Graph.Instance.ProcessState.TargetFrameworkVersion) : string.Empty,
compileReason);
var providerOptions = new System.Collections.Generic.Dictionary <string, string>();
providerOptions.Add("CompilerVersion", compilerVersion);
if (Graph.Instance.ProcessState.RunningMono)
{
Log.DebugMessage("Compiling assembly for Mono");
}
using (var provider = new Microsoft.CSharp.CSharpCodeProvider(providerOptions))
{
var compilerParameters = new System.CodeDom.Compiler.CompilerParameters();
compilerParameters.TreatWarningsAsErrors = true;
compilerParameters.WarningLevel = 4;
compilerParameters.GenerateExecutable = false;
compilerParameters.GenerateInMemory = false;
if (Graph.Instance.CompileWithDebugSymbols)
{
compilerParameters.OutputAssembly = System.IO.Path.Combine(System.IO.Path.GetTempPath(), Graph.Instance.MasterPackage.Name) + ".dll";
}
else
{
compilerParameters.OutputAssembly = cachedAssemblyPathname;
}
var compilerOptions = "/checked+ /unsafe-";
if (Graph.Instance.CompileWithDebugSymbols)
{
compilerParameters.IncludeDebugInformation = true;
compilerOptions += " /optimize-";
}
else
{
compilerOptions += " /optimize+";
}
compilerOptions += " /platform:anycpu";
// define strings
compilerOptions += " /define:" + definitions.ToString(';');
compilerParameters.CompilerOptions = compilerOptions;
if (provider.Supports(System.CodeDom.Compiler.GeneratorSupport.Resources))
{
// Bam assembly
// TODO: Q: why is it only for the master package? Why not all of them, which may have additional dependencies?
foreach (var assembly in Graph.Instance.MasterPackage.BamAssemblies)
{
var assemblyFileName = System.String.Format("{0}.dll", assembly.Name);
var assemblyPathName = System.IO.Path.Combine(Graph.Instance.ProcessState.ExecutableDirectory, assemblyFileName);
compilerParameters.ReferencedAssemblies.Add(assemblyPathName);
}
// DotNet assembly
foreach (var desc in Graph.Instance.MasterPackage.DotNetAssemblies)
{
var assemblyFileName = System.String.Format("{0}.dll", desc.Name);
compilerParameters.ReferencedAssemblies.Add(assemblyFileName);
}
if (Graph.Instance.ProcessState.RunningMono)
{
compilerParameters.ReferencedAssemblies.Add("Mono.Posix.dll");
}
}
else
{
throw new Exception("C# compiler does not support Resources");
}
// this will create the build root directory as necessary
IOWrapper.CreateDirectory(System.IO.Path.GetDirectoryName(compilerParameters.OutputAssembly));
var results = Graph.Instance.CompileWithDebugSymbols ?
provider.CompileAssemblyFromFile(compilerParameters, sourceCode.ToArray()) :
provider.CompileAssemblyFromSource(compilerParameters, sourceCode.ToArray());
if (results.Errors.HasErrors || results.Errors.HasWarnings)
{
var message = new System.Text.StringBuilder();
message.AppendFormat("Failed to compile package '{0}'. There are {1} errors.", Graph.Instance.MasterPackage.FullName, results.Errors.Count);
message.AppendLine();
foreach (System.CodeDom.Compiler.CompilerError error in results.Errors)
{
message.AppendFormat("\t{0}({1}): {2} {3}", error.FileName, error.Line, error.ErrorNumber, error.ErrorText);
message.AppendLine();
}
if (!Graph.Instance.CompileWithDebugSymbols)
{
message.AppendLine();
ICommandLineArgument debugOption = new Options.UseDebugSymbols();
message.AppendFormat("Use the {0}/{1} command line option with bam for more accurate error messages.", debugOption.LongName, debugOption.ShortName);
message.AppendLine();
}
message.AppendLine();
ICommandLineArgument createDebugProjectOption = new Options.CreateDebugProject();
message.AppendFormat("Use the {0}/{1} command line option with bam to create an editable IDE project containing the build scripts.", createDebugProjectOption.LongName, createDebugProjectOption.ShortName);
message.AppendLine();
throw new Exception(message.ToString());
}
if (!Graph.Instance.CompileWithDebugSymbols)
{
if (cacheAssembly)
{
using (var writer = new System.IO.StreamWriter(hashPathName))
{
writer.WriteLine(thisHashCode);
}
}
else
{
// will not throw if the file doesn't exist
System.IO.File.Delete(hashPathName);
}
}
Log.DebugMessage("Written assembly to '{0}'", compilerParameters.OutputAssembly);
Graph.Instance.ScriptAssemblyPathname = compilerParameters.OutputAssembly;
}
assemblyCompileProfile.StopProfile();
}
Execute(
Array <Environment> environments,
System.Reflection.Assembly packageAssembly = null)
{
PrintVersion();
if (0 == environments.Count)
{
throw new Exception("No build configurations were specified");
}
var graph = Graph.Instance;
if (null != packageAssembly)
{
PackageUtilities.IdentifyAllPackages();
graph.ScriptAssembly = packageAssembly;
graph.ScriptAssemblyPathname = packageAssembly.Location;
}
else
{
PackageUtilities.CompilePackageAssembly();
PackageUtilities.LoadPackageAssembly();
}
var packageMetaDataProfile = new TimeProfile(ETimingProfiles.PackageMetaData);
packageMetaDataProfile.StartProfile();
// validate that there is at most one local policy
// if test mode is enabled, then the '.tests' sub-namespaces are also checked
{
var localPolicies = graph.ScriptAssembly.GetTypes().Where(t => typeof(ISitePolicy).IsAssignableFrom(t));
var includeTests = CommandLineProcessor.Evaluate(new Options.UseTests());
if (!includeTests)
{
localPolicies = localPolicies.Where(item => !item.Namespace.EndsWith(".tests"));
}
var numLocalPolicies = localPolicies.Count();
if (numLocalPolicies > 0)
{
if (numLocalPolicies > 1)
{
var message = new System.Text.StringBuilder();
message.AppendLine("Too many site policies exist in the package assembly:");
foreach (var policy in localPolicies)
{
message.AppendFormat("\t{0}", policy.ToString());
message.AppendLine();
}
throw new Exception(message.ToString());
}
Settings.LocalPolicy = System.Activator.CreateInstance(localPolicies.First()) as ISitePolicy;
}
}
// find a product definition
{
var productDefinitions = graph.ScriptAssembly.GetTypes().Where(t => typeof(IProductDefinition).IsAssignableFrom(t));
var numProductDefinitions = productDefinitions.Count();
if (numProductDefinitions > 0)
{
if (numProductDefinitions > 1)
{
var message = new System.Text.StringBuilder();
message.AppendLine("Too many product definitions exist in the package assembly:");
foreach (var def in productDefinitions)
{
message.AppendFormat("\t{0}", def.ToString());
message.AppendLine();
}
throw new Exception(message.ToString());
}
graph.ProductDefinition = System.Activator.CreateInstance(productDefinitions.First()) as IProductDefinition;
}
}
// get the metadata from the build mode package
var metaName = System.String.Format("{0}Builder.{0}Meta", graph.Mode);
var metaDataType = graph.ScriptAssembly.GetType(metaName);
if (null == metaDataType)
{
throw new Exception("No build mode {0} meta data type {1}", graph.Mode, metaName);
}
if (!typeof(IBuildModeMetaData).IsAssignableFrom(metaDataType))
{
throw new Exception("Build mode package meta data type {0} does not implement the interface {1}", metaDataType.ToString(), typeof(IBuildModeMetaData).ToString());
}
graph.BuildModeMetaData = System.Activator.CreateInstance(metaDataType) as IBuildModeMetaData;
// packages can have meta data - instantiate where they exist
foreach (var package in graph.Packages)
{
var ns = package.Name;
var metaType = graph.ScriptAssembly.GetTypes().FirstOrDefault(item => item.Namespace == ns && typeof(PackageMetaData).IsAssignableFrom(item));
if (null == metaType)
{
continue;
}
try
{
package.MetaData = System.Activator.CreateInstance(metaType) as PackageMetaData;
}
catch (Exception exception)
{
throw exception;
}
catch (System.Reflection.TargetInvocationException exception)
{
throw new Exception(exception, "Failed to create package metadata");
}
}
packageMetaDataProfile.StopProfile();
var topLevelNamespace = graph.MasterPackage.Name;
var findBuildableModulesProfile = new TimeProfile(ETimingProfiles.IdentifyBuildableModules);
findBuildableModulesProfile.StartProfile();
// Phase 1: Instantiate all modules in the namespace of the package in which the tool was invoked
Log.Detail("Creating modules");
foreach (var env in environments)
{
graph.CreateTopLevelModules(graph.ScriptAssembly, env, topLevelNamespace);
}
findBuildableModulesProfile.StopProfile();
var populateGraphProfile = new TimeProfile(ETimingProfiles.PopulateGraph);
populateGraphProfile.StartProfile();
// Phase 2: Graph now has a linear list of modules; create a dependency graph
// NB: all those modules with 0 dependees are the top-level modules
// NB: default settings have already been defined here
// not only does this generate the dependency graph, but also creates the default settings for each module, and completes them
graph.SortDependencies();
populateGraphProfile.StopProfile();
// TODO: make validation optional, if it starts showing on profiles
var validateGraphProfile = new TimeProfile(ETimingProfiles.ValidateGraph);
validateGraphProfile.StartProfile();
graph.Validate();
validateGraphProfile.StopProfile();
// Phase 3: (Create default settings, and ) apply patches (build + shared) to each module
// NB: some builders can use the patch directly for child objects, so this may be dependent upon the builder
// Toolchains for modules need to be set here, as they might append macros into each module in order to evaluate paths
// TODO: a parallel thread can be spawned here, that can check whether command lines have changed
// the Settings object can be inspected, and a hash generated. This hash can be written to disk, and compared.
// If a 'verbose' mode is enabled, then more work can be done to figure out what has changed. This would also require
// serializing the binary Settings object
var createPatchesProfile = new TimeProfile(ETimingProfiles.CreatePatches);
createPatchesProfile.StartProfile();
graph.ApplySettingsPatches();
createPatchesProfile.StopProfile();
// expand paths after patching settings, because some of the patches may contain tokenized strings
// TODO: a thread can be spawned, to check for whether files were in date or not, which will
// be ready in time for graph execution
var parseStringsProfile = new TimeProfile(ETimingProfiles.ParseTokenizedStrings);
parseStringsProfile.StartProfile();
TokenizedString.ParseAll();
parseStringsProfile.StopProfile();
if (CommandLineProcessor.Evaluate(new Options.ViewDependencyGraph()))
{
// must come after all strings are parsed, in order to display useful paths
graph.Dump();
}
// Phase 4: Execute dependency graph
// N.B. all paths (including those with macros) have been delayed expansion until now
var graphExecutionProfile = new TimeProfile(ETimingProfiles.GraphExecution);
graphExecutionProfile.StartProfile();
var executor = new Executor();
executor.Run();
graphExecutionProfile.StopProfile();
}
RegisterProfile(
TimeProfile profile)
{
Profiles.Add(profile);
}
Main(
string[] args)
{
#if false
// take control of Ctrl+C
System.Console.CancelKeyPress += new System.ConsoleCancelEventHandler(HandleCancellation);
#endif
try
{
var totalTimeProfile = new Core.TimeProfile(Core.ETimingProfiles.TimedTotal);
var processCommandLineProfile = new Core.TimeProfile(Core.ETimingProfiles.ProcessCommandLine);
totalTimeProfile.StartProfile();
processCommandLineProfile.StartProfile();
var verbosityLevel = (Core.EVerboseLevel)Core.CommandLineProcessor.Evaluate(new Core.Options.VerbosityLevel());
switch (verbosityLevel)
{
case Core.EVerboseLevel.None:
case Core.EVerboseLevel.Info:
case Core.EVerboseLevel.Detail:
case Core.EVerboseLevel.Full:
Core.Graph.Instance.VerbosityLevel = verbosityLevel;
break;
default:
throw new Core.Exception("Unrecognized verbosity level, {0}", verbosityLevel);
}
if (Core.CommandLineProcessor.Evaluate(new Core.Options.PrintHelp()))
{
CommandLineArgumentHelper.PrintHelp();
return;
}
if (Core.CommandLineProcessor.Evaluate(new Core.Options.PrintVersion()))
{
CommandLineArgumentHelper.PrintVersion();
return;
}
if (Core.CommandLineProcessor.Evaluate(new Core.Options.CreateDebugProject()))
{
DebugProject.Create();
return;
}
if (Core.CommandLineProcessor.Evaluate(new Core.Options.MakePackage()))
{
Core.PackageUtilities.MakePackage();
return;
}
if (Core.CommandLineProcessor.Evaluate(new Core.Options.AddDependentPackage()))
{
Core.PackageUtilities.AddDependentPackage();
return;
}
if (Core.CommandLineProcessor.Evaluate(new Core.Options.ShowDefinitionFile()))
{
Core.PackageUtilities.IdentifyAllPackages(allowDuplicates: true, enforceBamAssemblyVersions: false);
Core.Graph.Instance.MasterPackage.Show();
return;
}
// configure
Core.Graph.Instance.BuildRoot = Core.CommandLineProcessor.Evaluate(new Core.Options.BuildRoot());
Core.Graph.Instance.Mode = Core.CommandLineProcessor.Evaluate(new Core.Options.BuildMode());
if (null == Core.Graph.Instance.Mode)
{
throw new Core.Exception("No build mode specified");
}
var configs = new Core.Array <Core.Environment>();
var requestedConfigs = Core.CommandLineProcessor.Evaluate(new Core.Options.BuildConfigurations());
if (0 == requestedConfigs.Count)
{
// default
requestedConfigs.Add(new Core.StringArray("debug"));
}
foreach (var configOption in requestedConfigs)
{
foreach (var config in configOption)
{
var env = new Core.Environment();
env.Configuration = Core.Configuration.FromString(config);
configs.Add(env);
}
}
processCommandLineProfile.StopProfile();
Core.EntryPoint.Execute(configs);
totalTimeProfile.StopProfile();
}
catch (Core.Exception exception)
{
Core.Exception.DisplayException(exception);
System.Environment.ExitCode = -1;
}
catch (System.Exception exception)
{
var message = new System.Text.StringBuilder();
message.AppendFormat("{0} not handled: {1}", exception.GetType().ToString(), exception.Message);
message.AppendLine();
message.AppendLine(exception.StackTrace);
Core.Log.ErrorMessage(message.ToString());
System.Environment.ExitCode = -2;
}
finally
{
if (Core.Graph.Instance.BuildEnvironments.Count > 0)
{
Core.Log.Info((0 == System.Environment.ExitCode) ? "\nBuild Succeeded" : "\nBuild Failed");
if (Core.CommandLineProcessor.Evaluate(new Core.Options.PrintStatistics()))
{
Core.Statistics.Display();
}
Core.Log.DebugMessage("Exit code {0}", System.Environment.ExitCode);
}
}
}
Main(
string[] args)
{
#if false
// take control of Ctrl+C
System.Console.CancelKeyPress += new System.ConsoleCancelEventHandler(HandleCancellation);
#endif
try
{
var totalTimeProfile = new Core.TimeProfile(Core.ETimingProfiles.TimedTotal);
var processCommandLineProfile = new Core.TimeProfile(Core.ETimingProfiles.ProcessCommandLine);
totalTimeProfile.StartProfile();
processCommandLineProfile.StartProfile();
var verbosityLevel = (Core.EVerboseLevel)Core.CommandLineProcessor.Evaluate(new Core.Options.VerbosityLevel());
switch (verbosityLevel)
{
case Core.EVerboseLevel.None:
case Core.EVerboseLevel.Info:
case Core.EVerboseLevel.Detail:
case Core.EVerboseLevel.Full:
Core.Graph.Instance.VerbosityLevel = verbosityLevel;
break;
default:
throw new Core.Exception("Unrecognized verbosity level, {0}", verbosityLevel);
}
if (Core.CommandLineProcessor.Evaluate(new Core.Options.PrintHelp()))
{
CommandLineArgumentHelper.PrintHelp();
return;
}
if (Core.CommandLineProcessor.Evaluate(new Core.Options.PrintVersion()))
{
CommandLineArgumentHelper.PrintVersion();
return;
}
if (Core.CommandLineProcessor.Evaluate(new Core.Options.CreateDebugProject()))
{
DebugProject.Create();
return;
}
if (Core.CommandLineProcessor.Evaluate(new Core.Options.MakePackage()))
{
Core.PackageUtilities.MakePackage();
return;
}
if (Core.CommandLineProcessor.Evaluate(new Core.Options.AddDependentPackage()))
{
Core.PackageUtilities.AddDependentPackage();
return;
}
if (Core.CommandLineProcessor.Evaluate(new Core.Options.ShowDefinitionFile()))
{
Core.PackageUtilities.IdentifyAllPackages(allowDuplicates: true, enforceBamAssemblyVersions: false);
Core.Graph.Instance.MasterPackage.Show();
return;
}
// configure
Core.Graph.Instance.BuildRoot = Core.CommandLineProcessor.Evaluate(new Core.Options.BuildRoot());
Core.Graph.Instance.Mode = Core.CommandLineProcessor.Evaluate(new Core.Options.BuildMode());
if (null == Core.Graph.Instance.Mode)
{
throw new Core.Exception("No build mode specified");
}
var configs = new Core.Array<Core.Environment>();
var requestedConfigs = Core.CommandLineProcessor.Evaluate(new Core.Options.BuildConfigurations());
if (0 == requestedConfigs.Count)
{
// default
requestedConfigs.Add(new Core.StringArray("debug"));
}
foreach (var configOption in requestedConfigs)
{
foreach (var config in configOption)
{
var env = new Core.Environment();
env.Configuration = Core.Configuration.FromString(config);
configs.Add(env);
}
}
processCommandLineProfile.StopProfile();
Core.EntryPoint.Execute(configs);
totalTimeProfile.StopProfile();
}
catch (Core.Exception exception)
{
Core.Exception.DisplayException(exception);
System.Environment.ExitCode = -1;
}
catch (System.Exception exception)
{
var message = new System.Text.StringBuilder();
message.AppendFormat("{0} not handled: {1}", exception.GetType().ToString(), exception.Message);
message.AppendLine();
message.AppendLine(exception.StackTrace);
Core.Log.ErrorMessage(message.ToString());
System.Environment.ExitCode = -2;
}
finally
{
if (Core.Graph.Instance.BuildEnvironments.Count > 0)
{
Core.Log.Info((0 == System.Environment.ExitCode) ? "\nBuild Succeeded" : "\nBuild Failed");
if (Core.CommandLineProcessor.Evaluate(new Core.Options.PrintStatistics()))
{
Core.Statistics.Display();
}
Core.Log.DebugMessage("Exit code {0}", System.Environment.ExitCode);
}
}
}
Execute(
Array<Environment> environments,
System.Reflection.Assembly packageAssembly = null)
{
PrintVersion();
if (0 == environments.Count)
{
throw new Exception("No build configurations were specified");
}
if (null != packageAssembly)
{
PackageUtilities.IdentifyAllPackages();
Graph.Instance.ScriptAssembly = packageAssembly;
Graph.Instance.ScriptAssemblyPathname = packageAssembly.Location;
}
else
{
var compiledSuccessfully = PackageUtilities.CompilePackageAssembly();
if (!compiledSuccessfully)
{
throw new Exception("Package compilation failed");
}
PackageUtilities.LoadPackageAssembly();
}
var packageMetaDataProfile = new TimeProfile(ETimingProfiles.PackageMetaData);
packageMetaDataProfile.StartProfile();
// get the metadata from the build mode package
var graph = Graph.Instance;
var metaName = System.String.Format("{0}Builder.{0}Meta", graph.Mode);
var metaDataType = graph.ScriptAssembly.GetType(metaName);
if (null == metaDataType)
{
throw new Exception("No build mode {0} meta data type {1}", graph.Mode, metaName);
}
if (!typeof(IBuildModeMetaData).IsAssignableFrom(metaDataType))
{
throw new Exception("Build mode package meta data type {0} does not implement the interface {1}", metaDataType.ToString(), typeof(IBuildModeMetaData).ToString());
}
graph.BuildModeMetaData = System.Activator.CreateInstance(metaDataType) as IBuildModeMetaData;
// packages can have meta data - instantiate where they exist
foreach (var package in graph.Packages)
{
var ns = package.Name;
var metaType = graph.ScriptAssembly.GetTypes().Where(item => item.Namespace == ns && typeof(PackageMetaData).IsAssignableFrom(item)).FirstOrDefault();
if (null != metaType)
{
try
{
package.MetaData = System.Activator.CreateInstance(metaType) as PackageMetaData;
}
catch (Exception exception)
{
throw exception;
}
catch (System.Reflection.TargetInvocationException exception)
{
throw new Exception(exception, "Failed to create package metadata");
}
}
}
packageMetaDataProfile.StopProfile();
var topLevelNamespace = graph.MasterPackage.Name;
var findBuildableModulesProfile = new TimeProfile(ETimingProfiles.IdentifyBuildableModules);
findBuildableModulesProfile.StartProfile();
// Phase 1: Instantiate all modules in the namespace of the package in which the tool was invoked
Log.Detail("Creating modules");
foreach (var env in environments)
{
graph.CreateTopLevelModules(graph.ScriptAssembly, env, topLevelNamespace);
}
findBuildableModulesProfile.StopProfile();
var populateGraphProfile = new TimeProfile(ETimingProfiles.PopulateGraph);
populateGraphProfile.StartProfile();
// Phase 2: Graph now has a linear list of modules; create a dependency graph
// NB: all those modules with 0 dependees are the top-level modules
// NB: default settings have already been defined here
// not only does this generate the dependency graph, but also creates the default settings for each module, and completes them
graph.SortDependencies();
populateGraphProfile.StopProfile();
// TODO: make validation optional, if it starts showing on profiles
var validateGraphProfile = new TimeProfile(ETimingProfiles.ValidateGraph);
validateGraphProfile.StartProfile();
graph.Validate();
validateGraphProfile.StopProfile();
// Phase 3: (Create default settings, and ) apply patches (build + shared) to each module
// NB: some builders can use the patch directly for child objects, so this may be dependent upon the builder
// Toolchains for modules need to be set here, as they might append macros into each module in order to evaluate paths
// TODO: a parallel thread can be spawned here, that can check whether command lines have changed
// the Settings object can be inspected, and a hash generated. This hash can be written to disk, and compared.
// If a 'verbose' mode is enabled, then more work can be done to figure out what has changed. This would also require
// serializing the binary Settings object
var createPatchesProfile = new TimeProfile(ETimingProfiles.CreatePatches);
createPatchesProfile.StartProfile();
graph.ApplySettingsPatches();
createPatchesProfile.StopProfile();
// expand paths after patching settings, because some of the patches may contain tokenized strings
// TODO: a thread can be spawned, to check for whether files were in date or not, which will
// be ready in time for graph execution
var parseStringsProfile = new TimeProfile(ETimingProfiles.ParseTokenizedStrings);
parseStringsProfile.StartProfile();
TokenizedString.ParseAll();
parseStringsProfile.StopProfile();
if (CommandLineProcessor.Evaluate(new Options.ViewDependencyGraph()))
{
// must come after all strings are parsed, in order to display useful paths
graph.Dump();
}
// Phase 4: Execute dependency graph
// N.B. all paths (including those with macros) have been delayed expansion until now
var graphExecutionProfile = new TimeProfile(ETimingProfiles.GraphExecution);
graphExecutionProfile.StartProfile();
var executor = new Executor();
executor.Run();
graphExecutionProfile.StopProfile();
}
RegisterProfile(
TimeProfile profile)
{
Profiles.Add(profile);
}
LoadPackageAssembly()
{
var assemblyLoadProfile = new TimeProfile(ETimingProfiles.LoadAssembly);
assemblyLoadProfile.StartProfile();
System.Reflection.Assembly scriptAssembly = null;
try
{
// this code works from an untrusted location, and debugging IS available when
// the pdb (.NET)/mdb (Mono) resides beside the assembly
byte[] asmBytes = System.IO.File.ReadAllBytes(Graph.Instance.ScriptAssemblyPathname);
if (Graph.Instance.CompileWithDebugSymbols)
{
var debugInfoFilename = Graph.Instance.ProcessState.RunningMono ?
Graph.Instance.ScriptAssemblyPathname + ".mdb" :
System.IO.Path.ChangeExtension(Graph.Instance.ScriptAssemblyPathname, ".pdb");
if (System.IO.File.Exists(debugInfoFilename))
{
byte[] pdbBytes = System.IO.File.ReadAllBytes(debugInfoFilename);
scriptAssembly = System.Reflection.Assembly.Load(asmBytes, pdbBytes);
}
}
if (null == scriptAssembly)
{
scriptAssembly = System.Reflection.Assembly.Load(asmBytes);
}
}
catch (System.IO.FileNotFoundException exception)
{
Log.ErrorMessage("Could not find assembly '{0}'", Graph.Instance.ScriptAssemblyPathname);
throw exception;
}
catch (System.Exception exception)
{
throw exception;
}
Graph.Instance.ScriptAssembly = scriptAssembly;
assemblyLoadProfile.StopProfile();
}
CompilePackageAssembly(
bool enforceBamAssemblyVersions = true,
bool enableClean = true)
{
// validate build root
if (null == Graph.Instance.BuildRoot)
{
throw new Exception("Build root has not been specified");
}
var gatherSourceProfile = new TimeProfile(ETimingProfiles.GatherSource);
gatherSourceProfile.StartProfile();
IdentifyAllPackages(enforceBamAssemblyVersions: enforceBamAssemblyVersions);
var cleanFirst = CommandLineProcessor.Evaluate(new Options.CleanFirst());
if (enableClean && cleanFirst && System.IO.Directory.Exists(Graph.Instance.BuildRoot))
{
Log.Info("Deleting build root '{0}'", Graph.Instance.BuildRoot);
try
{
System.IO.Directory.Delete(Graph.Instance.BuildRoot, true);
}
catch (System.IO.IOException ex)
{
Log.Info("Failed to delete build root, because {0}. Continuing", ex.Message);
}
}
if (!System.IO.Directory.Exists(Graph.Instance.BuildRoot))
{
System.IO.Directory.CreateDirectory(Graph.Instance.BuildRoot);
}
BuildModeUtilities.ValidateBuildModePackage();
var definitions = new StringArray();
// gather source files
var sourceCode = new StringArray();
int packageIndex = 0;
foreach (var package in Graph.Instance.Packages)
{
Log.DebugMessage("{0}: '{1}' @ '{2}'", packageIndex, package.Version, (package.PackageRepositories.Count > 0) ? package.PackageRepositories[0] : "Not in a repository");
// to compile with debug information, you must compile the files
// to compile without, we need to file contents to hash the source
if (Graph.Instance.CompileWithDebugSymbols)
{
var scripts = package.GetScriptFiles();
sourceCode.AddRange(scripts);
Log.DebugMessage(scripts.ToString("\n\t"));
}
else
{
foreach (var scriptFile in package.GetScriptFiles())
{
using (var reader = new System.IO.StreamReader(scriptFile))
{
sourceCode.Add(reader.ReadToEnd());
}
Log.DebugMessage("\t'{0}'", scriptFile);
}
}
foreach (var define in package.Definitions)
{
if (!definitions.Contains(define))
{
definitions.Add(define);
}
}
++packageIndex;
}
// add/remove other definitions
definitions.Add(VersionDefineForCompiler);
definitions.Add(HostPlatformDefineForCompiler);
definitions.Sort();
gatherSourceProfile.StopProfile();
var assemblyCompileProfile = new TimeProfile(ETimingProfiles.AssemblyCompilation);
assemblyCompileProfile.StartProfile();
// assembly is written to the build root
var cachedAssemblyPathname = System.IO.Path.Combine(Graph.Instance.BuildRoot, "CachedPackageAssembly");
cachedAssemblyPathname = System.IO.Path.Combine(cachedAssemblyPathname, Graph.Instance.MasterPackage.Name) + ".dll";
var hashPathName = System.IO.Path.ChangeExtension(cachedAssemblyPathname, "hash");
string thisHashCode = null;
var cacheAssembly = !CommandLineProcessor.Evaluate(new Options.DisableCacheAssembly());
string compileReason = null;
if (Graph.Instance.CompileWithDebugSymbols)
{
compileReason = "debug symbols were enabled";
}
else
{
// can an existing assembly be reused?
thisHashCode = GetPackageHash(sourceCode, definitions, Graph.Instance.MasterPackage.BamAssemblies);
if (cacheAssembly)
{
if (System.IO.File.Exists(hashPathName))
{
using (var reader = new System.IO.StreamReader(hashPathName))
{
var diskHashCode = reader.ReadLine();
if (diskHashCode.Equals(thisHashCode))
{
Log.DebugMessage("Cached assembly used '{0}', with hash {1}", cachedAssemblyPathname, diskHashCode);
Log.Detail("Re-using existing package assembly");
Graph.Instance.ScriptAssemblyPathname = cachedAssemblyPathname;
assemblyCompileProfile.StopProfile();
return true;
}
else
{
compileReason = "package source has changed since the last compile";
}
}
}
else
{
compileReason = "no previously compiled package assembly exists";
}
}
else
{
compileReason = "user has disabled package assembly caching";
}
}
// use the compiler in the current runtime version to build the assembly of packages
var clrVersion = System.Environment.Version;
var compilerVersion = System.String.Format("v{0}.{1}", clrVersion.Major, clrVersion.Minor);
Log.Detail("Compiling package assembly (C# compiler {0}{1}), because {2}.",
compilerVersion,
Graph.Instance.ProcessState.TargetFrameworkVersion != null ? (", targetting " + Graph.Instance.ProcessState.TargetFrameworkVersion) : string.Empty,
compileReason);
var providerOptions = new System.Collections.Generic.Dictionary<string, string>();
providerOptions.Add("CompilerVersion", compilerVersion);
if (Graph.Instance.ProcessState.RunningMono)
{
Log.DebugMessage("Compiling assembly for Mono");
}
using (var provider = new Microsoft.CSharp.CSharpCodeProvider(providerOptions))
{
var compilerParameters = new System.CodeDom.Compiler.CompilerParameters();
compilerParameters.TreatWarningsAsErrors = true;
compilerParameters.WarningLevel = 4;
compilerParameters.GenerateExecutable = false;
compilerParameters.GenerateInMemory = false;
if (Graph.Instance.CompileWithDebugSymbols)
{
compilerParameters.OutputAssembly = System.IO.Path.Combine(System.IO.Path.GetTempPath(), Graph.Instance.MasterPackage.Name) + ".dll";
}
else
{
compilerParameters.OutputAssembly = cachedAssemblyPathname;
}
var compilerOptions = "/checked+ /unsafe-";
if (Graph.Instance.CompileWithDebugSymbols)
{
compilerParameters.IncludeDebugInformation = true;
compilerOptions += " /optimize-";
}
else
{
compilerOptions += " /optimize+";
}
compilerOptions += " /platform:anycpu";
// define strings
compilerOptions += " /define:" + definitions.ToString(';');
compilerParameters.CompilerOptions = compilerOptions;
if (provider.Supports(System.CodeDom.Compiler.GeneratorSupport.Resources))
{
// Bam assembly
// TODO: Q: why is it only for the master package? Why not all of them, which may have additional dependencies?
foreach (var assembly in Graph.Instance.MasterPackage.BamAssemblies)
{
var assemblyFileName = System.String.Format("{0}.dll", assembly.Name);
var assemblyPathName = System.IO.Path.Combine(Graph.Instance.ProcessState.ExecutableDirectory, assemblyFileName);
compilerParameters.ReferencedAssemblies.Add(assemblyPathName);
}
// DotNet assembly
foreach (var desc in Graph.Instance.MasterPackage.DotNetAssemblies)
{
var assemblyFileName = System.String.Format("{0}.dll", desc.Name);
compilerParameters.ReferencedAssemblies.Add(assemblyFileName);
}
if (Graph.Instance.ProcessState.RunningMono)
{
compilerParameters.ReferencedAssemblies.Add("Mono.Posix.dll");
}
}
else
{
throw new Exception("C# compiler does not support Resources");
}
System.IO.Directory.CreateDirectory(System.IO.Path.GetDirectoryName(compilerParameters.OutputAssembly));
var results = Graph.Instance.CompileWithDebugSymbols ?
provider.CompileAssemblyFromFile(compilerParameters, sourceCode.ToArray()) :
provider.CompileAssemblyFromSource(compilerParameters, sourceCode.ToArray());
if (results.Errors.HasErrors || results.Errors.HasWarnings)
{
Log.ErrorMessage("Failed to compile package '{0}'. There are {1} errors.", Graph.Instance.MasterPackage.FullName, results.Errors.Count);
foreach (System.CodeDom.Compiler.CompilerError error in results.Errors)
{
Log.ErrorMessage("\t{0}({1}): {2} {3}", error.FileName, error.Line, error.ErrorNumber, error.ErrorText);
}
return false;
}
if (!Graph.Instance.CompileWithDebugSymbols)
{
if (cacheAssembly)
{
using (var writer = new System.IO.StreamWriter(hashPathName))
{
writer.WriteLine(thisHashCode);
}
}
else
{
// will not throw if the file doesn't exist
System.IO.File.Delete(hashPathName);
}
}
Log.DebugMessage("Written assembly to '{0}'", compilerParameters.OutputAssembly);
Graph.Instance.ScriptAssemblyPathname = compilerParameters.OutputAssembly;
}
assemblyCompileProfile.StopProfile();
return true;
}
