Exemplo n.º 1
0
        Initialize()
        {
            this.ProcessState = new BamState();

            OSUtilities.SetupPlatform();

            this.Modules                  = new System.Collections.Generic.Dictionary <Environment, System.Collections.Generic.List <Module> >();
            this.ReferencedModules        = new System.Collections.Generic.Dictionary <Environment, Array <Module> >();
            this.TopLevelModules          = new System.Collections.Generic.List <Module>();
            this.Macros                   = new MacroList(this.GetType().FullName);
            this.BuildEnvironmentInternal = null;
            this.CommonModuleType         = new System.Collections.Generic.Stack <System.Type>();
            this.DependencyGraph          = new DependencyGraph();
            this.MetaData                 = null;

            this.PackageRepositories = new StringArray();
            var primaryPackageRepo = System.IO.Path.Combine(
                System.IO.Directory.GetParent(System.IO.Directory.GetParent(this.ProcessState.ExecutableDirectory).FullName).FullName,
                "packages");

            this.PackageRepositories.AddUnique(primaryPackageRepo);

            this.ForceDefinitionFileUpdate = CommandLineProcessor.Evaluate(new Options.ForceDefinitionFileUpdate());
            this.CompileWithDebugSymbols   = CommandLineProcessor.Evaluate(new Options.UseDebugSymbols());
        }
Exemplo n.º 2
0
        CreateTopLevelModules(
            System.Reflection.Assembly assembly,
            Environment env,
            string ns)
        {
            this.BuildEnvironment = env;
            var includeTests    = CommandLineProcessor.Evaluate(new Options.UseTests());
            var allTypes        = assembly.GetTypes();
            var allPackageTypes = allTypes.Where(type => ((type.Namespace == ns) || (includeTests && (type.Namespace == ns + ".tests"))) && type.IsSubclassOf(typeof(Module)) && type.IsSealed);

            foreach (var moduleType in allPackageTypes)
            {
                var newModule = MakeModuleOfType(moduleType);
                if (newModule != null)
                {
                    this.TopLevelModules.Add(newModule);
                }
            }
            this.BuildEnvironment = null;
            if (0 == this.TopLevelModules.Count)
            {
                throw new Exception("No modules found in the namespace '{0}'", ns);
            }
            // remove all top level modules that have a reference count > 1
            foreach (var tlm in this.TopLevelModules.Reverse <Module>())
            {
                if (!tlm.TopLevel)
                {
                    this.TopLevelModules.Remove(tlm);
                }
            }
        }
Exemplo n.º 3
0
        /// <summary>
        /// Protected constructor (use Init function in general use to configure a module) for a new module. Use Module.Create
        /// to create new instances of a module.
        /// This defines the standard macros for all modules:
        /// 'bampackagedir' - the directory in which the 'bam' directory resides
        /// 'packagedir' - same as 'bampackagedir' unless a Bam.Core.PackageDirectoryRedirect attribute is specified
        /// 'packagename' - name of the package
        /// 'packagebuilddir' - equivalent to $(buildroot)/$(packagename)
        /// 'modulename' - name of the module
        /// 'OutputName' - the 'main' part of the filename for the output of the module, but may have a module specific prefix and/or suffix
        /// 'config' - the textual name of the configuration for the module
        /// </summary>
        protected Module()
        {
            var graph = Graph.Instance;

            if (null == graph.BuildEnvironment)
            {
                throw new Exception("No build environment for module {0}", this.GetType().ToString());
            }

            graph.AddModule(this);
            this.TokenizedStringCacheMap = new System.Collections.Generic.Dictionary <System.Int64, TokenizedString>();
            this.Macros = new MacroList(this.GetType().FullName);
            // TODO: Can this be generalized to be a collection of files?
            this.GeneratedPaths = new System.Collections.Generic.Dictionary <PathKey, TokenizedString>();

            // capture the details of the encapsulating module
            this.EncapsulatingModule = null;
            this.EncapsulatingType   = graph.CommonModuleType.Peek();

            // add the package root
            try
            {
                var packageNameSpace  = this.EncapsulatingType.Namespace;
                var packageDefinition = graph.Packages.FirstOrDefault(item => item.Name == packageNameSpace);
                if (null == packageDefinition)
                {
                    var includeTests = CommandLineProcessor.Evaluate(new Options.UseTests());
                    if (includeTests && packageNameSpace.EndsWith(".tests"))
                    {
                        packageNameSpace  = packageNameSpace.Replace(".tests", string.Empty);
                        packageDefinition = graph.Packages.FirstOrDefault(item => item.Name == packageNameSpace);
                    }

                    if (null == packageDefinition)
                    {
                        throw new Exception("Unable to locate package for namespace '{0}'", packageNameSpace);
                    }
                }
                this.PackageDefinition = packageDefinition;
                this.Macros.AddVerbatim("bampackagedir", packageDefinition.GetPackageDirectory());
                this.AddRedirectedPackageDirectory(this);
                this.Macros.AddVerbatim("packagename", packageDefinition.Name);
                this.Macros.AddVerbatim("packagebuilddir", packageDefinition.GetBuildDirectory());
            }
            catch (System.NullReferenceException)
            {
                // graph.Packages can be null during unittests
            }
            this.Macros.AddVerbatim("modulename", this.GetType().Name);
            this.Macros.Add("OutputName", this.Macros["modulename"]);

            this.OwningRank       = null;
            this.Tool             = null;
            this.MetaData         = null;
            this.BuildEnvironment = graph.BuildEnvironment;
            this.Macros.AddVerbatim("config", this.BuildEnvironment.Configuration.ToString());
            this.ReasonToExecute = ExecuteReasoning.Undefined();
            this.ExecutionTask   = null;
            this.EvaluationTask  = null;
        }
Exemplo n.º 4
0
        AddDependentPackage()
        {
            var packageNameArgument = new Options.PackageName();
            var packageName         = CommandLineProcessor.Evaluate(packageNameArgument);

            if (null == packageName)
            {
                throw new Exception("No name was defined. Use {0} on the command line to specify it.", (packageNameArgument as ICommandLineArgument).LongName);
            }

            var packageVersion = CommandLineProcessor.Evaluate(new Options.PackageVersion());

            var masterPackage = GetMasterPackage();

            // TODO: no checking if this package exists
            // TODO: is this dependent already present?
            if (null != masterPackage.Dependents.Where(item => item.Item1 == packageName && item.Item2 == packageVersion).FirstOrDefault())
            {
                if (null != packageVersion)
                {
                    throw new Exception("Dependency {0}, version {1}, already exists", packageName, packageVersion);
                }
                else
                {
                    throw new Exception("Dependency {0} already exists", packageName);
                }
            }

            masterPackage.Dependents.Add(new System.Tuple <string, string, bool?>(packageName, packageVersion, null));
            masterPackage.Write();
        }
Exemplo n.º 5
0
        CreateTopLevelModules(
            System.Reflection.Assembly assembly,
            Environment env,
            string ns)
        {
            var includeTests            = CommandLineProcessor.Evaluate(new Options.UseTests());
            var allTypes                = assembly.GetTypes();
            var allModuleTypesInPackage = allTypes.Where(type => ((type.Namespace == ns) || (includeTests && (type.Namespace == ns + ".tests"))) && type.IsSubclassOf(typeof(Module)));

            if (0 == allModuleTypesInPackage.Count())
            {
                throw new Exception("No modules found in the namespace '{0}'. Please define some modules in the build scripts to use {0} as a master package.", ns);
            }
            var allTopLevelModuleTypesInPackage = allModuleTypesInPackage.Where(type => type.IsSealed);

            if (0 == allTopLevelModuleTypesInPackage.Count())
            {
                var message = new System.Text.StringBuilder();
                message.AppendFormat("No top-level modules found in the namespace '{0}'. Please mark some of the modules below as 'sealed' to identify them as top-level, and thus buildable when {0} is the master package:", ns);
                message.AppendLine();
                foreach (var moduleType in allModuleTypesInPackage)
                {
                    message.AppendFormat("\t{0}", moduleType.ToString());
                    message.AppendLine();
                }
                throw new Exception(message.ToString());
            }
            this.BuildEnvironment = env;
            foreach (var moduleType in allTopLevelModuleTypesInPackage)
            {
                var newModule = MakeModuleOfType(moduleType);
                if (newModule != null)
                {
                    this.TopLevelModules.Add(newModule);
                }
            }
            this.BuildEnvironment = null;
            if (0 == this.TopLevelModules.Count)
            {
                var message = new System.Text.StringBuilder();
                message.AppendFormat("Namespace '{0}' contains top-level modules, but none could be instantiated", ns);
                message.AppendLine();
                foreach (var moduleType in allModuleTypesInPackage)
                {
                    message.AppendFormat("\t{0}", moduleType.ToString());
                    message.AppendLine();
                }
                throw new Exception(message.ToString());
            }
            // remove all top level modules that have a reference count > 1
            foreach (var tlm in this.TopLevelModules.Reverse <Module>())
            {
                if (!tlm.TopLevel)
                {
                    this.TopLevelModules.Remove(tlm);
                }
            }
        }
Exemplo n.º 6
0
        AddDependentPackage()
        {
            var packageNameArgument = new Options.PackageName();
            var packageName         = CommandLineProcessor.Evaluate(packageNameArgument);

            if (null == packageName)
            {
                throw new Exception("No name was defined. Use {0} on the command line to specify it.", (packageNameArgument as ICommandLineArgument).LongName);
            }

            var packageVersion = CommandLineProcessor.Evaluate(new Options.PackageVersion());

            var masterPackage = GetMasterPackage();

            if (null != masterPackage.Dependents.FirstOrDefault(item => item.Item1 == packageName && item.Item2 == packageVersion))
            {
                if (null != packageVersion)
                {
                    throw new Exception("Package dependency {0}, version {1}, is already present", packageName, packageVersion);
                }
                else
                {
                    throw new Exception("Package dependency {0} is already present", packageName);
                }
            }

            var newDepTuple = new System.Tuple <string, string, bool?>(packageName, packageVersion, null);

            masterPackage.Dependents.Add(newDepTuple);
            // TODO: this is unfortunate having to write the file in order to use it with IdentifyAllPackages
            masterPackage.Write();

            // validate that the addition is ok
            try
            {
                PackageUtilities.IdentifyAllPackages();
            }
            catch (Exception exception)
            {
                masterPackage.Dependents.Remove(newDepTuple);
                masterPackage.Write();
                throw new Exception(exception, "Failed to add dependent. Are all necessary package repositories specified?");
            }
        }
Exemplo n.º 7
0
        MakePackage()
        {
            var packageDir = Graph.Instance.ProcessState.WorkingDirectory;
            var bamDir     = System.IO.Path.Combine(packageDir, BamSubFolder);

            if (System.IO.Directory.Exists(bamDir))
            {
                throw new Exception("Cannot create new package: A Bam package already exists at {0}", packageDir);
            }

            var packageNameArgument = new Options.PackageName();
            var packageName         = CommandLineProcessor.Evaluate(packageNameArgument);

            if (null == packageName)
            {
                throw new Exception("Cannot create new package: No name was defined. Use {0} on the command line to specify it.", (packageNameArgument as ICommandLineArgument).LongName);
            }

            var packageVersion = CommandLineProcessor.Evaluate(new Options.PackageVersion());
            var definition     = new PackageDefinition(bamDir, packageName, packageVersion);

            IOWrapper.CreateDirectory(bamDir);
            definition.Write();

            var scriptsDir = System.IO.Path.Combine(bamDir, ScriptsSubFolder);

            IOWrapper.CreateDirectory(scriptsDir);

            var initialScriptFile = System.IO.Path.Combine(scriptsDir, packageName) + ".cs";

            using (System.IO.TextWriter writer = new System.IO.StreamWriter(initialScriptFile))
            {
                writer.NewLine = "\n";
                writer.WriteLine("using Bam.Core;");
                writer.WriteLine("namespace {0}", packageName);
                writer.WriteLine("{");
                writer.WriteLine("    // write modules here ...");
                writer.WriteLine("}");
            }

            Log.Info("Package {0} was successfully created at {1}", definition.FullName, packageDir);
        }
Exemplo n.º 8
0
        CreateTopLevelModules(
            System.Reflection.Assembly assembly,
            Environment env,
            string ns)
        {
            var includeTests            = CommandLineProcessor.Evaluate(new Options.UseTests());
            var allTypes                = assembly.GetTypes();
            var allModuleTypesInPackage = allTypes.Where(type => ((type.Namespace == ns) || (includeTests && (type.Namespace == ns + ".tests"))) && type.IsSubclassOf(typeof(Module)));

            if (0 == allModuleTypesInPackage.Count())
            {
                throw new Exception("No modules found in the namespace '{0}'. Please define some modules in the build scripts to use {0} as a master package.", ns);
            }
            var allTopLevelModuleTypesInPackage = allModuleTypesInPackage.Where(type => type.IsSealed);

            if (0 == allTopLevelModuleTypesInPackage.Count())
            {
                var message = new System.Text.StringBuilder();
                message.AppendFormat("No top-level modules found in the namespace '{0}'. Please mark some of the modules below as 'sealed' to identify them as top-level, and thus buildable when {0} is the master package:", ns);
                message.AppendLine();
                foreach (var moduleType in allModuleTypesInPackage)
                {
                    message.AppendFormat("\t{0}", moduleType.ToString());
                    message.AppendLine();
                }
                throw new Exception(message.ToString());
            }
            try
            {
                this.CreateTopLevelModuleFromTypes(allTopLevelModuleTypesInPackage, env);
            }
            catch (Exception ex)
            {
                throw new Exception(ex, "An error occurred creating top-level modules in namespace '{0}':", ns);
            }
        }
Exemplo n.º 9
0
        CompilePackageAssembly(
            bool enforceBamAssemblyVersions = true,
            bool enableClean = true)
        {
            // validate build root
            if (null == Graph.Instance.BuildRoot)
            {
                throw new Exception("Build root has not been specified");
            }

            var gatherSourceProfile = new TimeProfile(ETimingProfiles.GatherSource);

            gatherSourceProfile.StartProfile();

            IdentifyAllPackages(enforceBamAssemblyVersions: enforceBamAssemblyVersions);

            var cleanFirst = CommandLineProcessor.Evaluate(new Options.CleanFirst());

            if (enableClean && cleanFirst && System.IO.Directory.Exists(Graph.Instance.BuildRoot))
            {
                Log.Info("Deleting build root '{0}'", Graph.Instance.BuildRoot);
                try
                {
                    // make sure no files are read-only, which may have happened as part of collation preserving file attributes
                    var dirInfo = new System.IO.DirectoryInfo(Graph.Instance.BuildRoot);
                    foreach (var file in dirInfo.EnumerateFiles("*", System.IO.SearchOption.AllDirectories))
                    {
                        file.Attributes &= ~System.IO.FileAttributes.ReadOnly;
                    }

                    System.IO.Directory.Delete(Graph.Instance.BuildRoot, true);
                }
                catch (System.IO.IOException ex)
                {
                    Log.Info("Failed to delete build root, because {0}. Continuing", ex.Message);
                }
            }

            BuildModeUtilities.ValidateBuildModePackage();

            var definitions = new StringArray();

            // gather source files
            var sourceCode   = new StringArray();
            int packageIndex = 0;

            foreach (var package in Graph.Instance.Packages)
            {
                Log.DebugMessage("{0}: '{1}' @ '{2}'", packageIndex, package.Version, (package.PackageRepositories.Count > 0) ? package.PackageRepositories[0] : "Not in a repository");

                // to compile with debug information, you must compile the files
                // to compile without, we need to file contents to hash the source
                if (Graph.Instance.CompileWithDebugSymbols)
                {
                    var scripts = package.GetScriptFiles();
                    sourceCode.AddRange(scripts);
                    Log.DebugMessage(scripts.ToString("\n\t"));
                }
                else
                {
                    foreach (var scriptFile in package.GetScriptFiles())
                    {
                        using (var reader = new System.IO.StreamReader(scriptFile))
                        {
                            sourceCode.Add(reader.ReadToEnd());
                        }
                        Log.DebugMessage("\t'{0}'", scriptFile);
                    }
                }

                foreach (var define in package.Definitions)
                {
                    if (!definitions.Contains(define))
                    {
                        definitions.Add(define);
                    }
                }

                ++packageIndex;
            }

            // add/remove other definitions
            definitions.Add(VersionDefineForCompiler);
            definitions.Add(HostPlatformDefineForCompiler);
            definitions.Sort();

            gatherSourceProfile.StopProfile();

            var assemblyCompileProfile = new TimeProfile(ETimingProfiles.AssemblyCompilation);

            assemblyCompileProfile.StartProfile();

            // assembly is written to the build root
            var cachedAssemblyPathname = System.IO.Path.Combine(Graph.Instance.BuildRoot, ".CachedPackageAssembly");

            cachedAssemblyPathname = System.IO.Path.Combine(cachedAssemblyPathname, Graph.Instance.MasterPackage.Name) + ".dll";
            var    hashPathName = System.IO.Path.ChangeExtension(cachedAssemblyPathname, "hash");
            string thisHashCode = null;

            var cacheAssembly = !CommandLineProcessor.Evaluate(new Options.DisableCacheAssembly());

            string compileReason = null;

            if (Graph.Instance.CompileWithDebugSymbols)
            {
                compileReason = "debug symbols were enabled";
            }
            else
            {
                // can an existing assembly be reused?
                thisHashCode = GetPackageHash(sourceCode, definitions, Graph.Instance.MasterPackage.BamAssemblies);
                if (cacheAssembly)
                {
                    if (System.IO.File.Exists(hashPathName))
                    {
                        using (var reader = new System.IO.StreamReader(hashPathName))
                        {
                            var diskHashCode = reader.ReadLine();
                            if (diskHashCode.Equals(thisHashCode))
                            {
                                Log.DebugMessage("Cached assembly used '{0}', with hash {1}", cachedAssemblyPathname, diskHashCode);
                                Log.Detail("Re-using existing package assembly");
                                Graph.Instance.ScriptAssemblyPathname = cachedAssemblyPathname;

                                assemblyCompileProfile.StopProfile();
                                return;
                            }
                            else
                            {
                                compileReason = "package source has changed since the last compile";
                            }
                        }
                    }
                    else
                    {
                        compileReason = "no previously compiled package assembly exists";
                    }
                }
                else
                {
                    compileReason = "user has disabled package assembly caching";
                }
            }

            // use the compiler in the current runtime version to build the assembly of packages
            var clrVersion      = System.Environment.Version;
            var compilerVersion = System.String.Format("v{0}.{1}", clrVersion.Major, clrVersion.Minor);

            Log.Detail("Compiling package assembly (C# compiler {0}{1}), because {2}.",
                       compilerVersion,
                       Graph.Instance.ProcessState.TargetFrameworkVersion != null ? (", targetting " + Graph.Instance.ProcessState.TargetFrameworkVersion) : string.Empty,
                       compileReason);

            var providerOptions = new System.Collections.Generic.Dictionary <string, string>();

            providerOptions.Add("CompilerVersion", compilerVersion);

            if (Graph.Instance.ProcessState.RunningMono)
            {
                Log.DebugMessage("Compiling assembly for Mono");
            }

            using (var provider = new Microsoft.CSharp.CSharpCodeProvider(providerOptions))
            {
                var compilerParameters = new System.CodeDom.Compiler.CompilerParameters();
                compilerParameters.TreatWarningsAsErrors = true;
                compilerParameters.WarningLevel          = 4;
                compilerParameters.GenerateExecutable    = false;
                compilerParameters.GenerateInMemory      = false;

                if (Graph.Instance.CompileWithDebugSymbols)
                {
                    compilerParameters.OutputAssembly = System.IO.Path.Combine(System.IO.Path.GetTempPath(), Graph.Instance.MasterPackage.Name) + ".dll";
                }
                else
                {
                    compilerParameters.OutputAssembly = cachedAssemblyPathname;
                }

                var compilerOptions = "/checked+ /unsafe-";
                if (Graph.Instance.CompileWithDebugSymbols)
                {
                    compilerParameters.IncludeDebugInformation = true;
                    compilerOptions += " /optimize-";
                }
                else
                {
                    compilerOptions += " /optimize+";
                }
                compilerOptions += " /platform:anycpu";

                // define strings
                compilerOptions += " /define:" + definitions.ToString(';');

                compilerParameters.CompilerOptions = compilerOptions;

                if (provider.Supports(System.CodeDom.Compiler.GeneratorSupport.Resources))
                {
                    // Bam assembly
                    // TODO: Q: why is it only for the master package? Why not all of them, which may have additional dependencies?
                    foreach (var assembly in Graph.Instance.MasterPackage.BamAssemblies)
                    {
                        var assemblyFileName = System.String.Format("{0}.dll", assembly.Name);
                        var assemblyPathName = System.IO.Path.Combine(Graph.Instance.ProcessState.ExecutableDirectory, assemblyFileName);
                        compilerParameters.ReferencedAssemblies.Add(assemblyPathName);
                    }

                    // DotNet assembly
                    foreach (var desc in Graph.Instance.MasterPackage.DotNetAssemblies)
                    {
                        var assemblyFileName = System.String.Format("{0}.dll", desc.Name);
                        compilerParameters.ReferencedAssemblies.Add(assemblyFileName);
                    }

                    if (Graph.Instance.ProcessState.RunningMono)
                    {
                        compilerParameters.ReferencedAssemblies.Add("Mono.Posix.dll");
                    }
                }
                else
                {
                    throw new Exception("C# compiler does not support Resources");
                }

                // this will create the build root directory as necessary
                IOWrapper.CreateDirectory(System.IO.Path.GetDirectoryName(compilerParameters.OutputAssembly));

                var results = Graph.Instance.CompileWithDebugSymbols ?
                              provider.CompileAssemblyFromFile(compilerParameters, sourceCode.ToArray()) :
                              provider.CompileAssemblyFromSource(compilerParameters, sourceCode.ToArray());

                if (results.Errors.HasErrors || results.Errors.HasWarnings)
                {
                    var message = new System.Text.StringBuilder();
                    message.AppendFormat("Failed to compile package '{0}'. There are {1} errors.", Graph.Instance.MasterPackage.FullName, results.Errors.Count);
                    message.AppendLine();
                    foreach (System.CodeDom.Compiler.CompilerError error in results.Errors)
                    {
                        message.AppendFormat("\t{0}({1}): {2} {3}", error.FileName, error.Line, error.ErrorNumber, error.ErrorText);
                        message.AppendLine();
                    }
                    if (!Graph.Instance.CompileWithDebugSymbols)
                    {
                        message.AppendLine();
                        ICommandLineArgument debugOption = new Options.UseDebugSymbols();
                        message.AppendFormat("Use the {0}/{1} command line option with bam for more accurate error messages.", debugOption.LongName, debugOption.ShortName);
                        message.AppendLine();
                    }
                    message.AppendLine();
                    ICommandLineArgument createDebugProjectOption = new Options.CreateDebugProject();
                    message.AppendFormat("Use the {0}/{1} command line option with bam to create an editable IDE project containing the build scripts.", createDebugProjectOption.LongName, createDebugProjectOption.ShortName);
                    message.AppendLine();
                    throw new Exception(message.ToString());
                }

                if (!Graph.Instance.CompileWithDebugSymbols)
                {
                    if (cacheAssembly)
                    {
                        using (var writer = new System.IO.StreamWriter(hashPathName))
                        {
                            writer.WriteLine(thisHashCode);
                        }
                    }
                    else
                    {
                        // will not throw if the file doesn't exist
                        System.IO.File.Delete(hashPathName);
                    }
                }

                Log.DebugMessage("Written assembly to '{0}'", compilerParameters.OutputAssembly);
                Graph.Instance.ScriptAssemblyPathname = compilerParameters.OutputAssembly;
            }

            assemblyCompileProfile.StopProfile();
        }
Exemplo n.º 10
0
        IdentifyAllPackages(
            bool allowDuplicates            = false,
            bool enforceBamAssemblyVersions = true)
        {
            var packageRepos = new System.Collections.Generic.LinkedList <System.Tuple <string, PackageDefinition> >();

            foreach (var repo in Graph.Instance.PackageRepositories)
            {
                EnqueuePackageRepositoryToVisit(packageRepos, repo, null);
            }

            var masterDefinitionFile = GetMasterPackage();

            foreach (var repo in masterDefinitionFile.PackageRepositories)
            {
                EnqueuePackageRepositoryToVisit(packageRepos, repo, masterDefinitionFile);
            }

            // read the definition files of any package found in the package roots
            var candidatePackageDefinitions = new Array <PackageDefinition>();

            candidatePackageDefinitions.Add(masterDefinitionFile);
            while (packageRepos.Count > 0)
            {
                var repoTuple = packageRepos.First();
                packageRepos.RemoveFirst();
                var repo = repoTuple.Item1;
                if (!System.IO.Directory.Exists(repo))
                {
                    var message = new System.Text.StringBuilder();
                    message.AppendFormat("Package repository directory {0} does not exist.", repo);
                    message.AppendLine();
                    message.AppendFormat("Repository requested from {0}", repoTuple.Item2.XMLFilename);
                    message.AppendLine();
                    throw new Exception(message.ToString());
                }
                var candidatePackageDirs = System.IO.Directory.GetDirectories(repo, BamSubFolder, System.IO.SearchOption.AllDirectories);

                Graph.Instance.PackageRepositories.Add(repo);

                foreach (var bamDir in candidatePackageDirs)
                {
                    var packageDir            = System.IO.Path.GetDirectoryName(bamDir);
                    var packageDefinitionPath = GetPackageDefinitionPathname(packageDir);

                    // ignore any duplicates (can be found due to nested repositories)
                    if (null != candidatePackageDefinitions.FirstOrDefault(item => item.XMLFilename == packageDefinitionPath))
                    {
                        continue;
                    }

                    var definitionFile = new PackageDefinition(packageDefinitionPath);
                    definitionFile.Read();
                    candidatePackageDefinitions.Add(definitionFile);

                    foreach (var newRepo in definitionFile.PackageRepositories)
                    {
                        EnqueuePackageRepositoryToVisit(packageRepos, newRepo, definitionFile);
                    }
                }
            }

            // defaults come from
            // - the master definition file
            // - command line args (these trump the mdf)
            // and only requires resolving when referenced
            var packageDefinitions = new Array <PackageDefinition>();

            PackageDefinition.ResolveDependencies(masterDefinitionFile, packageDefinitions, candidatePackageDefinitions);

            // now resolve any duplicate names using defaults
            // unless duplicates are allowed
            var duplicatePackageNames    = packageDefinitions.GroupBy(item => item.Name).Where(item => item.Count() > 1).Select(item => item.Key);
            var uniquePackageNames       = packageDefinitions.GroupBy(item => item.Name).Where(item => item.Count() == 1).Select(item => item.Key);
            var versionSpeciferArgs      = new Options.PackageDefaultVersion();
            var packageVersionSpecifiers = CommandLineProcessor.Evaluate(versionSpeciferArgs);

            if ((duplicatePackageNames.Count() > 0) && !allowDuplicates)
            {
                var toRemove = new Array <PackageDefinition>();

                foreach (var dupName in duplicatePackageNames)
                {
                    var duplicates        = packageDefinitions.Where(item => item.Name == dupName);
                    var resolvedDuplicate = TryToResolveDuplicate(masterDefinitionFile, dupName, duplicates, packageDefinitions, packageVersionSpecifiers, toRemove);
                    if (null != resolvedDuplicate)
                    {
                        continue;
                    }

                    // try removing any packages that have already been resolved
                    // which can remove additional packages (recursive check) because they had been added solely by those we are just about to remove
                    packageDefinitions.RemoveAll(PackagesToRemove(toRemove, packageDefinitions, masterDefinitionFile));
                    packageDefinitions.RemoveAll(toRemove);

                    // and if that has reduced the duplicates for this package down to a single version, we're good to carry on
                    var numDuplicates = duplicates.Count();
                    if (1 == numDuplicates)
                    {
                        toRemove.Clear();
                        continue;
                    }

                    // otherwise, error
                    var resolveErrorMessage = new System.Text.StringBuilder();
                    if (numDuplicates > 0)
                    {
                        resolveErrorMessage.AppendFormat("Unable to resolve to a single version of package {0}. Use --{0}.version=<version> to resolve.", dupName);
                        resolveErrorMessage.AppendLine();
                        resolveErrorMessage.AppendLine("Available versions of the package are:");
                        foreach (var dup in duplicates)
                        {
                            resolveErrorMessage.AppendFormat("\t{0}", dup.Version);
                            resolveErrorMessage.AppendLine();
                        }
                    }
                    else
                    {
                        resolveErrorMessage.AppendFormat("No version of package {0} has been determined to be available.", dupName);
                        resolveErrorMessage.AppendLine();
                        if (toRemove.Count() > 0)
                        {
                            resolveErrorMessage.AppendFormat("If there were any references to {0}, they may have been removed from consideration by the following packages being discarded:", dupName);
                            resolveErrorMessage.AppendLine();
                            foreach (var removed in toRemove)
                            {
                                resolveErrorMessage.AppendFormat("\t{0}", removed.FullName);
                                resolveErrorMessage.AppendLine();
                            }
                        }
                        resolveErrorMessage.AppendFormat("Please add an explicit dependency to (a version of) the {0} package either in your master package or one of its dependencies.", dupName);
                        resolveErrorMessage.AppendLine();
                    }
                    throw new Exception(resolveErrorMessage.ToString());
                }

                // finally, clean up the package definition list to use, with all those that need to be deleted
                packageDefinitions.RemoveAll(toRemove);
            }

            // ensure that all packages with a single version in the definition files, does not have a command line override
            // that refers to a completely different version
            foreach (var uniquePkgName in uniquePackageNames)
            {
                foreach (var versionSpecifier in packageVersionSpecifiers)
                {
                    if (!versionSpecifier.Contains(uniquePkgName))
                    {
                        continue;
                    }

                    var versionFromDefinition = packageDefinitions.First(item => item.Name == uniquePkgName).Version;
                    if (versionSpecifier[1] != versionFromDefinition)
                    {
                        var noMatchMessage = new System.Text.StringBuilder();
                        noMatchMessage.AppendFormat("Command line version specified, {0}, could not resolve to one of the available versions of package {1}:", versionSpecifier[1], uniquePkgName);
                        noMatchMessage.AppendLine();
                        noMatchMessage.AppendFormat("\t{0}", versionFromDefinition);
                        noMatchMessage.AppendLine();
                        throw new Exception(noMatchMessage.ToString());
                    }
                }
            }

            if (enforceBamAssemblyVersions)
            {
                // for all packages that make up this assembly, ensure that their requirements on the version of the Bam
                // assemblies are upheld, prior to compiling the code
                foreach (var pkgDefn in packageDefinitions)
                {
                    pkgDefn.ValidateBamAssemblyRequirements();
                }
            }

            Graph.Instance.SetPackageDefinitions(packageDefinitions);
        }
Exemplo n.º 11
0
        Execute(
            Array <Environment> environments,
            System.Reflection.Assembly packageAssembly = null)
        {
            PrintVersion();

            if (0 == environments.Count)
            {
                throw new Exception("No build configurations were specified");
            }

            var graph = Graph.Instance;

            if (null != packageAssembly)
            {
                PackageUtilities.IdentifyAllPackages();
                graph.ScriptAssembly         = packageAssembly;
                graph.ScriptAssemblyPathname = packageAssembly.Location;
            }
            else
            {
                PackageUtilities.CompilePackageAssembly();
                PackageUtilities.LoadPackageAssembly();
            }

            var packageMetaDataProfile = new TimeProfile(ETimingProfiles.PackageMetaData);

            packageMetaDataProfile.StartProfile();

            // validate that there is at most one local policy
            // if test mode is enabled, then the '.tests' sub-namespaces are also checked
            {
                var localPolicies = graph.ScriptAssembly.GetTypes().Where(t => typeof(ISitePolicy).IsAssignableFrom(t));
                var includeTests  = CommandLineProcessor.Evaluate(new Options.UseTests());
                if (!includeTests)
                {
                    localPolicies = localPolicies.Where(item => !item.Namespace.EndsWith(".tests"));
                }
                var numLocalPolicies = localPolicies.Count();
                if (numLocalPolicies > 0)
                {
                    if (numLocalPolicies > 1)
                    {
                        var message = new System.Text.StringBuilder();
                        message.AppendLine("Too many site policies exist in the package assembly:");
                        foreach (var policy in localPolicies)
                        {
                            message.AppendFormat("\t{0}", policy.ToString());
                            message.AppendLine();
                        }
                        throw new Exception(message.ToString());
                    }

                    Settings.LocalPolicy = System.Activator.CreateInstance(localPolicies.First()) as ISitePolicy;
                }
            }

            // find a product definition
            {
                var productDefinitions    = graph.ScriptAssembly.GetTypes().Where(t => typeof(IProductDefinition).IsAssignableFrom(t));
                var numProductDefinitions = productDefinitions.Count();
                if (numProductDefinitions > 0)
                {
                    if (numProductDefinitions > 1)
                    {
                        var message = new System.Text.StringBuilder();
                        message.AppendLine("Too many product definitions exist in the package assembly:");
                        foreach (var def in productDefinitions)
                        {
                            message.AppendFormat("\t{0}", def.ToString());
                            message.AppendLine();
                        }
                        throw new Exception(message.ToString());
                    }

                    graph.ProductDefinition = System.Activator.CreateInstance(productDefinitions.First()) as IProductDefinition;
                }
            }

            // get the metadata from the build mode package
            var metaName     = System.String.Format("{0}Builder.{0}Meta", graph.Mode);
            var metaDataType = graph.ScriptAssembly.GetType(metaName);

            if (null == metaDataType)
            {
                throw new Exception("No build mode {0} meta data type {1}", graph.Mode, metaName);
            }

            if (!typeof(IBuildModeMetaData).IsAssignableFrom(metaDataType))
            {
                throw new Exception("Build mode package meta data type {0} does not implement the interface {1}", metaDataType.ToString(), typeof(IBuildModeMetaData).ToString());
            }
            graph.BuildModeMetaData = System.Activator.CreateInstance(metaDataType) as IBuildModeMetaData;

            // packages can have meta data - instantiate where they exist
            foreach (var package in graph.Packages)
            {
                var ns       = package.Name;
                var metaType = graph.ScriptAssembly.GetTypes().FirstOrDefault(item => item.Namespace == ns && typeof(PackageMetaData).IsAssignableFrom(item));
                if (null == metaType)
                {
                    continue;
                }

                try
                {
                    package.MetaData = System.Activator.CreateInstance(metaType) as PackageMetaData;
                }
                catch (Exception exception)
                {
                    throw exception;
                }
                catch (System.Reflection.TargetInvocationException exception)
                {
                    throw new Exception(exception, "Failed to create package metadata");
                }
            }

            packageMetaDataProfile.StopProfile();

            var topLevelNamespace = graph.MasterPackage.Name;

            var findBuildableModulesProfile = new TimeProfile(ETimingProfiles.IdentifyBuildableModules);

            findBuildableModulesProfile.StartProfile();

            // Phase 1: Instantiate all modules in the namespace of the package in which the tool was invoked
            Log.Detail("Creating modules");
            foreach (var env in environments)
            {
                graph.CreateTopLevelModules(graph.ScriptAssembly, env, topLevelNamespace);
            }

            findBuildableModulesProfile.StopProfile();

            var populateGraphProfile = new TimeProfile(ETimingProfiles.PopulateGraph);

            populateGraphProfile.StartProfile();
            // Phase 2: Graph now has a linear list of modules; create a dependency graph
            // NB: all those modules with 0 dependees are the top-level modules
            // NB: default settings have already been defined here
            // not only does this generate the dependency graph, but also creates the default settings for each module, and completes them
            graph.SortDependencies();
            populateGraphProfile.StopProfile();

            // TODO: make validation optional, if it starts showing on profiles
            var validateGraphProfile = new TimeProfile(ETimingProfiles.ValidateGraph);

            validateGraphProfile.StartProfile();
            graph.Validate();
            validateGraphProfile.StopProfile();

            // Phase 3: (Create default settings, and ) apply patches (build + shared) to each module
            // NB: some builders can use the patch directly for child objects, so this may be dependent upon the builder
            // Toolchains for modules need to be set here, as they might append macros into each module in order to evaluate paths
            // TODO: a parallel thread can be spawned here, that can check whether command lines have changed
            // the Settings object can be inspected, and a hash generated. This hash can be written to disk, and compared.
            // If a 'verbose' mode is enabled, then more work can be done to figure out what has changed. This would also require
            // serializing the binary Settings object
            var createPatchesProfile = new TimeProfile(ETimingProfiles.CreatePatches);

            createPatchesProfile.StartProfile();
            graph.ApplySettingsPatches();
            createPatchesProfile.StopProfile();

            // expand paths after patching settings, because some of the patches may contain tokenized strings
            // TODO: a thread can be spawned, to check for whether files were in date or not, which will
            // be ready in time for graph execution
            var parseStringsProfile = new TimeProfile(ETimingProfiles.ParseTokenizedStrings);

            parseStringsProfile.StartProfile();
            TokenizedString.ParseAll();
            parseStringsProfile.StopProfile();

            if (CommandLineProcessor.Evaluate(new Options.ViewDependencyGraph()))
            {
                // must come after all strings are parsed, in order to display useful paths
                graph.Dump();
            }

            // Phase 4: Execute dependency graph
            // N.B. all paths (including those with macros) have been delayed expansion until now
            var graphExecutionProfile = new TimeProfile(ETimingProfiles.GraphExecution);

            graphExecutionProfile.StartProfile();
            var executor = new Executor();

            executor.Run();
            graphExecutionProfile.StopProfile();
        }
Exemplo n.º 12
0
        Run()
        {
            Log.Detail("Running build...");

            // TODO: should the rank collections be sorted, so that modules with fewest dependencies are first?

            var graph           = Graph.Instance;
            var metaDataType    = graph.BuildModeMetaData.GetType();
            var useEvaluation   = CheckIfModulesNeedRebuilding(metaDataType);
            var explainRebuild  = CommandLineProcessor.Evaluate(new Options.ExplainBuildReason());
            var immediateOutput = CommandLineProcessor.Evaluate(new Options.ImmediateOutput());

            ExecutePreBuild(metaDataType);

            // necessary if built with debug symbols
            IOWrapper.CreateDirectoryIfNotExists(graph.BuildRoot);

            var threadCount = CommandLineProcessor.Evaluate(new Options.MultiThreaded());

            if (0 == threadCount)
            {
                threadCount = System.Environment.ProcessorCount;
            }

            System.Exception abortException = null;
            if (threadCount > 1)
            {
                using (var cancellationSource = new System.Threading.CancellationTokenSource())
                {
                    var cancellationToken = cancellationSource.Token;

                    // LongRunning is absolutely necessary in order to achieve paralleism
                    var creationOpts     = System.Threading.Tasks.TaskCreationOptions.LongRunning;
                    var continuationOpts = System.Threading.Tasks.TaskContinuationOptions.LongRunning;

                    var scheduler = new LimitedConcurrencyLevelTaskScheduler(threadCount);

                    var factory = new System.Threading.Tasks.TaskFactory(
                        cancellationToken,
                        creationOpts,
                        continuationOpts,
                        scheduler);

                    var tasks = new Array <System.Threading.Tasks.Task>();
                    foreach (var rank in graph.Reverse())
                    {
                        foreach (var module in rank)
                        {
                            var context = new ExecutionContext(useEvaluation, explainRebuild, immediateOutput);
                            var task    = factory.StartNew(() =>
                            {
                                if (cancellationToken.IsCancellationRequested)
                                {
                                    return;
                                }
                                var depTasks = new Array <System.Threading.Tasks.Task>();
                                foreach (var dep in module.Dependents)
                                {
                                    if (null == dep.ExecutionTask)
                                    {
                                        continue;
                                    }
                                    depTasks.Add(dep.ExecutionTask);
                                }
                                foreach (var dep in module.Requirements)
                                {
                                    if (null == dep.ExecutionTask)
                                    {
                                        continue;
                                    }
                                    depTasks.Add(dep.ExecutionTask);
                                }
                                System.Threading.Tasks.Task.WaitAll(depTasks.ToArray());
                                if (cancellationToken.IsCancellationRequested)
                                {
                                    return;
                                }

                                try
                                {
                                    (module as IModuleExecution).Execute(context);
                                }
                                catch (Exception ex)
                                {
                                    abortException = ex;
                                    cancellationSource.Cancel();
                                }
                                finally
                                {
                                    if (context.OutputStringBuilder != null && context.OutputStringBuilder.Length > 0)
                                    {
                                        Log.Info(context.OutputStringBuilder.ToString());
                                    }
                                    if (context.ErrorStringBuilder != null && context.ErrorStringBuilder.Length > 0)
                                    {
                                        Log.Info(context.ErrorStringBuilder.ToString());
                                    }
                                }
                            });
                            tasks.Add(task);
                            module.ExecutionTask = task;
                        }
                    }
                    try
                    {
                        System.Threading.Tasks.Task.WaitAll(tasks.ToArray());
                    }
                    catch (System.AggregateException exception)
                    {
                        if (!(exception.InnerException is System.Threading.Tasks.TaskCanceledException))
                        {
                            throw new Exception(exception, "Error during threaded build");
                        }
                    }
                }
            }
            else
            {
                foreach (var rank in graph.Reverse())
                {
                    if (null != abortException)
                    {
                        break;
                    }
                    foreach (IModuleExecution module in rank)
                    {
                        var context = new ExecutionContext(useEvaluation, explainRebuild, immediateOutput);
                        try
                        {
                            module.Execute(context);
                        }
                        catch (Exception ex)
                        {
                            abortException = ex;
                            break;
                        }
                        finally
                        {
                            if (context.OutputStringBuilder != null && context.OutputStringBuilder.Length > 0)
                            {
                                Log.Info(context.OutputStringBuilder.ToString());
                            }
                            if (context.ErrorStringBuilder != null && context.ErrorStringBuilder.Length > 0)
                            {
                                Log.Info(context.ErrorStringBuilder.ToString());
                            }
                        }
                    }
                }
            }

            if (null != abortException)
            {
                throw new Exception(abortException, "Error during {0}threaded build", (threadCount > 1) ? string.Empty : "non-");
            }

            ExecutePostBuild(metaDataType);
        }
Exemplo n.º 13
0
        IdentifyAllPackages(
            bool allowDuplicates            = false,
            bool enforceBamAssemblyVersions = true)
        {
            var packageRepos = new System.Collections.Generic.Queue <string>();

            foreach (var repo in Graph.Instance.PackageRepositories)
            {
                if (packageRepos.Contains(repo))
                {
                    continue;
                }
                packageRepos.Enqueue(repo);
            }

            var masterDefinitionFile = GetMasterPackage(enforceBamAssemblyVersions: enforceBamAssemblyVersions);

            foreach (var repo in masterDefinitionFile.PackageRepositories)
            {
                if (packageRepos.Contains(repo))
                {
                    continue;
                }
                packageRepos.Enqueue(repo);
            }

            // read the definition files of any package found in the package roots
            var candidatePackageDefinitions = new Array <PackageDefinition>();

            candidatePackageDefinitions.Add(masterDefinitionFile);
            while (packageRepos.Count > 0)
            {
                var repo = packageRepos.Dequeue();
                if (!System.IO.Directory.Exists(repo))
                {
                    throw new Exception("Package repository directory {0} does not exist", repo);
                }
                var candidatePackageDirs = System.IO.Directory.GetDirectories(repo, BamSubFolder, System.IO.SearchOption.AllDirectories);

                Graph.Instance.PackageRepositories.Add(repo);

                foreach (var bamDir in candidatePackageDirs)
                {
                    var packageDir            = System.IO.Path.GetDirectoryName(bamDir);
                    var packageDefinitionPath = GetPackageDefinitionPathname(packageDir);

                    // ignore any duplicates (can be found due to nested repositories)
                    if (null != candidatePackageDefinitions.Where(item => item.XMLFilename == packageDefinitionPath).FirstOrDefault())
                    {
                        continue;
                    }

                    var definitionFile = new PackageDefinition(packageDefinitionPath, !Graph.Instance.ForceDefinitionFileUpdate);
                    definitionFile.Read(true, enforceBamAssemblyVersions);
                    candidatePackageDefinitions.Add(definitionFile);

                    foreach (var newRepo in definitionFile.PackageRepositories)
                    {
                        if (Graph.Instance.PackageRepositories.Contains(newRepo))
                        {
                            continue;
                        }
                        packageRepos.Enqueue(newRepo);
                    }
                }
            }

            // defaults come from
            // - the master definition file
            // - command line args (these trump the mdf)
            // and only requires resolving when referenced
            var packageDefinitions = new Array <PackageDefinition>();

            PackageDefinition.ResolveDependencies(masterDefinitionFile, packageDefinitions, candidatePackageDefinitions);

            // now resolve any duplicate names using defaults
            // unless duplicates are allowed
            var duplicatePackageNames = packageDefinitions.GroupBy(item => item.Name).Where(item => item.Count() > 1).Select(item => item.Key);

            if ((duplicatePackageNames.Count() > 0) && !allowDuplicates)
            {
                var versionSpeciferArgs      = new Options.PackageDefaultVersion();
                var packageVersionSpecifiers = CommandLineProcessor.Evaluate(versionSpeciferArgs);
                var toRemove = new Array <PackageDefinition>();

                foreach (var dupName in duplicatePackageNames)
                {
                    var duplicates = packageDefinitions.Where(item => item.Name == dupName);
                    PackageDefinition resolvedDuplicate = null;
                    // command line specifications take precedence to resolve a duplicate
                    foreach (var specifier in packageVersionSpecifiers)
                    {
                        if (!specifier.Contains(dupName))
                        {
                            continue;
                        }

                        foreach (var dupPackage in duplicates)
                        {
                            if (specifier[1] == dupPackage.Version)
                            {
                                resolvedDuplicate = dupPackage;
                                break;
                            }
                        }

                        if (resolvedDuplicate != null)
                        {
                            break;
                        }

                        var noMatchMessage = new System.Text.StringBuilder();
                        noMatchMessage.AppendFormat("Command line version specified, {0}, could not resolve to one of the available versions of package {1}:", specifier[1], duplicates.First().Name);
                        noMatchMessage.AppendLine();
                        foreach (var dup in duplicates)
                        {
                            noMatchMessage.AppendFormat("\t{0}", dup.Version);
                            noMatchMessage.AppendLine();
                        }
                        throw new Exception(noMatchMessage.ToString());
                    }

                    if (resolvedDuplicate != null)
                    {
                        toRemove.AddRange(packageDefinitions.Where(item => (item.Name == dupName) && (item != resolvedDuplicate)));
                        continue;
                    }

                    // now look at the master dependency file, for any 'default' specifications
                    var masterDependency = masterDefinitionFile.Dependents.Where(item => item.Item1 == dupName && item.Item3.HasValue && item.Item3.Value).FirstOrDefault();
                    if (null != masterDependency)
                    {
                        toRemove.AddRange(packageDefinitions.Where(item => (item.Name == dupName) && (item.Version != masterDependency.Item2)));
                        continue;
                    }

                    var resolveErrorMessage = new System.Text.StringBuilder();
                    resolveErrorMessage.AppendFormat("Unable to resolve to a single version of package {0}. Use --{0}.version=<version> to resolve. Available versions of the package are:", duplicates.First().Name);
                    resolveErrorMessage.AppendLine();
                    foreach (var dup in duplicates)
                    {
                        resolveErrorMessage.AppendFormat("\t{0}", dup.Version);
                        resolveErrorMessage.AppendLine();
                    }
                    throw new Exception(resolveErrorMessage.ToString());
                }

                packageDefinitions.RemoveAll(toRemove);
            }

            Graph.Instance.SetPackageDefinitions(packageDefinitions);
        }
Exemplo n.º 14
0
        IdentifyAllPackages(
            bool allowDuplicates            = false,
            bool enforceBamAssemblyVersions = true)
        {
            var packageRepos = new System.Collections.Generic.LinkedList <System.Tuple <string, PackageDefinition> >();
            int reposHWM     = 0;

            foreach (var repo in Graph.Instance.PackageRepositories)
            {
                EnqueuePackageRepositoryToVisit(packageRepos, ref reposHWM, repo, null);
            }

            var masterDefinitionFile = GetMasterPackage();
            // inject any packages from the command line into the master definition file
            // and these will be defaults
            var injectPackages = CommandLineProcessor.Evaluate(new Options.InjectDefaultPackage());

            if (null != injectPackages)
            {
                foreach (var injected in injectPackages)
                {
                    var    name    = injected[0];
                    string version = null;
                    if (injected.Count > 1)
                    {
                        version = injected[1].TrimStart(new [] { '-' }); // see regex in InjectDefaultPackage
                    }
                    var is_default = true;
                    var dependent  = new System.Tuple <string, string, bool?>(name, version, is_default);
                    masterDefinitionFile.Dependents.AddUnique(dependent);
                }
            }
            foreach (var repo in masterDefinitionFile.PackageRepositories)
            {
                EnqueuePackageRepositoryToVisit(packageRepos, ref reposHWM, repo, masterDefinitionFile);
            }

            // read the definition files of any package found in the package roots
            var candidatePackageDefinitions = new Array <PackageDefinition>();

            candidatePackageDefinitions.Add(masterDefinitionFile);
            var packageReposVisited = 0;

            Log.Detail("Querying package repositories...");
            while (packageRepos.Count > 0)
            {
                var repoTuple = packageRepos.First();
                packageRepos.RemoveFirst();
                var repo = repoTuple.Item1;
                if (!System.IO.Directory.Exists(repo))
                {
                    var message = new System.Text.StringBuilder();
                    message.AppendFormat("Package repository directory {0} does not exist.", repo);
                    message.AppendLine();
                    message.AppendFormat("Repository requested from {0}", repoTuple.Item2.XMLFilename);
                    message.AppendLine();
                    throw new Exception(message.ToString());
                }

                // faster than System.IO.Directory.GetDirectories(repo, BamSubFolder, System.IO.SearchOption.AllDirectories);
                // when there are deep directories
                StringArray candidatePackageDirs = new StringArray();
                var         possiblePackages     = System.IO.Directory.GetDirectories(repo, "*", System.IO.SearchOption.TopDirectoryOnly);
                foreach (var packageDir in possiblePackages)
                {
                    var possibleBamFolder = System.IO.Path.Combine(packageDir, BamSubFolder);
                    if (System.IO.Directory.Exists(possibleBamFolder))
                    {
                        candidatePackageDirs.Add(packageDir);
                    }
                }

                Graph.Instance.PackageRepositories.Add(repo);

                foreach (var packageDir in candidatePackageDirs)
                {
                    var packageDefinitionPath = GetPackageDefinitionPathname(packageDir);

                    // ignore any duplicates (can be found due to nested repositories)
                    if (null != candidatePackageDefinitions.FirstOrDefault(item => item.XMLFilename == packageDefinitionPath))
                    {
                        continue;
                    }

                    var definitionFile = new PackageDefinition(packageDefinitionPath);
                    definitionFile.Read();
                    candidatePackageDefinitions.Add(definitionFile);

                    foreach (var newRepo in definitionFile.PackageRepositories)
                    {
                        EnqueuePackageRepositoryToVisit(packageRepos, ref reposHWM, newRepo, definitionFile);
                    }
                }

                ++packageReposVisited;
                Log.DetailProgress("{0,3}%", (int)(100 * ((float)packageReposVisited / reposHWM)));
            }
#if DEBUG
            if (packageReposVisited != reposHWM)
            {
                throw new Exception("Inconsistent package repository count: {0} added, {1} visited", reposHWM, packageReposVisited);
            }
#endif

            // defaults come from
            // - the master definition file
            // - command line args (these trump the mdf)
            // and only requires resolving when referenced
            var packageDefinitions = new Array <PackageDefinition>();
            PackageDefinition.ResolveDependencies(masterDefinitionFile, packageDefinitions, candidatePackageDefinitions);

            // now resolve any duplicate names using defaults
            // unless duplicates are allowed
            var duplicatePackageNames    = packageDefinitions.GroupBy(item => item.Name).Where(item => item.Count() > 1).Select(item => item.Key);
            var uniquePackageNames       = packageDefinitions.GroupBy(item => item.Name).Where(item => item.Count() == 1).Select(item => item.Key);
            var versionSpeciferArgs      = new Options.PackageDefaultVersion();
            var packageVersionSpecifiers = CommandLineProcessor.Evaluate(versionSpeciferArgs);
            if ((duplicatePackageNames.Count() > 0) && !allowDuplicates)
            {
                foreach (var dupName in duplicatePackageNames)
                {
                    Log.DebugMessage("Duplicate '{0}'; total packages {1}", dupName, packageDefinitions.Count);
                    var duplicates        = packageDefinitions.Where(item => item.Name == dupName);
                    var toRemove          = new Array <PackageDefinition>();
                    var resolvedDuplicate = TryToResolveDuplicate(masterDefinitionFile, dupName, duplicates, packageDefinitions, packageVersionSpecifiers, toRemove);

                    Log.DebugMessage("Attempting to remove:\n\t{0}", toRemove.ToString("\n\t"));
                    // try removing any packages that have already been resolved
                    // which, in turn, can remove additional packages that have become orphaned by other removals
                    packageDefinitions.RemoveAll(FindPackagesToRemove(toRemove, packageDefinitions, masterDefinitionFile));

                    if (null != resolvedDuplicate)
                    {
                        continue;
                    }

                    // and if that has reduced the duplicates for this package down to a single version, we're good to carry on
                    var numDuplicates = duplicates.Count(); // this is LINQ, so it's 'live'
                    if (1 == numDuplicates)
                    {
                        continue;
                    }

                    // otherwise, error
                    var resolveErrorMessage = new System.Text.StringBuilder();
                    if (numDuplicates > 0)
                    {
                        resolveErrorMessage.AppendFormat("Unable to resolve to a single version of package {0}. Use --{0}.version=<version> to resolve.", dupName);
                        resolveErrorMessage.AppendLine();
                        resolveErrorMessage.AppendLine("Available versions of the package are:");
                        foreach (var dup in duplicates)
                        {
                            resolveErrorMessage.AppendFormat("\t{0}", dup.Version);
                            resolveErrorMessage.AppendLine();
                        }
                    }
                    else
                    {
                        resolveErrorMessage.AppendFormat("No version of package {0} has been determined to be available.", dupName);
                        resolveErrorMessage.AppendLine();
                        if (toRemove.Count() > 0)
                        {
                            resolveErrorMessage.AppendFormat("If there were any references to {0}, they may have been removed from consideration by the following packages being discarded:", dupName);
                            resolveErrorMessage.AppendLine();
                            foreach (var removed in toRemove)
                            {
                                resolveErrorMessage.AppendFormat("\t{0}", removed.FullName);
                                resolveErrorMessage.AppendLine();
                            }
                        }
                        resolveErrorMessage.AppendFormat("Please add an explicit dependency to (a version of) the {0} package either in your master package or one of its dependencies.", dupName);
                        resolveErrorMessage.AppendLine();
                    }
                    throw new Exception(resolveErrorMessage.ToString());
                }
            }

            // ensure that all packages with a single version in the definition files, does not have a command line override
            // that refers to a completely different version
            foreach (var uniquePkgName in uniquePackageNames)
            {
                foreach (var versionSpecifier in packageVersionSpecifiers)
                {
                    if (!versionSpecifier.Contains(uniquePkgName))
                    {
                        continue;
                    }

                    var versionFromDefinition = packageDefinitions.First(item => item.Name == uniquePkgName).Version;
                    if (versionSpecifier[1] != versionFromDefinition)
                    {
                        var noMatchMessage = new System.Text.StringBuilder();
                        noMatchMessage.AppendFormat("Command line version specified, {0}, could not resolve to one of the available versions of package {1}:", versionSpecifier[1], uniquePkgName);
                        noMatchMessage.AppendLine();
                        noMatchMessage.AppendFormat("\t{0}", versionFromDefinition);
                        noMatchMessage.AppendLine();
                        throw new Exception(noMatchMessage.ToString());
                    }
                }
            }

            if (enforceBamAssemblyVersions)
            {
                // for all packages that make up this assembly, ensure that their requirements on the version of the Bam
                // assemblies are upheld, prior to compiling the code
                foreach (var pkgDefn in packageDefinitions)
                {
                    pkgDefn.ValidateBamAssemblyRequirements();
                }
            }

            Graph.Instance.SetPackageDefinitions(packageDefinitions);
        }