/*
         * Plan the build against N workers
         */
        public Dictionary <int, List <Dictionary <BuildTargetAdvanced, List <string> > > > GetBuildPlan(int workers)
        {
            BuildSourceFilesCollection <BuildTargetAdvanced> sourceFiles = this.GetSourceFiles();
            TES5BuildPlanBuilder buildPlanBuilder = new TES5BuildPlanBuilder(this.GetDependencyGraph());
            Dictionary <int, List <Dictionary <BuildTargetAdvanced, List <string> > > > buildPlan = buildPlanBuilder.CreateBuildPlan(sourceFiles, workers);

            return(buildPlan);
        }
        /*
         * Get source files, assigned per-build target
         * If intersected source files is not null, they will be intersected with build target source files,
         * otherwise all files will be claimed
         */
        public static BuildSourceFilesCollection <T> GetSourceFiles <T>(this IEnumerable <T> buildTargets, string[]?intersectedSourceFiles = null) where T : IBuildTarget
        {
            BuildSourceFilesCollection <T> collection = new BuildSourceFilesCollection <T>();

            foreach (var buildTarget in buildTargets)
            {
                collection.Add(buildTarget, buildTarget.GetSourceFileList(intersectedSourceFiles));
            }
            return(collection);
        }
Exemple #3
0
        /*
         * Get source files, assigned per-build target
         * If intersected source files is not null, they will be intersected with build target source files,
         * otherwise all files will be claimed
         */
        public BuildSourceFilesCollection GetSourceFiles(string[] intersectedSourceFiles = null)
        {
            BuildSourceFilesCollection collection = new BuildSourceFilesCollection();

            foreach (var buildTarget in this.buildTargets.Values)
            {
                collection.add(buildTarget, buildTarget.GetSourceFileList(intersectedSourceFiles));
            }

            return(collection);
        }
        public static int GetTotalSourceFiles(this IEnumerable <IBuildTarget> buildTargets)
        {
            BuildSourceFilesCollection <IBuildTarget> sourceFiles = buildTargets.GetSourceFiles();

            return(sourceFiles.Sum(sf => sf.Value.Length));
        }
Exemple #5
0
        public Dictionary <int, List <Dictionary <string, List <string> > > > CreateBuildPlan(BuildSourceFilesCollection scripts, int threads = BuildTargetCommand.DefaultThreads)
        {
            Dictionary <string, string> codeScripts = new Dictionary <string, string>();

            /*
             * Mapping script names to build names
             */
            Dictionary <string, string> scriptToBuild = new Dictionary <string, string>();

            foreach (var kvp in scripts)
            {
                var buildName    = kvp.Key;
                var buildScripts = kvp.Value;
                foreach (var script in buildScripts)
                {
                    string scriptName    = script.Substring(0, script.Length - 4);
                    string scriptNameKey = scriptName.ToLower();
                    codeScripts.Add(scriptNameKey, scriptName);
                    scriptToBuild.Add(scriptNameKey, buildName);
                }
            }

            List <Dictionary <string, List <string> > > preparedChunks = new List <Dictionary <string, List <string> > >();
            List <string> nonpairedScripts = new List <string>();
            int           previousCount    = codeScripts.Count;

            /*
             * Prepare chunks of scripts and push lone scripts into a different array
             */
            while (codeScripts.Any())
            {
                var      currentScript = codeScripts.First().Key;
                string[] preparedChunk = this.graph.getScriptsToCompile(currentScript);
                if (preparedChunk.Length > 1)
                {
                    /*
                     * Chunk mapped per-build
                     */
                    Dictionary <string, List <string> > preparedMappedChunk = new Dictionary <string, List <string> >();
                    foreach (var chunkScript in preparedChunk)
                    {
                        string chunkScriptKey = chunkScript.ToLower();
                        codeScripts.Remove(chunkScriptKey);
                        preparedMappedChunk.AddNewListIfNotContainsKeyAndAddValueToList(scriptToBuild[chunkScriptKey], chunkScript);
                    }

                    preparedChunks.Add(preparedMappedChunk);
                }
                else
                {
                    string nonpairedChunkScript = preparedChunk[0];
                    nonpairedScripts.Add(nonpairedChunkScript);
                    string nonpairedChunkScriptKey = nonpairedChunkScript.ToLower();
                    codeScripts.Remove(nonpairedChunkScriptKey);
                }

                if (codeScripts.Count >= previousCount)
                {
                    throw new InvalidOperationException("Error in planning build, circuit breaker on.");
                }
                else
                {
                    previousCount = codeScripts.Count;
                }
            }

            Dictionary <int, List <Dictionary <string, List <string> > > > threadBuckets = new Dictionary <int, List <Dictionary <string, List <string> > > >();
            Dictionary <int, int> threadBucketsSizes = new Dictionary <int, int>();
            int bucket = 0;

            foreach (var chunk in preparedChunks)
            {
                threadBucketsSizes.AddIfNotContainsKey(bucket, 0);
                threadBuckets.GetOrAddNewIfNotContainsKey(bucket).Add(chunk);
                foreach (var kvp in chunk)
                {
                    var chunkBuild   = kvp.Key;
                    var chunkScripts = kvp.Value;
                    threadBucketsSizes[bucket] += chunkScripts.Count;
                }

                bucket++;
                if (bucket == threads)
                {
                    bucket = 0;
                }
            }

            //Evening the buckets
            int biggestBucket = threadBucketsSizes.Max(kvp => kvp.Value);

            foreach (var kvp in threadBuckets)
            {
                var bucketKey     = kvp.Key;
                int bucketSize    = threadBucketsSizes[bucketKey];
                int neededScripts = biggestBucket - bucketSize;
                Dictionary <string, List <string> > eveningChunk = new Dictionary <string, List <string> >();
                if (neededScripts >= nonpairedScripts.Count)
                {
                    foreach (var nonpairedScript in nonpairedScripts)
                    {
                        string chunkScriptBuild = scriptToBuild[nonpairedScript.ToLower()];
                        eveningChunk.GetOrAddNewIfNotContainsKey(chunkScriptBuild).Add(nonpairedScript);
                    }

                    threadBuckets[bucketKey].Add(eveningChunk);
                    //Not sure if should be here but prolly yes?
                    threadBucketsSizes[bucketKey] += nonpairedScripts.Count;
                    nonpairedScripts = new List <string>();
                    break;
                }

                string[] sliceOfNonpairedScripts = nonpairedScripts.Take(neededScripts).ToArray();
                foreach (var sliceOfNonpairedScript in sliceOfNonpairedScripts)
                {
                    string chunkScriptBuild = scriptToBuild[sliceOfNonpairedScript.ToLower()];
                    eveningChunk.GetOrAddNewIfNotContainsKey(chunkScriptBuild).Add(sliceOfNonpairedScript);
                }

                threadBuckets[bucketKey].Add(eveningChunk);
                threadBucketsSizes[bucketKey] += neededScripts;
                nonpairedScripts = nonpairedScripts.Skip(neededScripts).ToList();
            }

            Dictionary <int, List <Dictionary <string, List <string> > > > restChunks = new Dictionary <int, List <Dictionary <string, List <string> > > >();
            int restChunkBucket = 0;

            foreach (var nonpairedScript in nonpairedScripts)
            {
                Dictionary <string, List <string> > singleScriptChunk = new Dictionary <string, List <string> >();
                string chunkScriptBuild = scriptToBuild[nonpairedScript.ToLower()];
                singleScriptChunk.AddNewListIfNotContainsKey(chunkScriptBuild);
                restChunks.AddNewListIfNotContainsKey(restChunkBucket);

                singleScriptChunk[chunkScriptBuild].Add(nonpairedScript);
                restChunks[restChunkBucket].Add(singleScriptChunk);
                restChunkBucket++;
                if (restChunkBucket == threads)
                {
                    restChunkBucket = 0;
                }
            }

            foreach (var kvp in restChunks)
            {
                var bucketKey           = kvp.Key;
                var restOfScriptsChunks = kvp.Value;
                foreach (var restOfScriptsChunk in restOfScriptsChunks)
                {
                    threadBuckets[bucketKey].Add(restOfScriptsChunk);
                }
            }

            return(threadBuckets);
        }
Exemple #6
0
        public int GetTotalSourceFiles()
        {
            BuildSourceFilesCollection sourceFiles = this.GetSourceFiles();

            return(sourceFiles.Sum(sf => sf.Value.Length));
        }