/* * Plan the build against N workers */ public Dictionary <int, List <Dictionary <BuildTargetAdvanced, List <string> > > > GetBuildPlan(int workers) { BuildSourceFilesCollection <BuildTargetAdvanced> sourceFiles = this.GetSourceFiles(); TES5BuildPlanBuilder buildPlanBuilder = new TES5BuildPlanBuilder(this.GetDependencyGraph()); Dictionary <int, List <Dictionary <BuildTargetAdvanced, List <string> > > > buildPlan = buildPlanBuilder.CreateBuildPlan(sourceFiles, workers); return(buildPlan); }
/* * Get source files, assigned per-build target * If intersected source files is not null, they will be intersected with build target source files, * otherwise all files will be claimed */ public static BuildSourceFilesCollection <T> GetSourceFiles <T>(this IEnumerable <T> buildTargets, string[]?intersectedSourceFiles = null) where T : IBuildTarget { BuildSourceFilesCollection <T> collection = new BuildSourceFilesCollection <T>(); foreach (var buildTarget in buildTargets) { collection.Add(buildTarget, buildTarget.GetSourceFileList(intersectedSourceFiles)); } return(collection); }
/* * Get source files, assigned per-build target * If intersected source files is not null, they will be intersected with build target source files, * otherwise all files will be claimed */ public BuildSourceFilesCollection GetSourceFiles(string[] intersectedSourceFiles = null) { BuildSourceFilesCollection collection = new BuildSourceFilesCollection(); foreach (var buildTarget in this.buildTargets.Values) { collection.add(buildTarget, buildTarget.GetSourceFileList(intersectedSourceFiles)); } return(collection); }
public static int GetTotalSourceFiles(this IEnumerable <IBuildTarget> buildTargets) { BuildSourceFilesCollection <IBuildTarget> sourceFiles = buildTargets.GetSourceFiles(); return(sourceFiles.Sum(sf => sf.Value.Length)); }
public Dictionary <int, List <Dictionary <string, List <string> > > > CreateBuildPlan(BuildSourceFilesCollection scripts, int threads = BuildTargetCommand.DefaultThreads) { Dictionary <string, string> codeScripts = new Dictionary <string, string>(); /* * Mapping script names to build names */ Dictionary <string, string> scriptToBuild = new Dictionary <string, string>(); foreach (var kvp in scripts) { var buildName = kvp.Key; var buildScripts = kvp.Value; foreach (var script in buildScripts) { string scriptName = script.Substring(0, script.Length - 4); string scriptNameKey = scriptName.ToLower(); codeScripts.Add(scriptNameKey, scriptName); scriptToBuild.Add(scriptNameKey, buildName); } } List <Dictionary <string, List <string> > > preparedChunks = new List <Dictionary <string, List <string> > >(); List <string> nonpairedScripts = new List <string>(); int previousCount = codeScripts.Count; /* * Prepare chunks of scripts and push lone scripts into a different array */ while (codeScripts.Any()) { var currentScript = codeScripts.First().Key; string[] preparedChunk = this.graph.getScriptsToCompile(currentScript); if (preparedChunk.Length > 1) { /* * Chunk mapped per-build */ Dictionary <string, List <string> > preparedMappedChunk = new Dictionary <string, List <string> >(); foreach (var chunkScript in preparedChunk) { string chunkScriptKey = chunkScript.ToLower(); codeScripts.Remove(chunkScriptKey); preparedMappedChunk.AddNewListIfNotContainsKeyAndAddValueToList(scriptToBuild[chunkScriptKey], chunkScript); } preparedChunks.Add(preparedMappedChunk); } else { string nonpairedChunkScript = preparedChunk[0]; nonpairedScripts.Add(nonpairedChunkScript); string nonpairedChunkScriptKey = nonpairedChunkScript.ToLower(); codeScripts.Remove(nonpairedChunkScriptKey); } if (codeScripts.Count >= previousCount) { throw new InvalidOperationException("Error in planning build, circuit breaker on."); } else { previousCount = codeScripts.Count; } } Dictionary <int, List <Dictionary <string, List <string> > > > threadBuckets = new Dictionary <int, List <Dictionary <string, List <string> > > >(); Dictionary <int, int> threadBucketsSizes = new Dictionary <int, int>(); int bucket = 0; foreach (var chunk in preparedChunks) { threadBucketsSizes.AddIfNotContainsKey(bucket, 0); threadBuckets.GetOrAddNewIfNotContainsKey(bucket).Add(chunk); foreach (var kvp in chunk) { var chunkBuild = kvp.Key; var chunkScripts = kvp.Value; threadBucketsSizes[bucket] += chunkScripts.Count; } bucket++; if (bucket == threads) { bucket = 0; } } //Evening the buckets int biggestBucket = threadBucketsSizes.Max(kvp => kvp.Value); foreach (var kvp in threadBuckets) { var bucketKey = kvp.Key; int bucketSize = threadBucketsSizes[bucketKey]; int neededScripts = biggestBucket - bucketSize; Dictionary <string, List <string> > eveningChunk = new Dictionary <string, List <string> >(); if (neededScripts >= nonpairedScripts.Count) { foreach (var nonpairedScript in nonpairedScripts) { string chunkScriptBuild = scriptToBuild[nonpairedScript.ToLower()]; eveningChunk.GetOrAddNewIfNotContainsKey(chunkScriptBuild).Add(nonpairedScript); } threadBuckets[bucketKey].Add(eveningChunk); //Not sure if should be here but prolly yes? threadBucketsSizes[bucketKey] += nonpairedScripts.Count; nonpairedScripts = new List <string>(); break; } string[] sliceOfNonpairedScripts = nonpairedScripts.Take(neededScripts).ToArray(); foreach (var sliceOfNonpairedScript in sliceOfNonpairedScripts) { string chunkScriptBuild = scriptToBuild[sliceOfNonpairedScript.ToLower()]; eveningChunk.GetOrAddNewIfNotContainsKey(chunkScriptBuild).Add(sliceOfNonpairedScript); } threadBuckets[bucketKey].Add(eveningChunk); threadBucketsSizes[bucketKey] += neededScripts; nonpairedScripts = nonpairedScripts.Skip(neededScripts).ToList(); } Dictionary <int, List <Dictionary <string, List <string> > > > restChunks = new Dictionary <int, List <Dictionary <string, List <string> > > >(); int restChunkBucket = 0; foreach (var nonpairedScript in nonpairedScripts) { Dictionary <string, List <string> > singleScriptChunk = new Dictionary <string, List <string> >(); string chunkScriptBuild = scriptToBuild[nonpairedScript.ToLower()]; singleScriptChunk.AddNewListIfNotContainsKey(chunkScriptBuild); restChunks.AddNewListIfNotContainsKey(restChunkBucket); singleScriptChunk[chunkScriptBuild].Add(nonpairedScript); restChunks[restChunkBucket].Add(singleScriptChunk); restChunkBucket++; if (restChunkBucket == threads) { restChunkBucket = 0; } } foreach (var kvp in restChunks) { var bucketKey = kvp.Key; var restOfScriptsChunks = kvp.Value; foreach (var restOfScriptsChunk in restOfScriptsChunks) { threadBuckets[bucketKey].Add(restOfScriptsChunk); } } return(threadBuckets); }
public int GetTotalSourceFiles() { BuildSourceFilesCollection sourceFiles = this.GetSourceFiles(); return(sourceFiles.Sum(sf => sf.Value.Length)); }