public void BuildSucceeded(BuildInstance instance)
{
// build succeeded; create/update the history entry
var entry = new HistoryEntry();
entry.Byproducts = instance.Byproducts.Select(b => b.ToLower()).ToSet();
var node = instance.Pipeline;
while (node != null)
{
entry.StageTypes.Add(node.GetType());
entry.StageHashes.Add(node.Hash());
node = node.InputNode;
}
bool hashInputs = (instance.Env.InputChangeDetection & ChangeDetection.Hash) != 0;
foreach (var input in instance.Inputs)
entry.InputCache.Add(CreateFileEntry(input, hashInputs));
bool hashOutputs = (instance.Env.OutputChangeDetection & ChangeDetection.Hash) != 0;
entry.OutputCache.Add(CreateFileEntry(instance.OutputPath, hashOutputs));
foreach (var output in instance.Byproducts)
entry.OutputCache.Add(CreateFileEntry(output, hashOutputs));
// add dependent builds
entry.Dependencies.AddRange(instance.Dependencies);
// create entries for each dependent temp build
foreach (var build in instance.TempBuilds)
entry.TempDependencies.Add(CreateTempEntry(build));
history.AddOrUpdate(instance.OutputName.ToLower(), entry, (k, h) => entry);
}
public IEnumerable<object> Run(BuildInstance instance, IEnumerable<object> inputs)
{
// parse the material as XML
var stream = inputs.First() as Stream;
if (stream == null)
{
instance.Log(LogLevel.Error, "Inputs to TestParser must be a single Stream object.");
return null;
}
var document = XDocument.Load(stream);
var root = document.Element("Test");
if (root == null)
{
instance.Log(LogLevel.Error, "Test file must start with a <Test> node.");
return null;
}
string data = null;
var dataElement = root.Element("Data");
if (dataElement != null)
data = dataElement.Value;
// example of finding all "dependencies" of this asset and kicking off builds for them
var dependencies = root.Element("Dependencies");
if (dependencies != null)
{
foreach (var element in dependencies.Elements())
instance.Start((string)element.Attribute("Name"));
}
// example of a "temporary" build, one that you can embed the results into this asset
var embed = (string)root.Attribute("Embed");
string embeddedData = null;
if (embed != null)
{
var tempBuild = instance.StartTemp(embed);
if (tempBuild == null) // some error ocurred while trying to build it
return null;
var tempStream = tempBuild.Results.First() as Stream;
embeddedData = string.Concat(ToByteArray(tempStream));
}
var output = new MemoryStream();
var writer = new StreamWriter(output) { AutoFlush = true };
writer.WriteLine("This is my asset data! Woohoo!");
if (data != null)
writer.WriteLine(data);
if (embeddedData != null)
writer.WriteLine("Some embedded data: " + embeddedData);
return new[] { output };
}
public override IEnumerable<object> Evaluate(BuildInstance instance, IEnumerable<object> inputs)
{
try
{
return Processor(instance, inputs);
}
catch (Exception e)
{
instance.Log.Error(e.ToString());
return null;
}
}
public override IEnumerable<object> Evaluate(BuildInstance instance, IEnumerable<object> unused)
{
// open up the filestreams
try
{
return instance.Inputs.Select(p => File.OpenRead(p)).ToArray();
}
catch (IOException e)
{
instance.Log.Error(e.Message);
return null;
}
}
public override IEnumerable<object> Evaluate(BuildInstance instance, IEnumerable<object> inputs)
{
// figure out the final names of each output
var outputs = new List<string>();
outputs.Add(instance.OutputPath);
outputs.AddRange(instance.Byproducts);
// make sure we have enough inputs to satisfy each output
var inputArray = inputs.ToArray();
if (inputArray.Length != outputs.Count)
{
instance.Log.Error("Number of inputs does not match number of outputs for '{0}' (line {1}).", instance.OutputName, LineNumber);
return null;
}
instance.Results = inputs;
if (instance.IsTempBuild && !instance.Env.WriteTempBuilds)
return null;
// match each input to an output name
for (int i = 0; i < inputArray.Length; i++)
{
// if we have a filestream, we can do a straight file copy because we know it hasn't been changed
var outputPath = outputs[i];
var stream = inputArray[i] as Stream;
if (stream == null)
return null;
var file = stream as FileStream;
if (file != null)
File.Copy(file.Name, outputPath, true);
else if (stream == null)
{
instance.Log.Error("Inputs to Build() node must all be of type stream ('{0}' on line {1}).", instance.OutputName, LineNumber);
return null;
}
else
{
// otherwise, write to file
stream.Seek(0, SeekOrigin.Begin);
using (var outputStream = File.Create(outputPath))
stream.CopyTo(outputStream);
}
stream.Close();
}
return Enumerable.Empty<Stream>();
}
public bool ResolveNames(BuildInstance instance)
{
// resolve input names into full paths; if any fail, an error occurs
var paths = new List<string>();
foreach (var input in inputs)
{
var fullName = instance.Match.Result(input);
var path = instance.Env.ResolveInput(fullName);
if (string.IsNullOrEmpty(path))
{
instance.Log.Error("Could not resolve input '{0}' (line {1}).", fullName, LineNumber);
return false;
}
paths.Add(path);
}
instance.Inputs = paths.ToArray();
return true;
}
public abstract IEnumerable<object> Evaluate(BuildInstance instance, IEnumerable<object> inputs);
BuildInstance InternalStart(string name, OutputNode rule, BuildInstance instance)
{
// walk down the pipeline and build from the bottom-up
var currentStage = rule.GetBottomNode();
var inputNode = currentStage as InputNode;
if (!inputNode.ResolveNames(instance) || !rule.ResolveNames(instance))
return BuildFailed(name, instance);
// check to see if we even need to do this build
if (!FullRebuild && !history.ShouldBuild(instance))
{
allAssets.Add(name);
foreach (var entry in history.GetDependencies(instance.OutputName))
allAssets.Add(entry);
Log.Info("Skipping '{0}' (up-to-date).", name);
instance.Status = BuildStatus.Skipped;
Interlocked.Increment(ref Stats.Skipped);
return instance;
}
// run the pipeline
IEnumerable<object> state = null;
while (currentStage != null)
{
// run the current stage, saving the results and passing them on to the next stage in the pipeline
try
{
state = currentStage.Evaluate(instance, state);
}
catch (Exception e)
{
Log.Error("Exception thrown while building '{0}': {1}", name, e);
return BuildFailed(name, instance);
}
if (state == null)
{
if (instance.IsTempBuild && currentStage is OutputNode)
return instance;
return BuildFailed(name, instance);
}
currentStage = currentStage.OutputNode;
}
if (instance.Status == BuildStatus.Failed)
return BuildFailed(name, instance);
history.BuildSucceeded(instance);
builtAssets.Add(instance.OutputName);
allAssets.Add(instance.OutputName);
foreach (var byproduct in instance.Byproducts)
{
allAssets.Add(byproduct);
builtAssets.Add(byproduct);
}
Log.Write("Build for '{0}' successful.", name);
instance.Status = BuildStatus.Succeeded;
Interlocked.Increment(ref Stats.Succeeded);
return instance;
}
BuildInstance BuildFailed(string name, BuildInstance instance)
{
history.BuildFailed(instance);
Log.Error("FAILED! Build for '{0}'", name);
instance.Status = BuildStatus.Failed;
Interlocked.Increment(ref Stats.Failed);
return instance;
}
public Task<BuildInstance> Start(string name, bool tempBuild = false)
{
// find best applicable rule
var best = rules
.Select(r => new { Node = r, Match = r.Match(name) })
.Where(m => m.Match.Success)
.GroupBy(m => m.Node.Priority)
.OrderBy(g => g.Key)
.FirstOrDefault();
if (best == null)
{
Log.Error("No applicable rule found for asset '{0}'.", name);
Interlocked.Increment(ref Stats.Failed);
return null;
}
else if (best.Count() != 1)
Log.Warning("More than one rule with the same priority matches asset '{0}' (rules on lines: {1})", name, string.Join(", ", best.Select(b => b.Node.LineNumber)));
// we've found the rule we will use. queue up the task to build the asset, or return the current one if it's already being built
var chosen = best.First();
var task = runningBuilds.GetOrAdd(name, new Lazy<Task<BuildInstance>>(() =>
{
var instance = new BuildInstance(this, chosen.Match, chosen.Node, tempBuild);
var job = Task.Run(() => InternalStart(name, chosen.Node, instance));
job.ContinueWith(t => runningBuilds.Remove(name));
return job;
})).Value;
// also register the task for any byproducts
foreach (var byproduct in chosen.Node.Byproducts)
{
var byproductName = chosen.Match.Result(byproduct);
runningBuilds.TryAdd(byproductName, new Lazy<Task<BuildInstance>>(() => task));
}
return task;
}
public void BuildFailed(BuildInstance instance)
{
// build failed; remove the history entry to force a rebuild.
HistoryEntry entry;
history.TryRemove(instance.OutputName.ToLower(), out entry);
}
TempEntry CreateTempEntry(BuildInstance instance)
{
var entry = new TempEntry();
bool hashInputs = (instance.Env.InputChangeDetection & ChangeDetection.Hash) != 0;
foreach (var input in instance.Inputs)
{
entry.Inputs.Add(CreateFileEntry(input, hashInputs));
entry.InputPaths.Add(input);
}
// create entries for each dependent temp build
foreach (var build in instance.TempBuilds)
entry.TempDependencies.Add(CreateTempEntry(build));
return entry;
}
public bool ShouldBuild(BuildInstance instance)
{
// do comparisons in order from cheapest to most expensive to try to early out when a change is obvious
// check 1: see if we have history for this output
HistoryEntry entry;
if (!history.TryGetValue(instance.OutputName.ToLower(), out entry))
return true;
// check 3: make sure the byproducts match
var byproductSet = instance.Byproducts.Select(b => b.ToLower()).ToSet();
if (!byproductSet.SetEquals(entry.Byproducts))
return true;
// check 4: compare number and type of pipeline stages
var node = instance.Pipeline;
for (int i = 0; i < entry.StageTypes.Count; i++)
{
if (node == null || node.GetType() != entry.StageTypes[i])
return true;
node = node.InputNode;
}
// any nodes left over mean that the pipeline was changed
if (node != null)
return true;
// check 5: check for pipeline processor changes
node = instance.Pipeline;
foreach (var stage in entry.StageHashes)
{
if (stage != node.Hash())
return true;
node = node.InputNode;
}
// check 6: changes in inputs
if (instance.Env.InputChangeDetection != ChangeDetection.None)
{
for (int i = 0; i < instance.Inputs.Length; i++)
{
if (CheckChanged(instance.Env.InputChangeDetection, new FileInfo(instance.Inputs[i]), entry.InputCache[i]))
return true;
}
}
// check 7: changes in outputs
if (instance.Env.OutputChangeDetection != ChangeDetection.None)
{
if (CheckChanged(instance.Env.OutputChangeDetection, new FileInfo(instance.OutputPath), entry.OutputCache[0]))
return true;
for (int i = 0; i < instance.Byproducts.Length; i++)
{
if (CheckChanged(instance.Env.OutputChangeDetection, new FileInfo(instance.Byproducts[i]), entry.OutputCache[i + 1]))
return true;
}
}
// check 8: look at any dependent temp builds and see if they have been changed
if (instance.Env.InputChangeDetection != ChangeDetection.None)
{
if (entry.TempDependencies.Any(t => TempInputsHaveChanged(t, instance.Env.InputChangeDetection)))
return true;
}
// at this point, we can safely say that the entire pipeline is the same. no need to do a build
// however, some of our dependencies may have changed, so let them sort themselves out
foreach (var dependency in entry.Dependencies)
context.Start(dependency);
return false;
}
public bool ResolveNames(BuildInstance instance)
{
var paths = new List<string>();
foreach (var output in Byproducts.Select(b => instance.Match.Result(b)))
{
var outputPath = instance.IsTempBuild ?
instance.Env.ResolveTemp(output) :
instance.Env.ResolveOutput(output);
if (string.IsNullOrEmpty(outputPath))
{
instance.Log.Error("Could not resolve output '{0}' (line {1}).", output, LineNumber);
return false;
}
paths.Add(outputPath);
}
instance.OutputPath = instance.IsTempBuild ? instance.Env.ResolveTemp(instance.OutputName) : instance.Env.ResolveOutput(instance.OutputName);
instance.Byproducts = paths.ToArray();
return !string.IsNullOrEmpty(instance.OutputPath);
}
internal ArgProvider(BuildInstance instance, IEnumerable<object> inputs, int line)
{
this.instance = instance;
this.inputs = inputs.ToArray();
this.lineNumber = line;
}
public override IEnumerable<object> Evaluate(BuildInstance instance, IEnumerable<object> inputs)
{
if (!File.Exists(fileName))
{
instance.Log.Error("Could not find external program '{0}'. (line {1})", fileName, LineNumber);
return null;
}
if (resultProviders.Count == 0)
{
instance.Log.Error("Running an external tool requires at least one Result specifier. (line {0})", LineNumber);
return null;
}
// perform argument replacement
var argProvider = new ArgProvider(instance, inputs, LineNumber);
string currentArguments = instance.Match.Result(argumentFormat);
currentArguments = string.Format(currentArguments, argProviders.Select(p => p(argProvider)).ToArray());
var startInfo = new ProcessStartInfo(fileName, currentArguments);
startInfo.CreateNoWindow = true;
startInfo.UseShellExecute = false;
startInfo.RedirectStandardOutput = (options & RunOptions.RedirectOutput) != 0;
startInfo.RedirectStandardError = (options & RunOptions.RedirectError) != 0;
var process = new Process();
process.StartInfo = startInfo;
process.EnableRaisingEvents = true;
process.OutputDataReceived += (o, e) =>
{
if (!string.IsNullOrEmpty(e.Data))
instance.Log.Info(e.Data);
};
process.ErrorDataReceived += (o, e) =>
{
if (!string.IsNullOrEmpty(e.Data))
instance.Log.Error(e.Data);
};
process.Start();
if (startInfo.RedirectStandardOutput)
process.BeginOutputReadLine();
if (startInfo.RedirectStandardError)
process.BeginErrorReadLine();
process.WaitForExit();
if ((options & RunOptions.DontCheckResultCode) == 0 && process.ExitCode != 0)
{
instance.Log.Error("Running tool '{0}' failed with result code {1}. (line {2})", fileName, process.ExitCode, LineNumber);
return null;
}
var results = new List<Stream>();
foreach (var output in resultProviders.Select(p => p(argProvider)))
{
string path = instance.Match.Result(output);
results.Add(File.OpenRead(path));
}
return results;
}
