/// <summary>
/// Calculates a score between 0.0 (completely different) and 1.0 (the same). Two token lists are
/// concerted to unit vector weights and Soft TF-IDF algorithm is applied to them
/// </summary>
/// <param name="source"></param>
/// <param name="target"></param>
/// <returns></returns>
public double GetScore(TokenizedString source, TokenizedString target)
{
if (_alwaysCompareLongerToShorter && source.Count < target.Count)
{
var temp = source;
source = target;
target = temp;
}
var sVector = GetTokenUnitVector(source);
var tVector = GetTokenUnitVector(target);
double sim = 0.0;
foreach (var sToken in sVector)
{
if (tVector.Contains(sToken))
{
sim += sVector.GetWeight(sToken) * tVector.GetWeight(sToken);
}
else
{
// find best matching token
var matchToken = tVector
.Select(t => new {Token = t, Score = _distance.Score(sToken, t)})
.Aggregate((next, current) => next.Score > current.Score ? next : current);
if (matchToken.Score >= _tokenMatchThreshold)
{
sim += sVector.GetWeight(sToken) * tVector.GetWeight(matchToken.Token) * matchToken.Score;
}
}
}
return sim;
}
private ExecuteReasoning(
EReason reason,
TokenizedString outputFilePath = null,
TokenizedString inputFilePath = null)
{
this.Reason = reason;
this.OutputFilePath = outputFilePath;
this.InputFilePath = inputFilePath;
}
Init(
Bam.Core.Module parent)
{
base.Init(parent);
this.Macros["OutputName"] = TokenizedString.CreateVerbatim("Cxx_NSISInstaller");
this.SourceFolder <CxxExecutableStripped>(Publisher.StrippedBinaryCollation.Key);
}
private ExecuteReasoning(
EReason reason,
TokenizedString outputFilePath = null,
TokenizedString inputFilePath = null)
{
this.Reason = reason;
this.OutputFilePath = outputFilePath;
this.InputFilePath = inputFilePath;
}
/// <summary>
/// Adds string tokens to the set
/// </summary>
/// <param name="tokens"></param>
public void AddDocument(TokenizedString tokens)
{
var seenTokens = new HashSet<Token>();
foreach (var token in tokens.Where(token => !seenTokens.Contains(token)))
{
seenTokens.Add(token);
int frequency = 0;
_dict.TryGetValue(token, out frequency);
_dict[token] = frequency + 1;
}
NumberOfDucoments++;
}
Display()
{
Log.Info("\nBuildAMation Statistics");
Log.Info("Memory Usage");
Log.Info("Peak working set size : {0:N2}MB", BytesToMegaBytes(System.Diagnostics.Process.GetCurrentProcess().PeakWorkingSet64));
Log.Info("Peak virtual size : {0:N2}MB", BytesToMegaBytes(System.Diagnostics.Process.GetCurrentProcess().PeakVirtualMemorySize64));
Log.Info("GC total memory : {0:N2}MB (after GC, {1:N2}MB)", BytesToMegaBytes(System.GC.GetTotalMemory(false)), BytesToMegaBytes(System.GC.GetTotalMemory(true)));
Log.Info("\nObject counts");
Log.Info("Tokenized strings : {0} ({1} unshared)", TokenizedString.Count, TokenizedString.UnsharedCount);
TokenizedString.DumpCache();
Log.Info("Modules : {0}", Module.Count);
TimingProfileUtilities.DumpProfiles();
}
IProceduralHeaderFromToolOutputPolicy.HeaderFromToolOutput(
ProceduralHeaderFileFromToolOutput sender,
ExecutionContext context,
TokenizedString outputPath,
ICommandLineTool tool)
{
var toolPath = tool.Executable.ToString();
CommandLineProcessor.Processor.Execute(context, toolPath);
var destPath = outputPath.ToString();
var destDir = System.IO.Path.GetDirectoryName(destPath);
Bam.Core.IOWrapper.CreateDirectoryIfNotExists(destDir);
var tempPath = Bam.Core.IOWrapper.CreateTemporaryFile();
using (System.IO.TextWriter writeFile = new System.IO.StreamWriter(tempPath))
{
writeFile.Write(context.OutputStringBuilder.ToString());
}
var moveFile = true;
if (System.IO.File.Exists(destPath))
{
// compare contents
using (System.IO.TextReader existingFile = new System.IO.StreamReader(destPath))
{
var contents = existingFile.ReadToEnd();
var contentsL = contents.Length;
var oldL = context.OutputStringBuilder.ToString().Length;
if (contents.Equals(context.OutputStringBuilder.ToString()))
{
moveFile = false;
}
}
}
if (moveFile)
{
Bam.Core.Log.Info("Written procedurally generated header : {0}, from the output of {1}", destPath, toolPath);
System.IO.File.Delete(destPath);
System.IO.File.Move(tempPath, destPath);
}
else
{
Bam.Core.Log.Info("{0} contents have not changed", destPath);
System.IO.File.Delete(tempPath);
}
context.OutputStringBuilder.Clear();
}
Add(
string key,
TokenizedString value)
{
if (key.StartsWith(TokenizedString.TokenPrefix) || key.EndsWith(TokenizedString.TokenSuffix))
{
throw new Exception("Invalid macro key: {0}", key);
}
if (null == value)
{
throw new Exception("Cannot assign null to macro '{0}'", key);
}
this.DictInternal[FormattedKey(key)] = value;
}
Add(
string key,
TokenizedString value)
{
if (key.StartsWith(TokenizedString.TokenPrefix) || key.EndsWith(TokenizedString.TokenSuffix))
{
throw new Exception("Invalid macro key: {0}", key);
}
if (null == value)
{
throw new Exception("Cannot assign null to macro '{0}'", key);
}
this.DictInternal[FormattedKey(key)] = value;
}
Init(
Bam.Core.Module parent)
{
base.Init(parent);
this.Macros["OutputName"] = TokenizedString.CreateVerbatim("C_SymbolsTarBall");
this.SourceFolder <CExecutableDebugSymbols>(Publisher.DebugSymbolCollation.Key);
this.PrivatePatch(settings =>
{
var tarSettings = settings as Installer.ITarBallSettings;
tarSettings.CompressionType = Installer.ETarCompressionType.gzip;
});
}
Init(
Bam.Core.Module parent)
{
base.Init(parent);
this.PrivatePatch(settings =>
{
var dmgSettings = settings as Installer.IDiskImageSettings;
dmgSettings.ImageSize = "128m";
});
this.Macros["OutputName"] = TokenizedString.CreateVerbatim("Cxx_DiskImageInstaller");
this.SourceFolder <CxxExecutableStripped>(Publisher.StrippedBinaryCollation.Key);
}
AssignLinkTarget(
TokenizedString path = null)
{
#if __MonoCS__
if (path == null)
{
var symlink = new Mono.Unix.UnixSymbolicLinkInfo(this.SourcePath.Parse());
this.Macros["LinkTarget"] = Bam.Core.TokenizedString.CreateVerbatim(symlink.ContentsPath);
}
else
{
this.Macros["LinkTarget"] = path;
}
#else
throw new System.NotSupportedException("Unable to get symbolic link target on Windows");
#endif
}
Init(
Bam.Core.Module parent)
{
base.Init(parent);
this.Macros["OutputName"] = TokenizedString.CreateVerbatim("C_TarBallInstaller");
this.SourceFolder <CExecutableStripped>(Publisher.StrippedBinaryCollation.Key);
this.PrivatePatch(settings =>
{
var tarSettings = settings as Installer.ITarBallSettings;
tarSettings.CompressionType = Installer.ETarCompressionType.gzip;
if (this.BuildEnvironment.Platform.Includes(Bam.Core.EPlatform.Linux))
{
tarSettings.TransformRegEx = "'s,^.,toplevelfolder,'";
}
});
}
IProceduralHeaderFromToolOutputPolicy.HeaderFromToolOutput(
ProceduralHeaderFileFromToolOutput sender,
ExecutionContext context,
TokenizedString outputPath,
ICommandLineTool tool)
{
var toolTarget = (tool as Bam.Core.Module).MetaData as XcodeBuilder.Target;
var toolConfiguration = toolTarget.GetConfiguration(tool as Bam.Core.Module);
var output = outputPath.Parse();
var commands = new Bam.Core.StringArray();
commands.Add(System.String.Format("[[ ! -d {0} ]] && mkdir -p {0}", System.IO.Path.GetDirectoryName(output)));
commands.Add(System.String.Format("{0} > {1}", CommandLineProcessor.Processor.StringifyTool(tool), output));
toolTarget.AddPostBuildCommands(commands, toolConfiguration);
// alias the tool's target so that inter-target dependencies can be set up
sender.MetaData = toolTarget;
}
IProceduralHeaderFromToolOutputPolicy.HeaderFromToolOutput(
ProceduralHeaderFileFromToolOutput sender,
ExecutionContext context,
TokenizedString outputPath,
ICommandLineTool tool)
{
var toolProject = (tool as Bam.Core.Module).MetaData as VSSolutionBuilder.VSProject;
var toolConfig = toolProject.GetConfiguration(tool as Bam.Core.Module);
var output = outputPath.Parse();
var commands = new Bam.Core.StringArray();
commands.Add(System.String.Format("IF NOT EXIST {0} MKDIR {0}", System.IO.Path.GetDirectoryName(output)));
commands.Add(System.String.Format("{0} > {1}", CommandLineProcessor.Processor.StringifyTool(tool), output));
toolConfig.AddPostBuildCommands(commands);
// alias the tool's project so that inter-project dependencies can be set up
sender.MetaData = toolProject;
}
Add(
string key,
TokenizedString value)
{
if (key.StartsWith(TokenizedString.TokenPrefix) || key.EndsWith(TokenizedString.TokenSuffix))
{
throw new Exception("Invalid macro key: {0}", key);
}
if (null == value)
{
throw new Exception("Cannot assign null to macro '{0}'", key);
}
var tokenizedMacro = System.String.Format("{0}{1}{2}", TokenizedString.TokenPrefix, key, TokenizedString.TokenSuffix);
if (value.RefersToMacro(tokenizedMacro))
{
throw new Exception("Circular reference; cannnot assign macro '{0}' when it is referred to in TokenizedString or one of it's positional strings", key);
}
this.DictInternal[FormattedKey(key)] = value;
}
IProceduralHeaderFromToolOutputPolicy.HeaderFromToolOutput(
ProceduralHeaderFileFromToolOutput sender,
ExecutionContext context,
TokenizedString outputPath,
ICommandLineTool tool)
{
var meta = new MakeFileBuilder.MakeFileMeta(sender);
var rule = meta.AddRule();
rule.AddTarget(outputPath);
rule.AddPrerequisite(tool.Executable);
var args = new Bam.Core.StringArray();
args.Add("$< > $@");
rule.AddShellCommand(args.ToString(' '));
var outputDir = System.IO.Path.GetDirectoryName(outputPath.Parse());
meta.CommonMetaData.AddDirectory(outputDir);
}
public object TokenizeString(string str)
{
if (str == null)
{
return(null);
}
var p = new StringPointer(str);
var tokens = new TokenizedString();
var sb = new StringBuilder();
while (!p.EOF)
{
if (p.Skip("$("))
{
var start = p.Position;
if (p.SkipUntilOnOf(')', ','))
{
var variableName = p.Extract(start, p.Position);
int width = 0;
if (p[0] == ',')
{
p++;
int multplier = 1;
if (p[0] == '-')
{
multplier = -1;
p++;
}
p.ParseInt(out width);
width *= multplier;
}
p++;
Func <string> resolver;
if (_variables.TryGetValue(variableName, out resolver))
{
if (sb.Length > 0)
{
tokens.Add(new LiteralToken(sb.ToString()));
sb.Length = 0;
}
tokens.Add(new VariableToken(variableName, width, resolver));
}
else
{
sb.AppendFormat("$({0})", variableName);
}
}
else
{
sb.Append("$(");
}
}
else
{
sb.Append(p[0]);
p++;
}
}
if (sb.Length > 0)
{
tokens.Add(new LiteralToken(sb.ToString()));
}
return(tokens);
}
DeferredUntilBuild(
TokenizedString outputPath)
{
return(new ExecuteReasoning(EReason.DeferredEvaluation, outputPath));
}
/// <summary>
/// Calculates similarity score which is between 0.0 and 1.0
/// </summary>
/// <param name="source"></param>
/// <param name="target"></param>
/// <returns></returns>
public double GetScore(TokenizedString source, TokenizedString target)
{
return _distance.Score(new Token(source.Original), new Token(target.Original));
}
InputFileNewer(
TokenizedString outputPath,
TokenizedString inputPath)
{
return(new ExecuteReasoning(EReason.InputFileIsNewer, outputPath, inputPath));
}
private TokenUnitVector GetTokenUnitVector(TokenizedString str)
{
TokenUnitVector vector;
if (!_vectorCache.TryGetValue(str, out vector))
{
vector = new TokenUnitVector(str, _tokenStatistics);
_vectorCache[str] = vector;
}
return vector;
}
FileDoesNotExist(
TokenizedString path)
{
return(new ExecuteReasoning(EReason.FileDoesNotExist, path));
}
AddVerbatim(
string key,
string value)
{
this.Add(key, TokenizedString.CreateVerbatim(value));
}
InputFileNewer(
TokenizedString outputPath,
TokenizedString inputPath)
{
return new ExecuteReasoning(EReason.InputFileIsNewer, outputPath, inputPath);
}
FileDoesNotExist(
TokenizedString path)
{
return new ExecuteReasoning(EReason.FileDoesNotExist, path);
}
Add(
string key,
string value)
{
this.Add(key, TokenizedString.Create(value, null));
}
