/// <summary> /// Calculates a score between 0.0 (completely different) and 1.0 (the same). Two token lists are /// concerted to unit vector weights and Soft TF-IDF algorithm is applied to them /// </summary> /// <param name="source"></param> /// <param name="target"></param> /// <returns></returns> public double GetScore(TokenizedString source, TokenizedString target) { if (_alwaysCompareLongerToShorter && source.Count < target.Count) { var temp = source; source = target; target = temp; } var sVector = GetTokenUnitVector(source); var tVector = GetTokenUnitVector(target); double sim = 0.0; foreach (var sToken in sVector) { if (tVector.Contains(sToken)) { sim += sVector.GetWeight(sToken) * tVector.GetWeight(sToken); } else { // find best matching token var matchToken = tVector .Select(t => new {Token = t, Score = _distance.Score(sToken, t)}) .Aggregate((next, current) => next.Score > current.Score ? next : current); if (matchToken.Score >= _tokenMatchThreshold) { sim += sVector.GetWeight(sToken) * tVector.GetWeight(matchToken.Token) * matchToken.Score; } } } return sim; }
private ExecuteReasoning( EReason reason, TokenizedString outputFilePath = null, TokenizedString inputFilePath = null) { this.Reason = reason; this.OutputFilePath = outputFilePath; this.InputFilePath = inputFilePath; }
Init( Bam.Core.Module parent) { base.Init(parent); this.Macros["OutputName"] = TokenizedString.CreateVerbatim("Cxx_NSISInstaller"); this.SourceFolder <CxxExecutableStripped>(Publisher.StrippedBinaryCollation.Key); }
/// <summary> /// Adds string tokens to the set /// </summary> /// <param name="tokens"></param> public void AddDocument(TokenizedString tokens) { var seenTokens = new HashSet<Token>(); foreach (var token in tokens.Where(token => !seenTokens.Contains(token))) { seenTokens.Add(token); int frequency = 0; _dict.TryGetValue(token, out frequency); _dict[token] = frequency + 1; } NumberOfDucoments++; }
Display() { Log.Info("\nBuildAMation Statistics"); Log.Info("Memory Usage"); Log.Info("Peak working set size : {0:N2}MB", BytesToMegaBytes(System.Diagnostics.Process.GetCurrentProcess().PeakWorkingSet64)); Log.Info("Peak virtual size : {0:N2}MB", BytesToMegaBytes(System.Diagnostics.Process.GetCurrentProcess().PeakVirtualMemorySize64)); Log.Info("GC total memory : {0:N2}MB (after GC, {1:N2}MB)", BytesToMegaBytes(System.GC.GetTotalMemory(false)), BytesToMegaBytes(System.GC.GetTotalMemory(true))); Log.Info("\nObject counts"); Log.Info("Tokenized strings : {0} ({1} unshared)", TokenizedString.Count, TokenizedString.UnsharedCount); TokenizedString.DumpCache(); Log.Info("Modules : {0}", Module.Count); TimingProfileUtilities.DumpProfiles(); }
IProceduralHeaderFromToolOutputPolicy.HeaderFromToolOutput( ProceduralHeaderFileFromToolOutput sender, ExecutionContext context, TokenizedString outputPath, ICommandLineTool tool) { var toolPath = tool.Executable.ToString(); CommandLineProcessor.Processor.Execute(context, toolPath); var destPath = outputPath.ToString(); var destDir = System.IO.Path.GetDirectoryName(destPath); Bam.Core.IOWrapper.CreateDirectoryIfNotExists(destDir); var tempPath = Bam.Core.IOWrapper.CreateTemporaryFile(); using (System.IO.TextWriter writeFile = new System.IO.StreamWriter(tempPath)) { writeFile.Write(context.OutputStringBuilder.ToString()); } var moveFile = true; if (System.IO.File.Exists(destPath)) { // compare contents using (System.IO.TextReader existingFile = new System.IO.StreamReader(destPath)) { var contents = existingFile.ReadToEnd(); var contentsL = contents.Length; var oldL = context.OutputStringBuilder.ToString().Length; if (contents.Equals(context.OutputStringBuilder.ToString())) { moveFile = false; } } } if (moveFile) { Bam.Core.Log.Info("Written procedurally generated header : {0}, from the output of {1}", destPath, toolPath); System.IO.File.Delete(destPath); System.IO.File.Move(tempPath, destPath); } else { Bam.Core.Log.Info("{0} contents have not changed", destPath); System.IO.File.Delete(tempPath); } context.OutputStringBuilder.Clear(); }
Add( string key, TokenizedString value) { if (key.StartsWith(TokenizedString.TokenPrefix) || key.EndsWith(TokenizedString.TokenSuffix)) { throw new Exception("Invalid macro key: {0}", key); } if (null == value) { throw new Exception("Cannot assign null to macro '{0}'", key); } this.DictInternal[FormattedKey(key)] = value; }
Init( Bam.Core.Module parent) { base.Init(parent); this.Macros["OutputName"] = TokenizedString.CreateVerbatim("C_SymbolsTarBall"); this.SourceFolder <CExecutableDebugSymbols>(Publisher.DebugSymbolCollation.Key); this.PrivatePatch(settings => { var tarSettings = settings as Installer.ITarBallSettings; tarSettings.CompressionType = Installer.ETarCompressionType.gzip; }); }
Init( Bam.Core.Module parent) { base.Init(parent); this.PrivatePatch(settings => { var dmgSettings = settings as Installer.IDiskImageSettings; dmgSettings.ImageSize = "128m"; }); this.Macros["OutputName"] = TokenizedString.CreateVerbatim("Cxx_DiskImageInstaller"); this.SourceFolder <CxxExecutableStripped>(Publisher.StrippedBinaryCollation.Key); }
AssignLinkTarget( TokenizedString path = null) { #if __MonoCS__ if (path == null) { var symlink = new Mono.Unix.UnixSymbolicLinkInfo(this.SourcePath.Parse()); this.Macros["LinkTarget"] = Bam.Core.TokenizedString.CreateVerbatim(symlink.ContentsPath); } else { this.Macros["LinkTarget"] = path; } #else throw new System.NotSupportedException("Unable to get symbolic link target on Windows"); #endif }
Init( Bam.Core.Module parent) { base.Init(parent); this.Macros["OutputName"] = TokenizedString.CreateVerbatim("C_TarBallInstaller"); this.SourceFolder <CExecutableStripped>(Publisher.StrippedBinaryCollation.Key); this.PrivatePatch(settings => { var tarSettings = settings as Installer.ITarBallSettings; tarSettings.CompressionType = Installer.ETarCompressionType.gzip; if (this.BuildEnvironment.Platform.Includes(Bam.Core.EPlatform.Linux)) { tarSettings.TransformRegEx = "'s,^.,toplevelfolder,'"; } }); }
IProceduralHeaderFromToolOutputPolicy.HeaderFromToolOutput( ProceduralHeaderFileFromToolOutput sender, ExecutionContext context, TokenizedString outputPath, ICommandLineTool tool) { var toolTarget = (tool as Bam.Core.Module).MetaData as XcodeBuilder.Target; var toolConfiguration = toolTarget.GetConfiguration(tool as Bam.Core.Module); var output = outputPath.Parse(); var commands = new Bam.Core.StringArray(); commands.Add(System.String.Format("[[ ! -d {0} ]] && mkdir -p {0}", System.IO.Path.GetDirectoryName(output))); commands.Add(System.String.Format("{0} > {1}", CommandLineProcessor.Processor.StringifyTool(tool), output)); toolTarget.AddPostBuildCommands(commands, toolConfiguration); // alias the tool's target so that inter-target dependencies can be set up sender.MetaData = toolTarget; }
IProceduralHeaderFromToolOutputPolicy.HeaderFromToolOutput( ProceduralHeaderFileFromToolOutput sender, ExecutionContext context, TokenizedString outputPath, ICommandLineTool tool) { var toolProject = (tool as Bam.Core.Module).MetaData as VSSolutionBuilder.VSProject; var toolConfig = toolProject.GetConfiguration(tool as Bam.Core.Module); var output = outputPath.Parse(); var commands = new Bam.Core.StringArray(); commands.Add(System.String.Format("IF NOT EXIST {0} MKDIR {0}", System.IO.Path.GetDirectoryName(output))); commands.Add(System.String.Format("{0} > {1}", CommandLineProcessor.Processor.StringifyTool(tool), output)); toolConfig.AddPostBuildCommands(commands); // alias the tool's project so that inter-project dependencies can be set up sender.MetaData = toolProject; }
Add( string key, TokenizedString value) { if (key.StartsWith(TokenizedString.TokenPrefix) || key.EndsWith(TokenizedString.TokenSuffix)) { throw new Exception("Invalid macro key: {0}", key); } if (null == value) { throw new Exception("Cannot assign null to macro '{0}'", key); } var tokenizedMacro = System.String.Format("{0}{1}{2}", TokenizedString.TokenPrefix, key, TokenizedString.TokenSuffix); if (value.RefersToMacro(tokenizedMacro)) { throw new Exception("Circular reference; cannnot assign macro '{0}' when it is referred to in TokenizedString or one of it's positional strings", key); } this.DictInternal[FormattedKey(key)] = value; }
IProceduralHeaderFromToolOutputPolicy.HeaderFromToolOutput( ProceduralHeaderFileFromToolOutput sender, ExecutionContext context, TokenizedString outputPath, ICommandLineTool tool) { var meta = new MakeFileBuilder.MakeFileMeta(sender); var rule = meta.AddRule(); rule.AddTarget(outputPath); rule.AddPrerequisite(tool.Executable); var args = new Bam.Core.StringArray(); args.Add("$< > $@"); rule.AddShellCommand(args.ToString(' ')); var outputDir = System.IO.Path.GetDirectoryName(outputPath.Parse()); meta.CommonMetaData.AddDirectory(outputDir); }
public object TokenizeString(string str) { if (str == null) { return(null); } var p = new StringPointer(str); var tokens = new TokenizedString(); var sb = new StringBuilder(); while (!p.EOF) { if (p.Skip("$(")) { var start = p.Position; if (p.SkipUntilOnOf(')', ',')) { var variableName = p.Extract(start, p.Position); int width = 0; if (p[0] == ',') { p++; int multplier = 1; if (p[0] == '-') { multplier = -1; p++; } p.ParseInt(out width); width *= multplier; } p++; Func <string> resolver; if (_variables.TryGetValue(variableName, out resolver)) { if (sb.Length > 0) { tokens.Add(new LiteralToken(sb.ToString())); sb.Length = 0; } tokens.Add(new VariableToken(variableName, width, resolver)); } else { sb.AppendFormat("$({0})", variableName); } } else { sb.Append("$("); } } else { sb.Append(p[0]); p++; } } if (sb.Length > 0) { tokens.Add(new LiteralToken(sb.ToString())); } return(tokens); }
DeferredUntilBuild( TokenizedString outputPath) { return(new ExecuteReasoning(EReason.DeferredEvaluation, outputPath)); }
/// <summary> /// Calculates similarity score which is between 0.0 and 1.0 /// </summary> /// <param name="source"></param> /// <param name="target"></param> /// <returns></returns> public double GetScore(TokenizedString source, TokenizedString target) { return _distance.Score(new Token(source.Original), new Token(target.Original)); }
InputFileNewer( TokenizedString outputPath, TokenizedString inputPath) { return(new ExecuteReasoning(EReason.InputFileIsNewer, outputPath, inputPath)); }
private TokenUnitVector GetTokenUnitVector(TokenizedString str) { TokenUnitVector vector; if (!_vectorCache.TryGetValue(str, out vector)) { vector = new TokenUnitVector(str, _tokenStatistics); _vectorCache[str] = vector; } return vector; }
FileDoesNotExist( TokenizedString path) { return(new ExecuteReasoning(EReason.FileDoesNotExist, path)); }
AddVerbatim( string key, string value) { this.Add(key, TokenizedString.CreateVerbatim(value)); }
InputFileNewer( TokenizedString outputPath, TokenizedString inputPath) { return new ExecuteReasoning(EReason.InputFileIsNewer, outputPath, inputPath); }
FileDoesNotExist( TokenizedString path) { return new ExecuteReasoning(EReason.FileDoesNotExist, path); }
Add( string key, string value) { this.Add(key, TokenizedString.Create(value, null)); }