protected int OnCacheHitLocked(ICompiler comp, DataHash hc, CacheManifest hm) { CopyStdio(comp, hc); CopyOutputFiles(comp, hc); // we dont need the lock now, it is highly unlikley someone else will // modify these files Unlock(CacheLockType.Read); var duration = comp.Age; var tstat = Task.Run(() => { Stats.LockStatsCall(() => { Stats.CacheHits++; if (hm.Duration < duration.TotalMilliseconds) { // this cached result was slow. record a stat. Stats.SlowHitCount++; Logging.Emit("slow cache hit {0}ms", (int)duration.TotalMilliseconds); } else { Logging.Emit("fast cache hit {0}ms", (int)duration.TotalMilliseconds); } }); }); tstat.Wait(); return(0); }
protected virtual void SaveOutputsLocked(CacheManifest m, ICompiler c) { outputCache.AddFile(m.SessionHash, c.ObjectTarget, F_Object); if (c.GeneratePdb) { var pdbhash = hasher.DigestBinaryFile(c.PdbFile); m.PdbHash = pdbhash.Hash; outputCache.AddFile(m.SessionHash, c.PdbFile, F_Pdb); Stats.LockStatsCall(() => Stats.CacheSize += new FileInfo(c.PdbFile).Length); } Stats.LockStatsCall(() => Stats.CacheObjects++); Stats.LockStatsCall(() => Stats.CacheSize += new FileInfo(c.ObjectTarget).Length); // write manifest var duration = c.Age; m.Duration = (int)duration.TotalMilliseconds; Logging.Emit("cache miss took {0}ms", (int)duration.TotalMilliseconds); using (var manifest = outputCache.OpenFileStream(m.SessionHash, F_Manifest, FileMode.OpenOrCreate, FileAccess.Write)) { m.Serialize(manifest); } }
protected CacheManifest GetCachedManifestLocked(DataHash commonkey) { CacheManifest manifest = null; if (outputCache.ContainsEntry(commonkey.SessionHash, F_Manifest)) { using (var mfs = outputCache.OpenFileStream(commonkey.SessionHash, F_Manifest, FileMode.Open, FileAccess.Read)) { manifest = CacheManifest.Deserialize(mfs); } } return(manifest); }
protected CacheManifest GetCachedManifestLocked(DataHash commonkey) { CacheManifest manifest = null; if (outputCache.ContainsEntry(commonkey.Hash, F_Manifest)) { var mn = outputCache.MakePath(commonkey.Hash, F_Manifest); using (var fs = new FileStream(mn, FileMode.Open)){ manifest = CacheManifest.Deserialize(fs); } } return(manifest); }
protected virtual void DoCacheMiss(ICompiler c, DataHash hc, IEnumerable <string> args, CClashRequest req, List <string> ifiles) { bool good = true; CacheManifest m = null; try { var idirs = c.GetUsedIncludeDirs(ifiles); if (idirs.Count < 1) { throw new InvalidDataException( string.Format("could not find any include folders?! [{0}]", string.Join(" ", args))); } #region process includes folders // save manifest and other things to cache var others = c.GetPotentialIncludeFiles(idirs, ifiles); m = new CacheManifest(); m.Request = req; m.PotentialNewIncludes = others; m.IncludeFiles = new Dictionary <string, string>(); m.TimeStamp = DateTime.Now.ToString("s"); m.SessionHash = hc.SessionHash; #endregion var hashes = GetHashes(ifiles, c.WorkingDirectory); #region check include files foreach (var x in hashes) { if (x.Value.Result == DataHashResult.Ok) { m.IncludeFiles[x.Key] = x.Value.Hash; } else { Logging.Emit("input hash error {0} {1}", x.Key, x.Value.Result); Logging.Miss(hc.SessionHash, x.Value.Result, c.WorkingDirectory, c.SingleSourceFile, x.Key); good = false; m.Disable = true; break; } } #endregion } finally { Unlock(CacheLockType.Read); if (good) { Lock(CacheLockType.ReadWrite); try { if (m != null) { SaveOutputsLocked(m, c); } } finally { Unlock(CacheLockType.ReadWrite); } } } }
/// <summary> /// When this returns, we will hold the output cache mutex. /// </summary> /// <param name="commonkey"></param> /// <param name="manifest"></param> /// <returns></returns> public override bool CheckCache(ICompiler comp, IEnumerable<string> args, DataHash commonkey, out CacheManifest manifest ) { manifest = null; Lock(CacheLockType.Read); manifest = GetCachedManifestLocked(commonkey); if (manifest != null) { #region build missed before if (manifest.Disable) { Logging.Emit("disabled by manifest"); return false; } #region check includes foreach (var f in manifest.PotentialNewIncludes) { if (!FileUtils.FileMissing(f)) { Logging.Emit("detected added include file {0}", f); Logging.Miss(commonkey.Hash, DataHashResult.FileAdded, Directory.GetCurrentDirectory(), comp.SingleSourceFile, f); return false; } } var hashes = GetHashes(manifest.IncludeFiles.Keys); foreach (var h in hashes) { if (h.Value.Result == DataHashResult.Ok) { string mhash; if (manifest.IncludeFiles.TryGetValue(h.Key, out mhash)) { if (mhash != h.Value.Hash) { Logging.Emit("include file hash changed {0}", h.Key); Logging.Miss(commonkey.Hash, DataHashResult.FileChanged, Directory.GetCurrentDirectory(), comp.SingleSourceFile, h.Key); return false; } } else { Logging.Emit("include file added {0}", h.Key); Logging.Miss(commonkey.Hash, DataHashResult.FileAdded, Directory.GetCurrentDirectory(), comp.SingleSourceFile, h.Key); return false; } } else { Logging.Emit("include file hash error {0} {1}", h.Key, h.Value.Result); Logging.Miss(commonkey.Hash, h.Value.Result, Directory.GetCurrentDirectory(), comp.SingleSourceFile, h.Key); return false; } } #endregion #region check pdb if (comp.AttemptPdb) { if (comp.PdbExistsAlready) { var pdbhash = hasher.DigestBinaryFile(comp.PdbFile); if (pdbhash.Hash != manifest.EarlierPdbHash) { outputCache.Remove(commonkey.Hash); Logging.Miss(commonkey.Hash, DataHashResult.FileChanged, commonkey.Hash, comp.PdbFile, ""); return false; } } } #endregion #region check cached data exists foreach (var f in new string[] { F_Manifest, F_Object }) { if (!FileUtils.Exists(outputCache.MakePath(commonkey.Hash, f))) { outputCache.Remove(commonkey.Hash); Logging.Miss(commonkey.Hash, DataHashResult.CacheCorrupt, commonkey.Hash, comp.SingleSourceFile, ""); return false; } } #endregion if (Settings.MissLogEnabled) { Logging.Emit("hit hc={0},dir={1},src={2}", commonkey.Hash, comp.WorkingDirectory, comp.SingleSourceFile); } return true; // cache hit, all includes match and no new files added #endregion } Logging.Miss(commonkey.Hash, DataHashResult.NoPreviousBuild, comp.WorkingDirectory, comp.SingleSourceFile, ""); return false; }
protected virtual void DoCacheMiss(ICompiler c, DataHash hc, IEnumerable<string> args, CacheManifest m, List<string> ifiles) { bool good = true; try { var idirs = c.GetUsedIncludeDirs(ifiles); if (idirs.Count < 1) { throw new InvalidDataException( string.Format("could not find any include folders?! [{0}]", string.Join(" ", args))); } #region process includes folders // save manifest and other things to cache var others = c.GetPotentialIncludeFiles(idirs, ifiles); m = new CacheManifest(); m.PotentialNewIncludes = others; m.IncludeFiles = new Dictionary<string, string>(); m.TimeStamp = DateTime.Now.ToString("s"); m.CommonHash = hc.Hash; #endregion var hashes = GetHashes(ifiles); #region check include files foreach (var x in hashes) { if (x.Value.Result == DataHashResult.Ok) { m.IncludeFiles[x.Key] = x.Value.Hash; } else { Logging.Emit("input hash error {0} {1}", x.Key, x.Value.Result); Logging.Miss(hc.Hash, x.Value.Result, c.WorkingDirectory, c.SingleSourceFile, x.Key); good = false; m.Disable = true; break; } } #endregion } finally { Unlock(CacheLockType.Read); if (good) { Lock(CacheLockType.ReadWrite); try { SaveOutputsLocked(m, c); } finally { Unlock(CacheLockType.ReadWrite); } } } }
protected override int OnCacheMissLocked(ICompiler comp, DataHash hc, IEnumerable<string> args, CacheManifest m) { throw new NotImplementedException(); }
protected abstract int OnCacheMissLocked(ICompiler comp, DataHash hc, IEnumerable <string> args, CacheManifest m);
public override bool CheckCache(ICompiler comp, IEnumerable <string> args, DataHash commonkey, out CacheManifest manifest) { throw new NotImplementedException(); }
public bool CheckCache(ICompiler comp, IEnumerable<string> args, DataHash commonkey, out CacheManifest manifest) { manifest = null; return false; }
protected override int OnCacheMissLocked(ICompiler comp, DataHash hc, IEnumerable <string> args, CacheManifest m) { Logging.Emit("cache miss"); outputCache.EnsureKey(hc.Hash); var stderrfile = outputCache.MakePath(hc.Hash, F_Stderr); var stdoutfile = outputCache.MakePath(hc.Hash, F_Stdout); var ifiles = new List <string>(); Stats.LockStatsCall(() => Stats.CacheMisses++); int rv = Compile(comp, args, stderrfile, stdoutfile, ifiles); // we still hold the cache lock, create the manifest asap or give up now! if (rv != 0) { Unlock(CacheLockType.Read); } else { // this unlocks for us try { DoCacheMiss(comp, hc, args, m, ifiles); } catch (CClashWarningException) { return(CompileOnly(comp, args)); } } return(rv); }
protected abstract int OnCacheMissLocked(ICompiler comp, DataHash hc, IEnumerable<string> args, CacheManifest m);
public abstract bool CheckCache(ICompiler comp, IEnumerable<string> args, DataHash commonkey, out CacheManifest manifest);
protected int OnCacheHitLocked(ICompiler comp, DataHash hc, CacheManifest hm) { CopyStdio(comp, hc); CopyOutputFiles(comp, hc); // we dont need the lock now, it is highly unlikley someone else will // modify these files Unlock(CacheLockType.Read); var duration = comp.Age; var tstat = Task.Run(() => { Stats.LockStatsCall(() => { Stats.CacheHits++; if (hm.Duration < duration.TotalMilliseconds) { // this cached result was slow. record a stat. Stats.SlowHitCount++; Logging.Emit("slow cache hit {0}ms", (int)duration.TotalMilliseconds); } else { Logging.Emit("fast cache hit {0}ms", (int)duration.TotalMilliseconds); } }); }); tstat.Wait(); return 0; }
/// <summary> /// When this returns, we will hold the output cache mutex. /// </summary> /// <param name="commonkey"></param> /// <param name="manifest"></param> /// <returns></returns> public override bool CheckCache(ICompiler comp, IEnumerable <string> args, DataHash commonkey, out CacheManifest manifest) { manifest = null; Lock(CacheLockType.Read); manifest = GetCachedManifestLocked(commonkey); if (manifest != null) { #region build missed before if (manifest.Disable) { Logging.Emit("disabled by manifest"); return(false); } #region check includes foreach (var f in manifest.PotentialNewIncludes) { if (!FileUtils.FileMissing(f)) { Logging.Emit("detected added include file {0}", f); Logging.Miss(commonkey.SessionHash, DataHashResult.FileAdded, Directory.GetCurrentDirectory(), comp.SingleSourceFile, f); return(false); } } var files = new List <string>(); files.AddRange(manifest.IncludeFiles.Keys); var hashes = GetHashes(files, comp.WorkingDirectory); foreach (var h in hashes) { if (h.Value.Result == DataHashResult.Ok) { string mhash; if (manifest.IncludeFiles.TryGetValue(h.Key, out mhash)) { if (mhash != h.Value.Hash) { Logging.Emit("include file hash changed {0}", h.Key); Logging.Miss(commonkey.SessionHash, DataHashResult.FileChanged, Directory.GetCurrentDirectory(), comp.SingleSourceFile, h.Key); return(false); } } else { Logging.Emit("include file added {0}", h.Key); Logging.Miss(commonkey.SessionHash, DataHashResult.FileAdded, Directory.GetCurrentDirectory(), comp.SingleSourceFile, h.Key); return(false); } } else { Logging.Emit("include file hash error {0} {1}", h.Key, h.Value.Result); Logging.Miss(commonkey.SessionHash, h.Value.Result, Directory.GetCurrentDirectory(), comp.SingleSourceFile, h.Key); return(false); } } #endregion #region check pdb if (comp.AttemptPdb) { if (comp.PdbExistsAlready) { var pdbhash = hasher.DigestBinaryFile(comp.PdbFile); if (pdbhash.Hash != manifest.EarlierPdbHash) { outputCache.Remove(commonkey.Hash); Logging.Miss(commonkey.Hash, DataHashResult.FileChanged, commonkey.Hash, comp.PdbFile, ""); return(false); } } } #endregion #region check cached data exists foreach (var f in new string[] { F_Manifest, F_Object }) { if (!outputCache.ContainsEntry(commonkey.SessionHash, f)) { outputCache.Remove(commonkey.SessionHash); Logging.Miss(commonkey.SessionHash, DataHashResult.CacheCorrupt, commonkey.SessionHash, comp.SingleSourceFile, ""); return(false); } } #endregion if (Settings.MissLogEnabled) { Logging.Emit("hit hc={0},dir={1},src={2}", commonkey.Hash, comp.WorkingDirectory, comp.SingleSourceFile); } return(true); // cache hit, all includes match and no new files added #endregion } Logging.Miss(commonkey.Hash, DataHashResult.NoPreviousBuild, comp.WorkingDirectory, comp.SingleSourceFile, ""); return(false); }
protected override int OnCacheMissLocked(ICompiler comp, DataHash hc, IEnumerable <string> args, CacheManifest m) { throw new NotImplementedException(); }
protected virtual void SaveOutputsLocked(CacheManifest m, ICompiler c ) { outputCache.AddFile(m.CommonHash, c.ObjectTarget, F_Object); if (c.GeneratePdb) { var pdbhash = hasher.DigestBinaryFile(c.PdbFile); m.PdbHash = pdbhash.Hash; outputCache.AddFile(m.CommonHash, c.PdbFile, F_Pdb); Stats.LockStatsCall(() => Stats.CacheSize += new FileInfo(c.PdbFile).Length); } Stats.LockStatsCall(() => Stats.CacheObjects++); Stats.LockStatsCall(() => Stats.CacheSize += new FileInfo(c.ObjectTarget).Length); // write manifest var duration = c.Age; m.Duration = (int)duration.TotalMilliseconds; Logging.Emit("cache miss took {0}ms", (int)duration.TotalMilliseconds); var fname = outputCache.MakePath(m.CommonHash, F_Manifest); using (var fs = new FileStream(fname, FileMode.OpenOrCreate, FileAccess.Write)) { m.Serialize(fs); } }
public abstract bool CheckCache(ICompiler comp, IEnumerable <string> args, DataHash commonkey, out CacheManifest manifest);
protected override int OnCacheMissLocked(ICompiler comp, DataHash hc, IEnumerable<string> args, CacheManifest m) { Logging.Emit("cache miss"); outputCache.EnsureKey(hc.Hash); var stderrfile = outputCache.MakePath(hc.Hash, F_Stderr); var stdoutfile = outputCache.MakePath(hc.Hash, F_Stdout); var ifiles = new List<string>(); Stats.LockStatsCall(() => Stats.CacheMisses++); int rv = Compile(comp, args, stderrfile, stdoutfile, ifiles ); // we still hold the cache lock, create the manifest asap or give up now! if (rv != 0) { Unlock(CacheLockType.Read); } else { // this unlocks for us try { DoCacheMiss(comp, hc, args, m, ifiles); } catch (CClashWarningException) { return CompileOnly(comp, args); } } return rv; }
public bool CheckCache(ICompiler comp, IEnumerable <string> args, DataHash commonkey, out CacheManifest manifest) { manifest = null; return(false); }
public override bool CheckCache(ICompiler comp, IEnumerable<string> args, DataHash commonkey, out CacheManifest manifest) { throw new NotImplementedException(); }