protected override int OnCacheMissLocked(ICompiler comp, DataHash hc, IEnumerable <string> args, CClashRequest req) { Logging.Emit("cache miss"); outputCache.EnsureKey(hc.Hash); var ifiles = new List <string>(); Stats.LockStatsCall(() => Stats.CacheMisses++); using (var stderr = outputCache.OpenFileStream(hc.Hash, F_Stderr, FileMode.OpenOrCreate, FileAccess.Write)) using (var stdout = outputCache.OpenFileStream(hc.Hash, F_Stdout, FileMode.OpenOrCreate, FileAccess.Write)) { int rv = Compile(comp, args, stderr, stdout, ifiles); // we still hold the cache lock, create the manifest asap or give up now! if (rv != 0) { Unlock(CacheLockType.Read); } else { // this unlocks for us try { DoCacheMiss(comp, hc, args, req, ifiles); } catch (CClashWarningException) { return(CompileOnly(comp, args)); } } return(rv); } }
protected override int OnCacheMissLocked(ICompiler comp, DataHash hc, IEnumerable <string> args, CacheManifest m) { Logging.Emit("cache miss"); outputCache.EnsureKey(hc.Hash); var stderrfile = outputCache.MakePath(hc.Hash, F_Stderr); var stdoutfile = outputCache.MakePath(hc.Hash, F_Stdout); var ifiles = new List <string>(); Stats.LockStatsCall(() => Stats.CacheMisses++); int rv = Compile(comp, args, stderrfile, stdoutfile, ifiles); // we still hold the cache lock, create the manifest asap or give up now! if (rv != 0) { Unlock(CacheLockType.Read); } else { // this unlocks for us try { DoCacheMiss(comp, hc, args, m, ifiles); } catch (CClashWarningException) { return(CompileOnly(comp, args)); } } return(rv); }
protected int OnCacheHitLocked(ICompiler comp, DataHash hc, CacheManifest hm) { CopyStdio(comp, hc); CopyOutputFiles(comp, hc); // we dont need the lock now, it is highly unlikley someone else will // modify these files Unlock(CacheLockType.Read); var duration = comp.Age; var tstat = Task.Run(() => { Stats.LockStatsCall(() => { Stats.CacheHits++; if (hm.Duration < duration.TotalMilliseconds) { // this cached result was slow. record a stat. Stats.SlowHitCount++; Logging.Emit("slow cache hit {0}ms", (int)duration.TotalMilliseconds); } else { Logging.Emit("fast cache hit {0}ms", (int)duration.TotalMilliseconds); } }); }); tstat.Wait(); return(0); }
void CopyStdio(ICompiler comp, DataHash hc) { var stderrfile = outputCache.MakePath(hc.Hash, F_Stderr); var stdoutfile = outputCache.MakePath(hc.Hash, F_Stdout); comp.StdOutputCallback(File.ReadAllText(stdoutfile)); comp.StdErrorCallback(File.ReadAllText(stderrfile)); }
public void CopyStdio(DataHash hc) { var stderrfile = Cache.OutputCache.MakePath(hc.Hash, CompilerCacheBase.F_Stderr); var stdoutfile = Cache.OutputCache.MakePath(hc.Hash, CompilerCacheBase.F_Stdout); StdOutput.Clear(); StdError.Clear(); StdOutput.Append(File.ReadAllText(stdoutfile)); StdError.Append(File.ReadAllText(stderrfile)); }
void CopyStdio(ICompiler comp, DataHash hc) { using (var stderr = outputCache.OpenFileStream(hc.SessionHash, F_Stderr, FileMode.Open, FileAccess.Read)) using (var reader = new StreamReader(stderr)) comp.StdErrorCallback(reader.ReadToEnd()); using (var stdout = outputCache.OpenFileStream(hc.SessionHash, F_Stdout, FileMode.Open, FileAccess.Read)) using (var reader = new StreamReader(stdout)) comp.StdOutputCallback(reader.ReadToEnd()); }
public void CopyOutputFiles(DataHash hc) { try { Cache.CopyFile(Cache.OutputCache.MakePath(hc.Hash, CompilerCacheBase.F_Object), Compiler.ObjectTarget); if (Compiler.GeneratePdb) Cache.CopyFile(Cache.OutputCache.MakePath(hc.Hash, CompilerCacheBase.F_Pdb), Compiler.PdbFile); } catch (Exception e) { Logging.Error("{0}", e); throw; } }
public DataHash DigestStream(Stream s) { MD5CryptoServiceProvider md5 = new MD5CryptoServiceProvider(); var rv = new DataHash() { InputName = "stream", Result = DataHashResult.Ok, Hash = new SoapHexBinary(md5.ComputeHash(s)).ToString() }; return(rv); }
public DataHash DigestString(string input) { MD5CryptoServiceProvider md5 = new MD5CryptoServiceProvider(); var rv = new DataHash() { InputName = "string", Result = DataHashResult.Ok, Hash = new SoapHexBinary( md5.ComputeHash( System.Text.Encoding.Unicode.GetBytes( input ) ) ).ToString() }; return rv; }
public DataHash DigestString(string input) { MD5CryptoServiceProvider md5 = new MD5CryptoServiceProvider(); var rv = new DataHash() { InputName = "string", Result = DataHashResult.Ok, Hash = new SoapHexBinary(md5.ComputeHash(System.Text.Encoding.Unicode.GetBytes(input))).ToString() }; return(rv); }
public void CopyOutputFiles(DataHash hc) { try { Cache.CopyFile(Cache.OutputCache.MakePath(hc.Hash, CompilerCacheBase.F_Object), Compiler.ObjectTarget); if (Compiler.GeneratePdb) { Cache.CopyFile(Cache.OutputCache.MakePath(hc.Hash, CompilerCacheBase.F_Pdb), Compiler.PdbFile); } } catch (Exception e) { Logging.Error("{0}", e); throw; } }
protected CacheManifest GetCachedManifestLocked(DataHash commonkey) { CacheManifest manifest = null; if (outputCache.ContainsEntry(commonkey.Hash, F_Manifest)) { var mn = outputCache.MakePath(commonkey.Hash, F_Manifest); using (var fs = new FileStream(mn, FileMode.Open)){ manifest = CacheManifest.Deserialize(fs); } } return(manifest); }
protected CacheManifest GetCachedManifestLocked(DataHash commonkey) { CacheManifest manifest = null; if (outputCache.ContainsEntry(commonkey.SessionHash, F_Manifest)) { using (var mfs = outputCache.OpenFileStream(commonkey.SessionHash, F_Manifest, FileMode.Open, FileAccess.Read)) { manifest = CacheManifest.Deserialize(mfs); } } return(manifest); }
public DataHash DeriveHashKey(ICompiler comp, IEnumerable <string> args) { Logging.Emit("compiler is {0}", comp.CompilerExe); var tmp = DigestCompiler(comp.CompilerExe); var comphash = new DataHash() { InputName = tmp.InputName, Result = tmp.Result, Hash = tmp.Hash, }; if (comphash.Result == DataHashResult.Ok) { var buf = new StringWriter(); DataHash session; buf.WriteLine(CacheInfo.CacheFormat); // our compiler and folder buf.WriteLine(comphash.Hash); buf.WriteLine(comp.WorkingDirectory); buf.WriteLine(comp.SingleSourceFile); // important env vars foreach (var ename in new string[] { "INCLUDE", "LIB" }) { string ev = null; if (comp.EnvironmentVariables.TryGetValue(ename, out ev)) { if (!string.IsNullOrEmpty(ev)) { buf.WriteLine(ev); } } } // now all the command line options foreach (var a in args) { buf.WriteLine(a); } session = hasher.DigestString(buf.ToString()); comphash.SessionHash = session.Hash; comphash.Hash = comphash.SessionHash; } return(comphash); }
void CopyOutputFiles(ICompiler comp, DataHash hc) { try { CopyFile(outputCache.MakePath(hc.Hash, F_Object), comp.ObjectTarget); if (comp.GeneratePdb && comp.AttemptPdb && comp.PdbFile != null) { CopyFile(outputCache.MakePath(hc.Hash, F_Pdb), comp.PdbFile); } } catch (Exception e) { Logging.Error("{0}", e); throw; } }
void CopyOutputFiles(ICompiler comp, DataHash hc) { try { using (var obj = outputCache.OpenFileStream(hc.SessionHash, F_Object, FileMode.Open, FileAccess.Read)) WriteFile(obj, comp.ObjectTarget); if (outputCache.ContainsEntry(hc.SessionHash, F_Pdb)) { using (var pdb = outputCache.OpenFileStream(hc.SessionHash, F_Pdb, FileMode.Open, FileAccess.Read)) { WriteFile(pdb, comp.PdbFile); } } } catch (Exception e) { Logging.Error("{0}", e); throw; } }
protected virtual void DoCacheMiss(ICompiler c, DataHash hc, IEnumerable <string> args, CClashRequest req, List <string> ifiles) { bool good = true; CacheManifest m = null; try { var idirs = c.GetUsedIncludeDirs(ifiles); if (idirs.Count < 1) { throw new InvalidDataException( string.Format("could not find any include folders?! [{0}]", string.Join(" ", args))); } #region process includes folders // save manifest and other things to cache var others = c.GetPotentialIncludeFiles(idirs, ifiles); m = new CacheManifest(); m.Request = req; m.PotentialNewIncludes = others; m.IncludeFiles = new Dictionary <string, string>(); m.TimeStamp = DateTime.Now.ToString("s"); m.SessionHash = hc.SessionHash; #endregion var hashes = GetHashes(ifiles, c.WorkingDirectory); #region check include files foreach (var x in hashes) { if (x.Value.Result == DataHashResult.Ok) { m.IncludeFiles[x.Key] = x.Value.Hash; } else { Logging.Emit("input hash error {0} {1}", x.Key, x.Value.Result); Logging.Miss(hc.SessionHash, x.Value.Result, c.WorkingDirectory, c.SingleSourceFile, x.Key); good = false; m.Disable = true; break; } } #endregion } finally { Unlock(CacheLockType.Read); if (good) { Lock(CacheLockType.ReadWrite); try { if (m != null) { SaveOutputsLocked(m, c); } } finally { Unlock(CacheLockType.ReadWrite); } } } }
protected int OnCacheHitLocked(ICompiler comp, DataHash hc, CacheManifest hm) { CopyStdio(comp, hc); CopyOutputFiles(comp, hc); // we dont need the lock now, it is highly unlikley someone else will // modify these files Unlock(CacheLockType.Read); var duration = comp.Age; var tstat = Task.Run(() => { Stats.LockStatsCall(() => { Stats.CacheHits++; if (hm.Duration < duration.TotalMilliseconds) { // this cached result was slow. record a stat. Stats.SlowHitCount++; Logging.Emit("slow cache hit {0}ms", (int)duration.TotalMilliseconds); } else { Logging.Emit("fast cache hit {0}ms", (int)duration.TotalMilliseconds); } }); }); tstat.Wait(); return 0; }
public abstract bool CheckCache(ICompiler comp, IEnumerable<string> args, DataHash commonkey, out CacheManifest manifest);
DataHash DigestFile(MD5 provider, string filepath, Regex findDateTime) { var rv = new DataHash() { Result = DataHashResult.FileNotFound, InputName = filepath, }; if (!FileUtils.Exists(filepath)) { return(rv); } provider.Initialize(); var fs = new FileStream(filepath, FileMode.Open, FileAccess.Read, FileShare.Read); using (var bs = new BufferedStream(fs)) { Logging.Emit("digest {0}", filepath); rv.Hash = new SoapHexBinary(provider.ComputeHash(bs)).ToString(); rv.Result = DataHashResult.Ok; if (findDateTime != null) { // check include cache for this file if (includeCache.ContainsEntry(rv.Hash, F_NotDateTime)) { return(rv); } if (includeCache.ContainsEntry(rv.Hash, F_HasDateTime)) { rv.Result = DataHashResult.ContainsTimeOrDate; return(rv); } bs.Seek(0, SeekOrigin.Begin); using (var ts = new StreamReader(bs)) { string line = null; do { line = ts.ReadLine(); if (line == null) { includeCache.WaitOne(); try { includeCache.AddTextFileContent(rv.Hash, F_NotDateTime, ""); } finally { includeCache.ReleaseMutex(); } break; } if (findDateTime.IsMatch(line)) { rv.Result = DataHashResult.ContainsTimeOrDate; includeCache.WaitOne(); try { includeCache.AddTextFileContent(rv.Hash, F_HasDateTime, ""); } finally { includeCache.ReleaseMutex(); } break; } } while (true); } } } rv.Result = DataHashResult.Ok; return(rv); }
public DataHash DigestStream(Stream s) { MD5CryptoServiceProvider md5 = new MD5CryptoServiceProvider(); var rv = new DataHash() { InputName = "stream", Result = DataHashResult.Ok, Hash = new SoapHexBinary( md5.ComputeHash(s) ).ToString() }; return rv; }
protected CacheManifest GetCachedManifestLocked(DataHash commonkey) { CacheManifest manifest = null; if (outputCache.ContainsEntry(commonkey.Hash, F_Manifest)) { var mn = outputCache.MakePath(commonkey.Hash, F_Manifest); using ( var fs = new FileStream( mn, FileMode.Open ) ){ manifest = CacheManifest.Deserialize(fs); } } return manifest; }
protected override int OnCacheMissLocked(ICompiler comp, DataHash hc, IEnumerable <string> args, CacheManifest m) { throw new NotImplementedException(); }
protected override int OnCacheMissLocked(ICompiler comp, DataHash hc, IEnumerable<string> args, CacheManifest m) { Logging.Emit("cache miss"); outputCache.EnsureKey(hc.Hash); var stderrfile = outputCache.MakePath(hc.Hash, F_Stderr); var stdoutfile = outputCache.MakePath(hc.Hash, F_Stdout); var ifiles = new List<string>(); Stats.LockStatsCall(() => Stats.CacheMisses++); int rv = Compile(comp, args, stderrfile, stdoutfile, ifiles ); // we still hold the cache lock, create the manifest asap or give up now! if (rv != 0) { Unlock(CacheLockType.Read); } else { // this unlocks for us try { DoCacheMiss(comp, hc, args, m, ifiles); } catch (CClashWarningException) { return CompileOnly(comp, args); } } return rv; }
public bool CheckCache(ICompiler comp, IEnumerable<string> args, DataHash commonkey, out CacheManifest manifest) { manifest = null; return false; }
public override bool CheckCache(ICompiler comp, IEnumerable <string> args, DataHash commonkey, out CacheManifest manifest) { throw new NotImplementedException(); }
protected override int OnCacheMissLocked(ICompiler comp, DataHash hc, IEnumerable<string> args, CacheManifest m) { throw new NotImplementedException(); }
protected abstract int OnCacheMissLocked(ICompiler comp, DataHash hc, IEnumerable <string> args, CacheManifest m);
protected abstract int OnCacheMissLocked(ICompiler comp, DataHash hc, IEnumerable <string> args, CClashRequest req);
/// <summary> /// When this returns, we will hold the output cache mutex. /// </summary> /// <param name="commonkey"></param> /// <param name="manifest"></param> /// <returns></returns> public override bool CheckCache(ICompiler comp, IEnumerable <string> args, DataHash commonkey, out CacheManifest manifest) { manifest = null; Lock(CacheLockType.Read); manifest = GetCachedManifestLocked(commonkey); if (manifest != null) { #region build missed before if (manifest.Disable) { Logging.Emit("disabled by manifest"); return(false); } #region check includes foreach (var f in manifest.PotentialNewIncludes) { if (!FileUtils.FileMissing(f)) { Logging.Emit("detected added include file {0}", f); Logging.Miss(commonkey.SessionHash, DataHashResult.FileAdded, Directory.GetCurrentDirectory(), comp.SingleSourceFile, f); return(false); } } var files = new List <string>(); files.AddRange(manifest.IncludeFiles.Keys); var hashes = GetHashes(files, comp.WorkingDirectory); foreach (var h in hashes) { if (h.Value.Result == DataHashResult.Ok) { string mhash; if (manifest.IncludeFiles.TryGetValue(h.Key, out mhash)) { if (mhash != h.Value.Hash) { Logging.Emit("include file hash changed {0}", h.Key); Logging.Miss(commonkey.SessionHash, DataHashResult.FileChanged, Directory.GetCurrentDirectory(), comp.SingleSourceFile, h.Key); return(false); } } else { Logging.Emit("include file added {0}", h.Key); Logging.Miss(commonkey.SessionHash, DataHashResult.FileAdded, Directory.GetCurrentDirectory(), comp.SingleSourceFile, h.Key); return(false); } } else { Logging.Emit("include file hash error {0} {1}", h.Key, h.Value.Result); Logging.Miss(commonkey.SessionHash, h.Value.Result, Directory.GetCurrentDirectory(), comp.SingleSourceFile, h.Key); return(false); } } #endregion #region check pdb if (comp.AttemptPdb) { if (comp.PdbExistsAlready) { var pdbhash = hasher.DigestBinaryFile(comp.PdbFile); if (pdbhash.Hash != manifest.EarlierPdbHash) { outputCache.Remove(commonkey.Hash); Logging.Miss(commonkey.Hash, DataHashResult.FileChanged, commonkey.Hash, comp.PdbFile, ""); return(false); } } } #endregion #region check cached data exists foreach (var f in new string[] { F_Manifest, F_Object }) { if (!outputCache.ContainsEntry(commonkey.SessionHash, f)) { outputCache.Remove(commonkey.SessionHash); Logging.Miss(commonkey.SessionHash, DataHashResult.CacheCorrupt, commonkey.SessionHash, comp.SingleSourceFile, ""); return(false); } } #endregion if (Settings.MissLogEnabled) { Logging.Emit("hit hc={0},dir={1},src={2}", commonkey.Hash, comp.WorkingDirectory, comp.SingleSourceFile); } return(true); // cache hit, all includes match and no new files added #endregion } Logging.Miss(commonkey.Hash, DataHashResult.NoPreviousBuild, comp.WorkingDirectory, comp.SingleSourceFile, ""); return(false); }
protected virtual void DoCacheMiss(ICompiler c, DataHash hc, IEnumerable<string> args, CacheManifest m, List<string> ifiles) { bool good = true; try { var idirs = c.GetUsedIncludeDirs(ifiles); if (idirs.Count < 1) { throw new InvalidDataException( string.Format("could not find any include folders?! [{0}]", string.Join(" ", args))); } #region process includes folders // save manifest and other things to cache var others = c.GetPotentialIncludeFiles(idirs, ifiles); m = new CacheManifest(); m.PotentialNewIncludes = others; m.IncludeFiles = new Dictionary<string, string>(); m.TimeStamp = DateTime.Now.ToString("s"); m.CommonHash = hc.Hash; #endregion var hashes = GetHashes(ifiles); #region check include files foreach (var x in hashes) { if (x.Value.Result == DataHashResult.Ok) { m.IncludeFiles[x.Key] = x.Value.Hash; } else { Logging.Emit("input hash error {0} {1}", x.Key, x.Value.Result); Logging.Miss(hc.Hash, x.Value.Result, c.WorkingDirectory, c.SingleSourceFile, x.Key); good = false; m.Disable = true; break; } } #endregion } finally { Unlock(CacheLockType.Read); if (good) { Lock(CacheLockType.ReadWrite); try { SaveOutputsLocked(m, c); } finally { Unlock(CacheLockType.ReadWrite); } } } }
/// <summary> /// When this returns, we will hold the output cache mutex. /// </summary> /// <param name="commonkey"></param> /// <param name="manifest"></param> /// <returns></returns> public override bool CheckCache(ICompiler comp, IEnumerable<string> args, DataHash commonkey, out CacheManifest manifest ) { manifest = null; Lock(CacheLockType.Read); manifest = GetCachedManifestLocked(commonkey); if (manifest != null) { #region build missed before if (manifest.Disable) { Logging.Emit("disabled by manifest"); return false; } #region check includes foreach (var f in manifest.PotentialNewIncludes) { if (!FileUtils.FileMissing(f)) { Logging.Emit("detected added include file {0}", f); Logging.Miss(commonkey.Hash, DataHashResult.FileAdded, Directory.GetCurrentDirectory(), comp.SingleSourceFile, f); return false; } } var hashes = GetHashes(manifest.IncludeFiles.Keys); foreach (var h in hashes) { if (h.Value.Result == DataHashResult.Ok) { string mhash; if (manifest.IncludeFiles.TryGetValue(h.Key, out mhash)) { if (mhash != h.Value.Hash) { Logging.Emit("include file hash changed {0}", h.Key); Logging.Miss(commonkey.Hash, DataHashResult.FileChanged, Directory.GetCurrentDirectory(), comp.SingleSourceFile, h.Key); return false; } } else { Logging.Emit("include file added {0}", h.Key); Logging.Miss(commonkey.Hash, DataHashResult.FileAdded, Directory.GetCurrentDirectory(), comp.SingleSourceFile, h.Key); return false; } } else { Logging.Emit("include file hash error {0} {1}", h.Key, h.Value.Result); Logging.Miss(commonkey.Hash, h.Value.Result, Directory.GetCurrentDirectory(), comp.SingleSourceFile, h.Key); return false; } } #endregion #region check pdb if (comp.AttemptPdb) { if (comp.PdbExistsAlready) { var pdbhash = hasher.DigestBinaryFile(comp.PdbFile); if (pdbhash.Hash != manifest.EarlierPdbHash) { outputCache.Remove(commonkey.Hash); Logging.Miss(commonkey.Hash, DataHashResult.FileChanged, commonkey.Hash, comp.PdbFile, ""); return false; } } } #endregion #region check cached data exists foreach (var f in new string[] { F_Manifest, F_Object }) { if (!FileUtils.Exists(outputCache.MakePath(commonkey.Hash, f))) { outputCache.Remove(commonkey.Hash); Logging.Miss(commonkey.Hash, DataHashResult.CacheCorrupt, commonkey.Hash, comp.SingleSourceFile, ""); return false; } } #endregion if (Settings.MissLogEnabled) { Logging.Emit("hit hc={0},dir={1},src={2}", commonkey.Hash, comp.WorkingDirectory, comp.SingleSourceFile); } return true; // cache hit, all includes match and no new files added #endregion } Logging.Miss(commonkey.Hash, DataHashResult.NoPreviousBuild, comp.WorkingDirectory, comp.SingleSourceFile, ""); return false; }
void CopyOutputFiles( ICompiler comp, DataHash hc) { try { CopyFile(outputCache.MakePath(hc.Hash, F_Object), comp.ObjectTarget); if (comp.GeneratePdb && comp.AttemptPdb && comp.PdbFile != null) CopyFile(outputCache.MakePath(hc.Hash, F_Pdb), comp.PdbFile); } catch (Exception e) { Logging.Error("{0}",e); throw; } }
protected abstract int OnCacheMissLocked(ICompiler comp, DataHash hc, IEnumerable<string> args, CacheManifest m);
public abstract bool CheckCache(ICompiler comp, IEnumerable <string> args, DataHash commonkey, out CacheManifest manifest);
DataHash DigestFile( MD5 provider, string filepath, Regex findDateTime) { var rv = new DataHash() { Result = DataHashResult.FileNotFound, InputName = filepath, }; if (!FileUtils.Exists(filepath)) return rv; provider.Initialize(); var fs = new FileStream(filepath, FileMode.Open, FileAccess.Read, FileShare.Read); using (var bs = new BufferedStream(fs)) { Logging.Emit("digest {0}", filepath); rv.Hash = new SoapHexBinary(provider.ComputeHash(bs)).ToString(); rv.Result = DataHashResult.Ok; if (findDateTime != null) { // check include cache for this file if (includeCache.ContainsEntry(rv.Hash, F_NotDateTime)) { return rv; } if (includeCache.ContainsEntry(rv.Hash, F_HasDateTime)) { rv.Result = DataHashResult.ContainsTimeOrDate; return rv; } bs.Seek(0, SeekOrigin.Begin); using (var ts = new StreamReader(bs)) { string line = null; do { line = ts.ReadLine(); if (line == null) { includeCache.WaitOne(); try { includeCache.AddTextFileContent(rv.Hash, F_NotDateTime, ""); } finally { includeCache.ReleaseMutex(); } break; } if (findDateTime.IsMatch(line)) { rv.Result = DataHashResult.ContainsTimeOrDate; includeCache.WaitOne(); try { includeCache.AddTextFileContent(rv.Hash, F_HasDateTime, ""); } finally { includeCache.ReleaseMutex(); } break; } } while (true); } } } rv.Result = DataHashResult.Ok; return rv; }
public bool CheckCache(ICompiler comp, IEnumerable <string> args, DataHash commonkey, out CacheManifest manifest) { manifest = null; return(false); }
public override bool CheckCache(ICompiler comp, IEnumerable<string> args, DataHash commonkey, out CacheManifest manifest) { throw new NotImplementedException(); }