protected int OnCacheHitLocked(ICompiler comp, DataHash hc, CacheManifest hm)
{
CopyStdio(comp, hc);
CopyOutputFiles(comp, hc);
// we dont need the lock now, it is highly unlikley someone else will
// modify these files
Unlock(CacheLockType.Read);
var duration = comp.Age;
var tstat = Task.Run(() =>
{
Stats.LockStatsCall(() =>
{
Stats.CacheHits++;
if (hm.Duration < duration.TotalMilliseconds)
{
// this cached result was slow. record a stat.
Stats.SlowHitCount++;
Logging.Emit("slow cache hit {0}ms", (int)duration.TotalMilliseconds);
}
else
{
Logging.Emit("fast cache hit {0}ms", (int)duration.TotalMilliseconds);
}
});
});
tstat.Wait();
return(0);
}
protected virtual void SaveOutputsLocked(CacheManifest m, ICompiler c)
{
outputCache.AddFile(m.SessionHash, c.ObjectTarget, F_Object);
if (c.GeneratePdb)
{
var pdbhash = hasher.DigestBinaryFile(c.PdbFile);
m.PdbHash = pdbhash.Hash;
outputCache.AddFile(m.SessionHash, c.PdbFile, F_Pdb);
Stats.LockStatsCall(() => Stats.CacheSize += new FileInfo(c.PdbFile).Length);
}
Stats.LockStatsCall(() => Stats.CacheObjects++);
Stats.LockStatsCall(() => Stats.CacheSize += new FileInfo(c.ObjectTarget).Length);
// write manifest
var duration = c.Age;
m.Duration = (int)duration.TotalMilliseconds;
Logging.Emit("cache miss took {0}ms", (int)duration.TotalMilliseconds);
using (var manifest = outputCache.OpenFileStream(m.SessionHash, F_Manifest, FileMode.OpenOrCreate, FileAccess.Write))
{
m.Serialize(manifest);
}
}
protected CacheManifest GetCachedManifestLocked(DataHash commonkey)
{
CacheManifest manifest = null;
if (outputCache.ContainsEntry(commonkey.SessionHash, F_Manifest))
{
using (var mfs = outputCache.OpenFileStream(commonkey.SessionHash, F_Manifest, FileMode.Open, FileAccess.Read))
{
manifest = CacheManifest.Deserialize(mfs);
}
}
return(manifest);
}
protected CacheManifest GetCachedManifestLocked(DataHash commonkey)
{
CacheManifest manifest = null;
if (outputCache.ContainsEntry(commonkey.Hash, F_Manifest))
{
var mn = outputCache.MakePath(commonkey.Hash, F_Manifest);
using (var fs = new FileStream(mn, FileMode.Open)){
manifest = CacheManifest.Deserialize(fs);
}
}
return(manifest);
}
protected virtual void DoCacheMiss(ICompiler c, DataHash hc, IEnumerable <string> args, CClashRequest req, List <string> ifiles)
{
bool good = true;
CacheManifest m = null;
try
{
var idirs = c.GetUsedIncludeDirs(ifiles);
if (idirs.Count < 1)
{
throw new InvalidDataException(
string.Format("could not find any include folders?! [{0}]",
string.Join(" ", args)));
}
#region process includes folders
// save manifest and other things to cache
var others = c.GetPotentialIncludeFiles(idirs, ifiles);
m = new CacheManifest();
m.Request = req;
m.PotentialNewIncludes = others;
m.IncludeFiles = new Dictionary <string, string>();
m.TimeStamp = DateTime.Now.ToString("s");
m.SessionHash = hc.SessionHash;
#endregion
var hashes = GetHashes(ifiles, c.WorkingDirectory);
#region check include files
foreach (var x in hashes)
{
if (x.Value.Result == DataHashResult.Ok)
{
m.IncludeFiles[x.Key] = x.Value.Hash;
}
else
{
Logging.Emit("input hash error {0} {1}", x.Key, x.Value.Result);
Logging.Miss(hc.SessionHash, x.Value.Result, c.WorkingDirectory, c.SingleSourceFile, x.Key);
good = false;
m.Disable = true;
break;
}
}
#endregion
}
finally
{
Unlock(CacheLockType.Read);
if (good)
{
Lock(CacheLockType.ReadWrite);
try
{
if (m != null)
{
SaveOutputsLocked(m, c);
}
}
finally
{
Unlock(CacheLockType.ReadWrite);
}
}
}
}
/// <summary>
/// When this returns, we will hold the output cache mutex.
/// </summary>
/// <param name="commonkey"></param>
/// <param name="manifest"></param>
/// <returns></returns>
public override bool CheckCache(ICompiler comp, IEnumerable<string> args, DataHash commonkey, out CacheManifest manifest )
{
manifest = null;
Lock(CacheLockType.Read);
manifest = GetCachedManifestLocked(commonkey);
if (manifest != null)
{
#region build missed before
if (manifest.Disable)
{
Logging.Emit("disabled by manifest");
return false;
}
#region check includes
foreach (var f in manifest.PotentialNewIncludes)
{
if (!FileUtils.FileMissing(f))
{
Logging.Emit("detected added include file {0}", f);
Logging.Miss(commonkey.Hash, DataHashResult.FileAdded, Directory.GetCurrentDirectory(), comp.SingleSourceFile, f);
return false;
}
}
var hashes = GetHashes(manifest.IncludeFiles.Keys);
foreach (var h in hashes)
{
if (h.Value.Result == DataHashResult.Ok)
{
string mhash;
if (manifest.IncludeFiles.TryGetValue(h.Key, out mhash))
{
if (mhash != h.Value.Hash)
{
Logging.Emit("include file hash changed {0}", h.Key);
Logging.Miss(commonkey.Hash, DataHashResult.FileChanged, Directory.GetCurrentDirectory(), comp.SingleSourceFile, h.Key);
return false;
}
}
else
{
Logging.Emit("include file added {0}", h.Key);
Logging.Miss(commonkey.Hash, DataHashResult.FileAdded, Directory.GetCurrentDirectory(), comp.SingleSourceFile, h.Key);
return false;
}
}
else
{
Logging.Emit("include file hash error {0} {1}", h.Key, h.Value.Result);
Logging.Miss(commonkey.Hash, h.Value.Result, Directory.GetCurrentDirectory(), comp.SingleSourceFile, h.Key);
return false;
}
}
#endregion
#region check pdb
if (comp.AttemptPdb)
{
if (comp.PdbExistsAlready)
{
var pdbhash = hasher.DigestBinaryFile(comp.PdbFile);
if (pdbhash.Hash != manifest.EarlierPdbHash)
{
outputCache.Remove(commonkey.Hash);
Logging.Miss(commonkey.Hash, DataHashResult.FileChanged, commonkey.Hash, comp.PdbFile, "");
return false;
}
}
}
#endregion
#region check cached data exists
foreach (var f in new string[] { F_Manifest, F_Object })
{
if (!FileUtils.Exists(outputCache.MakePath(commonkey.Hash, f)))
{
outputCache.Remove(commonkey.Hash);
Logging.Miss(commonkey.Hash, DataHashResult.CacheCorrupt, commonkey.Hash, comp.SingleSourceFile, "");
return false;
}
}
#endregion
if (Settings.MissLogEnabled)
{
Logging.Emit("hit hc={0},dir={1},src={2}", commonkey.Hash, comp.WorkingDirectory, comp.SingleSourceFile);
}
return true; // cache hit, all includes match and no new files added
#endregion
}
Logging.Miss(commonkey.Hash, DataHashResult.NoPreviousBuild, comp.WorkingDirectory, comp.SingleSourceFile, "");
return false;
}
protected virtual void DoCacheMiss(ICompiler c, DataHash hc, IEnumerable<string> args, CacheManifest m, List<string> ifiles)
{
bool good = true;
try
{
var idirs = c.GetUsedIncludeDirs(ifiles);
if (idirs.Count < 1)
{
throw new InvalidDataException(
string.Format("could not find any include folders?! [{0}]",
string.Join(" ", args)));
}
#region process includes folders
// save manifest and other things to cache
var others = c.GetPotentialIncludeFiles(idirs, ifiles);
m = new CacheManifest();
m.PotentialNewIncludes = others;
m.IncludeFiles = new Dictionary<string, string>();
m.TimeStamp = DateTime.Now.ToString("s");
m.CommonHash = hc.Hash;
#endregion
var hashes = GetHashes(ifiles);
#region check include files
foreach (var x in hashes)
{
if (x.Value.Result == DataHashResult.Ok)
{
m.IncludeFiles[x.Key] = x.Value.Hash;
}
else
{
Logging.Emit("input hash error {0} {1}", x.Key, x.Value.Result);
Logging.Miss(hc.Hash, x.Value.Result, c.WorkingDirectory, c.SingleSourceFile, x.Key);
good = false;
m.Disable = true;
break;
}
}
#endregion
}
finally
{
Unlock(CacheLockType.Read);
if (good)
{
Lock(CacheLockType.ReadWrite);
try
{
SaveOutputsLocked(m, c);
}
finally
{
Unlock(CacheLockType.ReadWrite);
}
}
}
}
protected override int OnCacheMissLocked(ICompiler comp, DataHash hc, IEnumerable<string> args, CacheManifest m)
{
throw new NotImplementedException();
}
protected abstract int OnCacheMissLocked(ICompiler comp, DataHash hc, IEnumerable <string> args, CacheManifest m);
public override bool CheckCache(ICompiler comp, IEnumerable <string> args, DataHash commonkey, out CacheManifest manifest)
{
throw new NotImplementedException();
}
public bool CheckCache(ICompiler comp, IEnumerable<string> args, DataHash commonkey, out CacheManifest manifest)
{
manifest = null;
return false;
}
protected override int OnCacheMissLocked(ICompiler comp, DataHash hc, IEnumerable <string> args, CacheManifest m)
{
Logging.Emit("cache miss");
outputCache.EnsureKey(hc.Hash);
var stderrfile = outputCache.MakePath(hc.Hash, F_Stderr);
var stdoutfile = outputCache.MakePath(hc.Hash, F_Stdout);
var ifiles = new List <string>();
Stats.LockStatsCall(() => Stats.CacheMisses++);
int rv = Compile(comp, args, stderrfile, stdoutfile, ifiles);
// we still hold the cache lock, create the manifest asap or give up now!
if (rv != 0)
{
Unlock(CacheLockType.Read);
}
else
{
// this unlocks for us
try
{
DoCacheMiss(comp, hc, args, m, ifiles);
}
catch (CClashWarningException)
{
return(CompileOnly(comp, args));
}
}
return(rv);
}
protected abstract int OnCacheMissLocked(ICompiler comp, DataHash hc, IEnumerable<string> args, CacheManifest m);
public abstract bool CheckCache(ICompiler comp, IEnumerable<string> args, DataHash commonkey, out CacheManifest manifest);
protected int OnCacheHitLocked(ICompiler comp, DataHash hc, CacheManifest hm)
{
CopyStdio(comp, hc);
CopyOutputFiles(comp, hc);
// we dont need the lock now, it is highly unlikley someone else will
// modify these files
Unlock(CacheLockType.Read);
var duration = comp.Age;
var tstat = Task.Run(() =>
{
Stats.LockStatsCall(() =>
{
Stats.CacheHits++;
if (hm.Duration < duration.TotalMilliseconds)
{
// this cached result was slow. record a stat.
Stats.SlowHitCount++;
Logging.Emit("slow cache hit {0}ms", (int)duration.TotalMilliseconds);
}
else
{
Logging.Emit("fast cache hit {0}ms", (int)duration.TotalMilliseconds);
}
});
});
tstat.Wait();
return 0;
}
/// <summary>
/// When this returns, we will hold the output cache mutex.
/// </summary>
/// <param name="commonkey"></param>
/// <param name="manifest"></param>
/// <returns></returns>
public override bool CheckCache(ICompiler comp, IEnumerable <string> args, DataHash commonkey, out CacheManifest manifest)
{
manifest = null;
Lock(CacheLockType.Read);
manifest = GetCachedManifestLocked(commonkey);
if (manifest != null)
{
#region build missed before
if (manifest.Disable)
{
Logging.Emit("disabled by manifest");
return(false);
}
#region check includes
foreach (var f in manifest.PotentialNewIncludes)
{
if (!FileUtils.FileMissing(f))
{
Logging.Emit("detected added include file {0}", f);
Logging.Miss(commonkey.SessionHash, DataHashResult.FileAdded, Directory.GetCurrentDirectory(), comp.SingleSourceFile, f);
return(false);
}
}
var files = new List <string>();
files.AddRange(manifest.IncludeFiles.Keys);
var hashes = GetHashes(files, comp.WorkingDirectory);
foreach (var h in hashes)
{
if (h.Value.Result == DataHashResult.Ok)
{
string mhash;
if (manifest.IncludeFiles.TryGetValue(h.Key, out mhash))
{
if (mhash != h.Value.Hash)
{
Logging.Emit("include file hash changed {0}", h.Key);
Logging.Miss(commonkey.SessionHash, DataHashResult.FileChanged, Directory.GetCurrentDirectory(), comp.SingleSourceFile, h.Key);
return(false);
}
}
else
{
Logging.Emit("include file added {0}", h.Key);
Logging.Miss(commonkey.SessionHash, DataHashResult.FileAdded, Directory.GetCurrentDirectory(), comp.SingleSourceFile, h.Key);
return(false);
}
}
else
{
Logging.Emit("include file hash error {0} {1}", h.Key, h.Value.Result);
Logging.Miss(commonkey.SessionHash, h.Value.Result, Directory.GetCurrentDirectory(), comp.SingleSourceFile, h.Key);
return(false);
}
}
#endregion
#region check pdb
if (comp.AttemptPdb)
{
if (comp.PdbExistsAlready)
{
var pdbhash = hasher.DigestBinaryFile(comp.PdbFile);
if (pdbhash.Hash != manifest.EarlierPdbHash)
{
outputCache.Remove(commonkey.Hash);
Logging.Miss(commonkey.Hash, DataHashResult.FileChanged, commonkey.Hash, comp.PdbFile, "");
return(false);
}
}
}
#endregion
#region check cached data exists
foreach (var f in new string[] { F_Manifest, F_Object })
{
if (!outputCache.ContainsEntry(commonkey.SessionHash, f))
{
outputCache.Remove(commonkey.SessionHash);
Logging.Miss(commonkey.SessionHash, DataHashResult.CacheCorrupt, commonkey.SessionHash, comp.SingleSourceFile, "");
return(false);
}
}
#endregion
if (Settings.MissLogEnabled)
{
Logging.Emit("hit hc={0},dir={1},src={2}", commonkey.Hash, comp.WorkingDirectory, comp.SingleSourceFile);
}
return(true); // cache hit, all includes match and no new files added
#endregion
}
Logging.Miss(commonkey.Hash, DataHashResult.NoPreviousBuild, comp.WorkingDirectory, comp.SingleSourceFile, "");
return(false);
}
protected override int OnCacheMissLocked(ICompiler comp, DataHash hc, IEnumerable <string> args, CacheManifest m)
{
throw new NotImplementedException();
}
protected virtual void SaveOutputsLocked(CacheManifest m, ICompiler c )
{
outputCache.AddFile(m.CommonHash, c.ObjectTarget, F_Object);
if (c.GeneratePdb)
{
var pdbhash = hasher.DigestBinaryFile(c.PdbFile);
m.PdbHash = pdbhash.Hash;
outputCache.AddFile(m.CommonHash, c.PdbFile, F_Pdb);
Stats.LockStatsCall(() => Stats.CacheSize += new FileInfo(c.PdbFile).Length);
}
Stats.LockStatsCall(() => Stats.CacheObjects++);
Stats.LockStatsCall(() => Stats.CacheSize += new FileInfo(c.ObjectTarget).Length);
// write manifest
var duration = c.Age;
m.Duration = (int)duration.TotalMilliseconds;
Logging.Emit("cache miss took {0}ms", (int)duration.TotalMilliseconds);
var fname = outputCache.MakePath(m.CommonHash, F_Manifest);
using (var fs = new FileStream(fname, FileMode.OpenOrCreate, FileAccess.Write))
{
m.Serialize(fs);
}
}
public abstract bool CheckCache(ICompiler comp, IEnumerable <string> args, DataHash commonkey, out CacheManifest manifest);
protected override int OnCacheMissLocked(ICompiler comp, DataHash hc, IEnumerable<string> args, CacheManifest m)
{
Logging.Emit("cache miss");
outputCache.EnsureKey(hc.Hash);
var stderrfile = outputCache.MakePath(hc.Hash, F_Stderr);
var stdoutfile = outputCache.MakePath(hc.Hash, F_Stdout);
var ifiles = new List<string>();
Stats.LockStatsCall(() => Stats.CacheMisses++);
int rv = Compile(comp, args, stderrfile, stdoutfile, ifiles );
// we still hold the cache lock, create the manifest asap or give up now!
if (rv != 0)
{
Unlock(CacheLockType.Read);
}
else
{
// this unlocks for us
try
{
DoCacheMiss(comp, hc, args, m, ifiles);
}
catch (CClashWarningException)
{
return CompileOnly(comp, args);
}
}
return rv;
}
public bool CheckCache(ICompiler comp, IEnumerable <string> args, DataHash commonkey, out CacheManifest manifest)
{
manifest = null;
return(false);
}
public override bool CheckCache(ICompiler comp, IEnumerable<string> args, DataHash commonkey, out CacheManifest manifest)
{
throw new NotImplementedException();
}
