public CacheInfo(IFileCacheStore statCache) { cache = statCache; Logging.Emit("creating cache info mutex"); statMtx = new Mutex(false, "cclash_stat_" + cache.FolderPath.ToLower().GetHashCode()); Logging.Emit("created cache info mutex"); }
protected override int OnCacheMissLocked(ICompiler comp, DataHash hc, IEnumerable <string> args, CacheManifest m) { Logging.Emit("cache miss"); outputCache.EnsureKey(hc.Hash); var stderrfile = outputCache.MakePath(hc.Hash, F_Stderr); var stdoutfile = outputCache.MakePath(hc.Hash, F_Stdout); var ifiles = new List <string>(); Stats.LockStatsCall(() => Stats.CacheMisses++); int rv = Compile(comp, args, stderrfile, stdoutfile, ifiles); // we still hold the cache lock, create the manifest asap or give up now! if (rv != 0) { Unlock(CacheLockType.Read); } else { // this unlocks for us try { DoCacheMiss(comp, hc, args, m, ifiles); } catch (CClashWarningException) { return(CompileOnly(comp, args)); } } return(rv); }
public static string Find() { var compiler = _Find(); Logging.Emit("chose compiler {}", compiler); return(compiler); }
public bool Preflight(string cachedir) { Logging.Emit("cclash server preflight check"); var mtx = new Mutex(false, "cclash_serv_" + cachedir.ToLower().GetHashCode()); serverMutex = mtx; try { if (!mtx.WaitOne(1000)) { quitnow = true; Logging.Error("another server is already running"); return(false); // some other process is holding it! } else { Logging.Emit("cclash server preflight ok"); } } catch (AbandonedMutexException) { Logging.Warning("previous instance did not exit cleanly!"); } return(true); }
public List <string> GetUsedIncludeDirs(List <string> files) { var incdirs = new List <string>(); var tmplist = new List <string>(1000); string iinc = null; EnvironmentVariables.TryGetValue("INCLUDE", out iinc); Logging.Emit("INCLUDE={0}", iinc); if (iinc != null) { incs.Clear(); incs.AddRange(cliincs); foreach (var i in iinc.Split(';')) { incs.Add(i); Logging.Emit("notice include folder: {0}", i); } incdirs.AddRange(incs); } var srcfolder = Path.GetDirectoryName(SingleSourceFile); if (string.IsNullOrEmpty(srcfolder)) { srcfolder = WorkingDirectory; } Logging.Emit("notice source folder: {0}", srcfolder); incdirs.Add(Path.GetFullPath(srcfolder)); return(incdirs); }
protected virtual void SaveOutputsLocked(CacheManifest m, ICompiler c) { outputCache.AddFile(m.SessionHash, c.ObjectTarget, F_Object); if (c.GeneratePdb) { var pdbhash = hasher.DigestBinaryFile(c.PdbFile); m.PdbHash = pdbhash.Hash; outputCache.AddFile(m.SessionHash, c.PdbFile, F_Pdb); Stats.LockStatsCall(() => Stats.CacheSize += new FileInfo(c.PdbFile).Length); } Stats.LockStatsCall(() => Stats.CacheObjects++); Stats.LockStatsCall(() => Stats.CacheSize += new FileInfo(c.ObjectTarget).Length); // write manifest var duration = c.Age; m.Duration = (int)duration.TotalMilliseconds; Logging.Emit("cache miss took {0}ms", (int)duration.TotalMilliseconds); using (var manifest = outputCache.OpenFileStream(m.SessionHash, F_Manifest, FileMode.OpenOrCreate, FileAccess.Write)) { m.Serialize(manifest); } }
protected override int OnCacheMissLocked(ICompiler comp, DataHash hc, IEnumerable <string> args, CClashRequest req) { Logging.Emit("cache miss"); outputCache.EnsureKey(hc.Hash); var ifiles = new List <string>(); Stats.LockStatsCall(() => Stats.CacheMisses++); using (var stderr = outputCache.OpenFileStream(hc.Hash, F_Stderr, FileMode.OpenOrCreate, FileAccess.Write)) using (var stdout = outputCache.OpenFileStream(hc.Hash, F_Stdout, FileMode.OpenOrCreate, FileAccess.Write)) { int rv = Compile(comp, args, stderr, stdout, ifiles); // we still hold the cache lock, create the manifest asap or give up now! if (rv != 0) { Unlock(CacheLockType.Read); } else { // this unlocks for us try { DoCacheMiss(comp, hc, args, req, ifiles); } catch (CClashWarningException) { return(CompileOnly(comp, args)); } } return(rv); } }
public DataHash DeriveHashKey(ICompiler comp, IEnumerable <string> args) { Logging.Emit("compiler is {0}", comp.CompilerExe); var comphash = DigestCompiler(comp.CompilerExe); if (comphash.Result == DataHashResult.Ok) { var srchash = hasher.DigestBinaryFile(comp.SingleSourceFile); if (srchash.Result == DataHashResult.Ok) { var buf = new StringBuilder(); buf.AppendLine(CacheInfo.CacheFormat); buf.AppendLine(srchash.Hash); buf.AppendLine(comp.WorkingDirectory); string incs = null; comp.EnvironmentVariables.TryGetValue("INCLUDE", out incs); if (incs != null) { buf.AppendLine(incs); } foreach (var a in args) { buf.AppendLine(a); } buf.AppendLine(comphash.Hash); comphash = hasher.DigestString(buf.ToString()); } } return(comphash); }
public static ICompilerCache Get(bool direct, string cachedir, string compiler, string workdir, Dictionary <string, string> envs, out ICompiler comp) { comp = null; ICompilerCache rv = null; if (Settings.ServiceMode) { try { rv = new CClashServerClient(cachedir); } catch (CClashWarningException) { rv = new NullCompilerCache(cachedir); } } if (rv == null) { if (direct) { Logging.Emit("use direct mode"); rv = new DirectCompilerCache(cachedir); } else { throw new NotSupportedException("ppmode is not supported yet"); } } comp = rv.SetCompiler(compiler, workdir, envs); return(rv); }
protected int OnCacheHitLocked(ICompiler comp, DataHash hc, CacheManifest hm) { CopyStdio(comp, hc); CopyOutputFiles(comp, hc); // we dont need the lock now, it is highly unlikley someone else will // modify these files Unlock(CacheLockType.Read); var duration = comp.Age; var tstat = Task.Run(() => { Stats.LockStatsCall(() => { Stats.CacheHits++; if (hm.Duration < duration.TotalMilliseconds) { // this cached result was slow. record a stat. Stats.SlowHitCount++; Logging.Emit("slow cache hit {0}ms", (int)duration.TotalMilliseconds); } else { Logging.Emit("fast cache hit {0}ms", (int)duration.TotalMilliseconds); } }); }); tstat.Wait(); return(0); }
void Connect() { var exe = GetType().Assembly.Location; if (ncs == null) { Open(); } try { ConnectClient(); return; } catch (IOException ex) { Logging.Emit("error connecting {0}", ex.Message); try { ncs.Dispose(); Open(); } catch { } } catch (TimeoutException) { Logging.Emit("could not connect to cclash service"); throw new CClashServerNotReadyException(); } }
public void WaitOne() { Logging.Emit("WaitOne {0}", FolderPath); if (!mtx.WaitOne()) { throw new InvalidProgramException("mutex lock failed " + mtx.ToString()); } }
public static void Input(string dir, string target, IEnumerable <string> args) { if (Settings.DebugEnabled) { var cfiles = from a in args where a.Contains(".c") select a; Logging.Emit("invoked: dir={0}, target={1} srcs={2}", dir, target, string.Join(",", cfiles.ToArray())); } }
public DirectCompilerCacheServer(string cachedir) : base(cachedir) { SetupStats(); base.includeCache.CacheEntryChecksInMemory = true; Logging.Emit("server locking cache data"); base.Lock(CacheLockType.ReadWrite); // base is a multi-process lock, keep this forever }
public void Open(string folderPath) { FolderPath = Path.GetFullPath(folderPath); SetupLocks(); Logging.Emit("locking file store: {0}", FolderPath); WaitOne(); var tlist = new List <Thread>(); try { if (Directory.Exists(FolderPath)) { bool bad_cache_format = false; if (File.Exists(Path.Combine(FolderPath, CacheInfo.F_CacheVersion))) { var cdv = File.ReadAllText(Path.Combine(FolderPath, CacheInfo.F_CacheVersion)); bad_cache_format = cdv != CacheInfo.CacheFormat; } if (File.Exists(Path.Combine(FolderPath, CacheInfo.F_CacheType))) { var ct = File.ReadAllText(Path.Combine(FolderPath, CacheInfo.F_CacheType)); bad_cache_format = ct != Settings.CacheType.ToString(); } if (bad_cache_format) { Logging.Emit("corrupt/new filestore, deleting: {0}", FolderPath); Directory.Delete(FolderPath, true); } } if (!Directory.Exists(FolderPath)) { Logging.Emit("create fresh filestore"); Directory.CreateDirectory(FolderPath); File.WriteAllText(Path.Combine(FolderPath, CacheInfo.F_CacheVersion), CacheInfo.CacheFormat); File.WriteAllText(Path.Combine(FolderPath, CacheInfo.F_CacheType), Settings.CacheType.ToString()); } Logging.Emit("filestore ready: {0}", FolderPath); } catch (IOException) { throw new CClashErrorException("could not clear cache!"); } catch (UnauthorizedAccessException uae) { throw new CClashWarningException("cache access error: " + uae.Message); } finally { ReleaseMutex(); } }
private int RunSlaveCompiler(ICompiler comp, string optionsStr) { var myProcess = Process.GetCurrentProcess(); var psi = new ProcessStartInfo(myProcess.MainModule.FileName, optionsStr) { UseShellExecute = false, RedirectStandardError = true, RedirectStandardOutput = true, WorkingDirectory = comp.WorkingDirectory, ErrorDialog = true, }; var newCompiler = Process.Start(psi); newCompiler.OutputDataReceived += (o, a) => { if (a.Data != null) { if (comp.StdOutputCallback != null) { comp.StdOutputCallback(a.Data + Environment.NewLine); } if (Settings.DebugEnabled) { Logging.Emit("stdout {0}", a.Data); } } }; newCompiler.ErrorDataReceived += (o, a) => { if (a.Data != null) { if (comp.StdErrorCallback != null) { comp.StdErrorCallback(a.Data + Environment.NewLine); } if (Settings.DebugEnabled) { Logging.Emit("stderr {0}", a.Data); } } }; newCompiler.BeginErrorReadLine(); newCompiler.BeginOutputReadLine(); newCompiler.WaitForExit(); return(newCompiler.ExitCode); }
public void Listen(string cachedir) { Environment.CurrentDirectory = cdto; Logging.Emit("creating direct cache server.."); cache = new DirectCompilerCacheServer(cachedir); Logging.Emit("starting server threads.."); while (serverThreads.Count < MaxServerThreads) { NewServerThread(cachedir); } // maintain the threadpool while (!quitnow) { foreach (var t in serverThreads.ToArray()) { if (busyThreads > 0) { Logging.Emit("{0} busy threads", busyThreads); } if (t.Join(1000)) { serverThreads.Remove(t); Logging.Emit("replacing thread"); NewServerThread(cachedir); } } if (busyThreads < 1) { Logging.Emit("server is idle.."); } if (DateTime.Now.Subtract(lastRequest).TotalMinutes > QuitAfterIdleMinutes) { quitnow = true; } } Logging.Emit("waiting for threads to finish"); foreach (var t in serverThreads) { Logging.Emit("joining thread {0}", t.ManagedThreadId); if (!t.Join(2000)) { Logging.Emit("thread still running.."); } } Logging.Emit("commiting stats"); cache.SetupStats(); Logging.Emit("server quitting"); serverMutex.ReleaseMutex(); }
public virtual bool IsSupported(IEnumerable <string> args) { OperationStart = DateTime.Now; if (FileUtils.Exists(Compiler.CompilerExe)) { var rv = Compiler.ProcessArguments(args.ToArray()); if (!rv) { Logging.Emit("args not supported {0}", Cache.GetType().Name); } return(rv); } throw new FileNotFoundException(Compiler.CompilerExe); }
public CompilerCacheBase(string cacheFolder) : this() { Logging.Emit("setting up file stores"); if (string.IsNullOrEmpty(cacheFolder)) { throw new ArgumentNullException("cacheFolder"); } outputCache = FileCacheStore.Load(Path.Combine(cacheFolder, "outputs")); includeCache = FileCacheStore.Load(Path.Combine(cacheFolder, "includes")); Logging.Emit("setup cache info"); stats = new CacheInfo(outputCache); Logging.Emit("setup hasher"); hasher = new HashUtil(includeCache); }
public int CompileOrCache(ICompiler comp, IEnumerable <string> args) { Logging.Emit("client args: {0}", string.Join(" ", args.ToArray())); if (comp != null) // compiler is set, server wasnt ready { return(comp.InvokeCompiler(args, null, null, false, new List <string>())); } try { var req = new CClashRequest() { cmd = Command.Run, compiler = compilerPath, envs = environment, workdir = workingdir, argv = new List <string> (args), }; var resp = Transact(req); if (resp != null) { if (stdErrCallback != null) { stdErrCallback(resp.stderr); } else { Console.Error.Write(resp.stderr); } if (stdOutCallback != null) { stdOutCallback(resp.stdout); } else { Console.Out.Write(resp.stdout); } return(resp.exitcode); } else { throw new CClashErrorException("server returned no response"); } } catch (Exception e) { Logging.Emit("server error! {0}", e); throw new CClashWarningException("server error"); } }
public static void StartBackgroundServer() { using (var ssm = new System.Threading.Mutex(false, "cclash_server_spawn")) { var can_start_server = ssm.WaitOne(500); try { if (can_start_server) { Logging.Emit("starting new server"); // start the server var p = new Process(); var ours = FileUtils.GetShortPath(typeof(CClashServerClient).Assembly.Location); var exedir = Path.GetDirectoryName(ours); var exepath = Path.Combine(exedir, "cclash.exe"); if (!File.Exists(exepath)) { exepath = ours; } var pargs = new List <string> { exepath, "--cclash-server" }; if (Settings.DebugFile != null) { pargs.Add("--debug"); } var command = "cmd"; var command_args = "/c " + string.Join(" ", pargs.ToArray()); p.StartInfo = new ProcessStartInfo(command); p.StartInfo.UseShellExecute = false; p.StartInfo.CreateNoWindow = true; p.StartInfo.Arguments = command_args; p.StartInfo.ErrorDialog = false; p.StartInfo.WorkingDirectory = Path.GetPathRoot(Environment.CurrentDirectory); p.StartInfo.WindowStyle = ProcessWindowStyle.Hidden; p.Start(); } System.Threading.Thread.Sleep(1000); } finally { if (can_start_server) { ssm.ReleaseMutex(); } } } }
FileCacheStore(string folderPath) { FolderPath = Path.GetFullPath(folderPath); mtx = new Mutex(false, "cclash_mtx_" + FolderPath.ToLower().GetHashCode()); Logging.Emit("locking file store: {0}", FolderPath); WaitOne(); var tlist = new List <Thread>(); try { if (!Directory.Exists(FolderPath)) { Directory.CreateDirectory(FolderPath); } else { bool bad_cache_format = true; if (File.Exists(Path.Combine(FolderPath, CacheInfo.F_CacheVersion))) { var cdv = File.ReadAllText(Path.Combine(FolderPath, CacheInfo.F_CacheVersion)); bad_cache_format = cdv != CacheInfo.CacheFormat; } if (bad_cache_format) { Logging.Emit("corrupt filestore, deleting: {0}", FolderPath); // cache is too old, wiping Directory.Delete(FolderPath, true); Directory.CreateDirectory(FolderPath); File.WriteAllText(Path.Combine(FolderPath, CacheInfo.F_CacheVersion), CacheInfo.CacheFormat); } } Logging.Emit("filestore ready: {0}", FolderPath); } catch (IOException) { throw new CClashErrorException("could not clear cache!"); } catch (UnauthorizedAccessException uae) { throw new CClashWarningException("cache access error: " + uae.Message); } finally { ReleaseMutex(); } }
public DataHash DeriveHashKey(ICompiler comp, IEnumerable <string> args) { Logging.Emit("compiler is {0}", comp.CompilerExe); var tmp = DigestCompiler(comp.CompilerExe); var comphash = new DataHash() { InputName = tmp.InputName, Result = tmp.Result, Hash = tmp.Hash, }; if (comphash.Result == DataHashResult.Ok) { var buf = new StringWriter(); DataHash session; buf.WriteLine(CacheInfo.CacheFormat); // our compiler and folder buf.WriteLine(comphash.Hash); buf.WriteLine(comp.WorkingDirectory); buf.WriteLine(comp.SingleSourceFile); // important env vars foreach (var ename in new string[] { "INCLUDE", "LIB" }) { string ev = null; if (comp.EnvironmentVariables.TryGetValue(ename, out ev)) { if (!string.IsNullOrEmpty(ev)) { buf.WriteLine(ev); } } } // now all the command line options foreach (var a in args) { buf.WriteLine(a); } session = hasher.DigestString(buf.ToString()); comphash.SessionHash = session.Hash; comphash.Hash = comphash.SessionHash; } return(comphash); }
public override Dictionary <string, DataHash> GetHashes(IEnumerable <string> fnames) { if (hashcache.Count > 20000) { lock (hashcache) { hashcache.Clear(); } } var unknown = new List <string>(); var rv = new Dictionary <string, DataHash>(); foreach (var n in fnames) { var x = n.ToLower(); lock (hashcache) { if (hashcache.ContainsKey(x)) { rv[x] = hashcache[x]; } else { unknown.Add(x); } } } if (unknown.Count > 0) { Logging.Emit("hash {0}/{1} new/changed files", unknown.Count, fnames.Count()); var tmp = base.GetHashes(fnames); lock (hashcache) { foreach (var filename in tmp.Keys) { var flow = filename.ToLower(); hashcache[flow] = tmp[filename]; rv[flow] = tmp[filename]; WatchFile(flow); } } } return(rv); }
public virtual bool IsSupported(ICompiler comp, IEnumerable <string> args) { if (FileUtils.Exists(compilerPath)) { var rv = comp.ProcessArguments(args.ToArray()); if (!rv) { Logging.Emit("unsupported args: {0}", string.Join(" ", args.ToArray())); } else { Logging.Input(comp.WorkingDirectory, comp.ObjectTarget, args); } return(rv); } throw new FileNotFoundException(compilerPath); }
void NewServerThread(string cachedir) { var t = new Thread(new ParameterizedThreadStart(ConnectionThreadFn)); t.IsBackground = true; serverThreads.Add(t); var nss = new NamedPipeServerStream(MakePipeName(cachedir), PipeDirection.InOut, MaxServerThreads, PipeTransmissionMode.Message, PipeOptions.WriteThrough | PipeOptions.Asynchronous); if (Settings.PipeSecurityEveryone) { var npa = new PipeAccessRule("Everyone", PipeAccessRights.ReadWrite, System.Security.AccessControl.AccessControlType.Allow); var nps = new PipeSecurity(); nps.AddAccessRule(npa); nss.SetAccessControl(nps); } t.Start(nss); Logging.Emit("server thread started"); }
public void WatchFile(string path) { if (!path.ToLower().Contains(":\\progra")) { return; } var dir = Path.GetDirectoryName(path); if (!Path.IsPathRooted(dir)) { dir = Path.GetFullPath(dir); } if (!Directory.Exists(dir)) { Logging.Error("ignored watch on missing folder {0}", dir); return; } DirectoryWatcher w; if (!dwatchers.TryGetValue(dir, out w)) { if (!FileExists(path)) { Logging.Error("ignored watch on missing file {0}", path); return; } Logging.Emit("create new watcher for {0}", dir); w = new DirectoryWatcher(dir); dwatchers.Add(dir, w); w.FileChanged += OnWatchedFileChanged; w.Enable(); } var file = Path.GetFileName(path); w.Watch(file); }
public void ConnectionThreadFn(object con) { using (var nss = con as NamedPipeServerStream) { try { while (!quitnow) { var w = nss.BeginWaitForConnection(null, null); FirstThreadReady = true; Logging.Emit("waiting for client.."); while (!w.AsyncWaitHandle.WaitOne(1000)) { if (quitnow) { return; } } nss.EndWaitForConnection(w); Logging.Emit("got client"); if (nss.IsConnected) { Logging.Emit("server connected"); ThreadBeforeProcessRequest(); ThreadIsBusy(); ServiceRequest(nss); } ThreadIsIdle(); } } catch (IOException ex) { Logging.Error("server thread got {0}, {1}", ex.GetType().Name, ex.Message); Logging.Error(":{0}", ex.ToString()); } } }
public void ServiceRequest(NamedPipeServerStream nss) { var msgbuf = new List <byte>(8192); var rxbuf = new byte[256 * 1024]; int count = 0; Logging.Emit("reading from client"); do { count = nss.Read(rxbuf, msgbuf.Count, rxbuf.Length); if (count > 0) { msgbuf.AddRange(rxbuf.Take(count)); } } while (!nss.IsMessageComplete); Logging.Emit("server read {0} bytes", msgbuf.Count); // deserialize message from msgbuf var req = CClashMessage.Deserialize <CClashRequest>(msgbuf.ToArray()); cache.Setup(); // needed? Logging.Emit("processing request"); var resp = ProcessRequest(req); Logging.Emit("request complete: supported={0}, exitcode={1}", resp.supported, resp.exitcode); var tx = resp.Serialize(); nss.Write(tx, 0, tx.Length); nss.Flush(); Logging.Emit("server written {0} bytes", tx.Length); nss.WaitForPipeDrain(); nss.Disconnect(); Logging.Emit("request done"); }
private static int RunBuild(string[] args, DateTime start, Action <string> stdout, Action <string> stderr) { Logging.Emit("client mode = {0}", Settings.ServiceMode); try { if (!Settings.Disabled) { string compiler = Compiler.Find(); if (compiler == null) { throw new System.IO.FileNotFoundException("cant find real cl compiler"); } var cachedir = Settings.CacheDirectory; Logging.Emit("compiler: {0}", compiler); ICompiler comp; using (ICompilerCache cc = CompilerCacheFactory.Get(Settings.DirectMode, cachedir, compiler, Environment.CurrentDirectory, Compiler.GetEnvironmentDictionary(), out comp)) { if (comp != null) { spawnServer = true; } cc.SetCaptureCallback(comp, stdout, stderr); long last_hits = 0; if (!Settings.ServiceMode) { last_hits = cc.Stats.CacheHits; } int res = cc.CompileOrCache(comp, args, null); if (!Settings.ServiceMode) { if (last_hits < cc.Stats.CacheHits) { WasHit = true; } } return(res); } } else { Logging.Emit("disabled by environment"); } } catch (CClashWarningException e) { Logging.Warning(e.Message); } catch (Exception e) { Logging.Emit("{0} after {1} ms", e.GetType().Name, DateTime.Now.Subtract(start).TotalMilliseconds); Logging.Emit("{0} {1}", e.GetType().Name + " message: " + e.Message); #if DEBUG Logging.Error("Exception from cacher {0}!!!", e); #endif } int rv = -1; try { var c = new Compiler() { CompilerExe = Compiler.Find(), }; c.SetEnvironment(Compiler.GetEnvironmentDictionary()); c.SetWorkingDirectory(Environment.CurrentDirectory); rv = c.InvokeCompiler(args, stderr, stdout, false, null); Logging.Emit("exit {0} after {1} ms", rv, DateTime.Now.Subtract(start).TotalMilliseconds); } catch (CClashErrorException e) { Logging.Error(e.Message); throw; } catch (CClashWarningException e) { Logging.Warning(e.Message); } return(rv); }