private async Task TaskDispatcher(System.Threading.CancellationToken token) { try { //we need to distribute work from the queue while (true) { while (taskQueue.Count > 0 && !token.IsCancellationRequested) { //get number of available tasks (this can only increase from another thread, so should be fine) int taskQueueSize = taskQueue.Count; //get first available worker GolemWorker worker = null; if (workerPool.TryDequeue(out worker)) { System.Threading.Interlocked.Increment(ref runningWorkers); Logger.LogMessage("Worker " + worker.Peer.NodeId + " started"); worker.ClearTasks(); List <string> compilersUsed = new List <string>(); //get the number of tasks to process int taskCount = Math.Min(worker.TaskCapacity, taskQueueSize); for (int i = 0; i < taskCount; ++i) { CompilationTask task = null; if (taskQueue.TryDequeue(out task)) { if (!compilersUsed.Contains(task.Compiler)) { compilersUsed.Add(task.Compiler); } worker.AddTask(task); } } string hash = GolemCache.GetCompilerPackageHash(compilersUsed); DeploymentSpecImage specImg = new DeploymentSpecImage("SHA1:" + hash, "http://" + myIP + ":" + ServerPort + "/requestID/compiler/" + hash); //create deployment DeploymentSpec spec = new DeploymentSpec(EnvType.Hd, specImg, "Compiler", new List <string>() { }); worker.Dispatch(golemApi, spec, () => { workerPool.Enqueue(worker); Logger.LogMessage("Worker " + worker.Peer.NodeId + " finished"); System.Threading.Interlocked.Decrement(ref runningWorkers); }); } } await Task.Delay(1000, token); } } catch (TaskCanceledException) { } }
private async Task TaskProc(PeerApi golemApi, DeploymentSpec spec) { try { if (deployment == null) { deployment = spec; deploymentID = await golemApi.CreateDeploymentAsync(Peer.NodeId, deployment); } //1. Take all input files and includes and package them into one TAR package + notify HttpServer about that file string packedFileName = PackFilesPreProcessed(taskList); //2. Create command to compile those source files -> cl.exe .... ExecCommand compileCmd = GenerateCompileCommand(packedFileName, taskList); var results = await golemApi.UpdateDeploymentAsync(Peer.NodeId, deploymentID, new List <Command>() { new DownloadFileCommand(Service.GetHttpDownloadUri(packedFileName), packedFileName + ".tar", FileFormat.Tar), compileCmd }); bool error = false; string[] lines = results[0].Split(new[] { "\r\n", "\r", "\n" }, StringSplitOptions.None); foreach (string line in lines) { if (line.Contains(" error") || line.Contains("fatal error")) { Logger.LogError("[ERROR] " + packedFileName + ": " + line); error = true; } else if (line.Contains("warning")) { Logger.LogMessage("[WARNING] " + packedFileName + ": " + line); } } if (!error) { Logger.LogMessage("[SUCCESS] " + packedFileName); // Upload output.zip results = await golemApi.UpdateDeploymentAsync(Peer.NodeId, deploymentID, new List <Command>() { new UploadFileCommand(Service.GetHttpUploadUri(packedFileName), packedFileName + ".tar/output.zip") }); } } catch (Exception ex) { Console.WriteLine(ex.Message); } }
static void Main(string[] args) { PeerApi peerApi = new PeerApi("http://10.30.10.121:6162"); var info = peerApi.GetHubInfo(); System.Console.WriteLine(info.ToString()); string output = ""; foreach (NetworkInterface item in NetworkInterface.GetAllNetworkInterfaces()) { if (item.NetworkInterfaceType == NetworkInterfaceType.Ethernet && item.OperationalStatus == OperationalStatus.Up) { foreach (UnicastIPAddressInformation ip in item.GetIPProperties().UnicastAddresses) { if (ip.Address.AddressFamily == AddressFamily.InterNetwork) { output = ip.Address.ToString(); } } } } var myIP = Dns.GetHostAddresses(Dns.GetHostName()); runServer(); var peers = peerApi.ListPeers(); int myPeer = -1; for (int i = 0; i < peers.Count; i++) { if (peers[i].PeerAddr.Contains("10.30.8.5")) { myPeer = i; break; } } if (myPeer == -1) { return; } { string hash = "SHA1:213fad4e430ded42e6a949f61cf560ac96ec9878"; DeploymentSpecImage specImg = new DeploymentSpecImage(hash, "http://10.30.8.5:6000/generatedID/test1.hdi"); DeploymentSpec spec = new DeploymentSpec(EnvType.Hd, specImg, "compiler", new List <string>() { "dupa" }); var peer = peers[myPeer]; string depId = peerApi.CreateDeployment(peer.NodeId, spec); depId = depId.Replace("\"", ""); // Run batch file var results = peerApi.UpdateDeployment(peer.NodeId, depId, new List <Command>() { new ExecCommand("Debug/golemtest.bat", new List <string>()) }); // Upload output.zip results = peerApi.UpdateDeployment(peer.NodeId, depId, new List <Command>() { new UploadFileCommand("http://10.30.8.5:6000/generatedID/", "output.zip") }); peerApi.DropDeployment(peer.NodeId, depId); System.Console.WriteLine(depId); stopServer(); return; } //session { SessionApi sessionApi = new SessionApi("http://10.30.10.121:6162"); //create session var body = new HubSession(); long?sessionId = sessionApi.CreateSession(body); body = sessionApi.GetSession(sessionId); //add peer var res1 = sessionApi.AddSessionPeers(sessionId, new List <string>() { peers[0].NodeId }); var sPeers = sessionApi.ListSessionPeers(sessionId); body = sessionApi.GetSession(sessionId); var deploymentSpec = new DeploymentInfo(); deploymentSpec.Name = "dupa"; string result = sessionApi.CreateDeployment(sessionId, peers[0].NodeId, deploymentSpec); System.Console.WriteLine(result); } }
public void Dispatch(PeerApi golemApi, DeploymentSpec spec, Action onSuccess) { TaskProc(golemApi, spec).ContinueWith((task) => { onSuccess(); }); }