public void DownloadFileSafe(DBWorkFile file, string directory) { ExecuteSafe("download workfile", delegate() { DownloadFile(file, directory); }); }
public static void WriteToDisk(this DBWorkFile wf, DB db, string dir) { byte [] buffer = new byte [1024]; int read; string filename = Path.Combine(dir, wf.filename); DBFile file = DBFile_Extensions.Create(db, wf.file_id); if (!Directory.Exists(dir)) { Directory.CreateDirectory(dir); } using (Stream stream = db.Download(wf)) { using (FileStream fs = new FileStream(filename, FileMode.Create, FileAccess.Write, FileShare.Read)) { while (0 != (read = stream.Read(buffer, 0, buffer.Length))) { fs.Write(buffer, 0, read); } } } if (file.compressed_mime == "application/x-gzip") { FileUtilities.GZUncompress(filename); } }
private void DownloadFile(DBWorkFile file, string directory) { string filename = Path.Combine(directory, file.filename); if (!Directory.Exists(directory)) { Directory.CreateDirectory(directory); } using (WebClient web = new WebClient()) { web.Headers.Add("Accept-Encoding", "gzip"); web.DownloadFile(CreateWebServiceDownloadUrl(file.id), filename); if (web.ResponseHeaders ["Content-Encoding"] == "gzip") { FileUtilities.GZUncompress(filename); } } }
public Stream Download(DBWorkFile file) { return(new DBFileStream(DBFile_Extensions.Create(this, file.file_id), this)); }
private static void Build(List <BuildInfoEntry> list) { string temp_dir = null; List <Thread> threads = new List <Thread> (); List <BuildInfo> infos = new List <BuildInfo> (); Logger.Log("Building {0} work items", list.Count); try { // Get the path of the temporary directory for this process using (Process p = Process.GetCurrentProcess()) { temp_dir = Path.Combine(Path.Combine(Path.GetTempPath(), Configuration.ApplicationName), p.Id.ToString()); if (Directory.Exists(temp_dir)) { // This should happen very rarely Directory.Delete(temp_dir, true); } Directory.CreateDirectory(temp_dir); } for (int i = 0; i < list.Count; i++) { BuildInfoEntry entry = list [i]; BuildInfo info; if (failed_hostlanes != null) { foreach (DBHostLane failed in failed_hostlanes) { if (failed.id == entry.HostLane.id) { Logger.Log("Skipping work, the hostlane {0} has failed work in this run, which has disabled any further work (in this run).", failed.id); entry = null; break; } } } if (entry == null) { continue; } info = new BuildInfo(); infos.Add(info); info.lane = entry.Lane; info.revision = entry.Revision; // download dependent files if (entry.FilesToDownload != null) { for (int f = 0; f < entry.FilesToDownload.Count; f++) { DBWorkFile file = entry.FilesToDownload [f]; DBLane dependent_lane = entry.DependentLaneOfFiles [f]; WebService.DownloadFileSafe(file, Configuration.GetDependentDownloadDirectory(info.lane.id, dependent_lane.lane, info.revision.revision)); } } // Set revision-specific paths info.BUILDER_DATA_INSTALL_DIR = Configuration.GetDataInstallDir(entry.Lane.id, entry.Revision.revision); info.BUILDER_DATA_LOG_DIR = Configuration.GetDataLogDir(entry.Lane.id, entry.Revision.revision); info.BUILDER_DATA_SOURCE_DIR = Configuration.GetDataSourceDir(entry.Lane.id, entry.Revision.revision); info.environment_variables = entry.EnvironmentVariables; if (!Directory.Exists(info.BUILDER_DATA_SOURCE_DIR)) { Directory.CreateDirectory(info.BUILDER_DATA_SOURCE_DIR); } if (!Directory.Exists(info.BUILDER_DATA_LOG_DIR)) { Directory.CreateDirectory(info.BUILDER_DATA_LOG_DIR); } info.temp_dir = Path.Combine(temp_dir, i.ToString()); Directory.CreateDirectory(info.temp_dir); // this directory should not exist. // Write all the files to the temporary directory foreach (DBLanefile file in entry.LaneFiles) { File.WriteAllText(Path.Combine(info.temp_dir, file.name), Dos2Unix(file.contents)); } info.command = entry.Command; info.host = response.Host; info.work = entry.Work; info.lane = entry.Lane; info.hostlane = entry.HostLane; info.number = i; info.revision = entry.Revision; info.host_being_worked_for = entry.Host.host; } threads.Clear(); // Start worker threads. for (int i = 0; i < infos.Count; i++) { threads.Add(new Thread(Build)); threads [i].Start(infos [i]); } // Wait until all threads have stopped. // Don't try to abort the threads after a certain time has passed // when threads are aborted they leave things in a pretty messed-up state, // and that pain is worse than the one caused if something hangs. for (int i = 0; i < threads.Count; i++) { threads [i].Join(); } Logger.Log("Finished building {0} work items", list.Count); } catch (Exception ex) { Logger.Log("Exception while building lane: {0}", ex); } finally { // clean up after us if (temp_dir != null && Directory.Exists(temp_dir)) { Directory.Delete(temp_dir, true); } } }