public void DownloadFileSafe(DBWorkFile file, string directory)
{
ExecuteSafe("download workfile", delegate()
{
DownloadFile(file, directory);
});
}
public static void WriteToDisk(this DBWorkFile wf, DB db, string dir)
{
byte [] buffer = new byte [1024];
int read;
string filename = Path.Combine(dir, wf.filename);
DBFile file = DBFile_Extensions.Create(db, wf.file_id);
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
using (Stream stream = db.Download(wf)) {
using (FileStream fs = new FileStream(filename, FileMode.Create, FileAccess.Write, FileShare.Read)) {
while (0 != (read = stream.Read(buffer, 0, buffer.Length)))
{
fs.Write(buffer, 0, read);
}
}
}
if (file.compressed_mime == "application/x-gzip")
{
FileUtilities.GZUncompress(filename);
}
}
private void DownloadFile(DBWorkFile file, string directory)
{
string filename = Path.Combine(directory, file.filename);
if (!Directory.Exists(directory))
{
Directory.CreateDirectory(directory);
}
using (WebClient web = new WebClient()) {
web.Headers.Add("Accept-Encoding", "gzip");
web.DownloadFile(CreateWebServiceDownloadUrl(file.id), filename);
if (web.ResponseHeaders ["Content-Encoding"] == "gzip")
{
FileUtilities.GZUncompress(filename);
}
}
}
public Stream Download(DBWorkFile file)
{
return(new DBFileStream(DBFile_Extensions.Create(this, file.file_id), this));
}
private static void Build(List <BuildInfoEntry> list)
{
string temp_dir = null;
List <Thread> threads = new List <Thread> ();
List <BuildInfo> infos = new List <BuildInfo> ();
Logger.Log("Building {0} work items", list.Count);
try {
// Get the path of the temporary directory for this process
using (Process p = Process.GetCurrentProcess()) {
temp_dir = Path.Combine(Path.Combine(Path.GetTempPath(), Configuration.ApplicationName), p.Id.ToString());
if (Directory.Exists(temp_dir))
{
// This should happen very rarely
Directory.Delete(temp_dir, true);
}
Directory.CreateDirectory(temp_dir);
}
for (int i = 0; i < list.Count; i++)
{
BuildInfoEntry entry = list [i];
BuildInfo info;
if (failed_hostlanes != null)
{
foreach (DBHostLane failed in failed_hostlanes)
{
if (failed.id == entry.HostLane.id)
{
Logger.Log("Skipping work, the hostlane {0} has failed work in this run, which has disabled any further work (in this run).", failed.id);
entry = null;
break;
}
}
}
if (entry == null)
{
continue;
}
info = new BuildInfo();
infos.Add(info);
info.lane = entry.Lane;
info.revision = entry.Revision;
// download dependent files
if (entry.FilesToDownload != null)
{
for (int f = 0; f < entry.FilesToDownload.Count; f++)
{
DBWorkFile file = entry.FilesToDownload [f];
DBLane dependent_lane = entry.DependentLaneOfFiles [f];
WebService.DownloadFileSafe(file, Configuration.GetDependentDownloadDirectory(info.lane.id, dependent_lane.lane, info.revision.revision));
}
}
// Set revision-specific paths
info.BUILDER_DATA_INSTALL_DIR = Configuration.GetDataInstallDir(entry.Lane.id, entry.Revision.revision);
info.BUILDER_DATA_LOG_DIR = Configuration.GetDataLogDir(entry.Lane.id, entry.Revision.revision);
info.BUILDER_DATA_SOURCE_DIR = Configuration.GetDataSourceDir(entry.Lane.id, entry.Revision.revision);
info.environment_variables = entry.EnvironmentVariables;
if (!Directory.Exists(info.BUILDER_DATA_SOURCE_DIR))
{
Directory.CreateDirectory(info.BUILDER_DATA_SOURCE_DIR);
}
if (!Directory.Exists(info.BUILDER_DATA_LOG_DIR))
{
Directory.CreateDirectory(info.BUILDER_DATA_LOG_DIR);
}
info.temp_dir = Path.Combine(temp_dir, i.ToString());
Directory.CreateDirectory(info.temp_dir); // this directory should not exist.
// Write all the files to the temporary directory
foreach (DBLanefile file in entry.LaneFiles)
{
File.WriteAllText(Path.Combine(info.temp_dir, file.name), Dos2Unix(file.contents));
}
info.command = entry.Command;
info.host = response.Host;
info.work = entry.Work;
info.lane = entry.Lane;
info.hostlane = entry.HostLane;
info.number = i;
info.revision = entry.Revision;
info.host_being_worked_for = entry.Host.host;
}
threads.Clear();
// Start worker threads.
for (int i = 0; i < infos.Count; i++)
{
threads.Add(new Thread(Build));
threads [i].Start(infos [i]);
}
// Wait until all threads have stopped.
// Don't try to abort the threads after a certain time has passed
// when threads are aborted they leave things in a pretty messed-up state,
// and that pain is worse than the one caused if something hangs.
for (int i = 0; i < threads.Count; i++)
{
threads [i].Join();
}
Logger.Log("Finished building {0} work items", list.Count);
} catch (Exception ex) {
Logger.Log("Exception while building lane: {0}", ex);
} finally {
// clean up after us
if (temp_dir != null && Directory.Exists(temp_dir))
{
Directory.Delete(temp_dir, true);
}
}
}