Exemplo n.º 1
0
 public void DownloadFileSafe(DBWorkFile file, string directory)
 {
     ExecuteSafe("download workfile", delegate()
     {
         DownloadFile(file, directory);
     });
 }
Exemplo n.º 2
0
        public static void WriteToDisk(this DBWorkFile wf, DB db, string dir)
        {
            byte [] buffer = new byte [1024];
            int     read;
            string  filename = Path.Combine(dir, wf.filename);
            DBFile  file     = DBFile_Extensions.Create(db, wf.file_id);

            if (!Directory.Exists(dir))
            {
                Directory.CreateDirectory(dir);
            }

            using (Stream stream = db.Download(wf)) {
                using (FileStream fs = new FileStream(filename, FileMode.Create, FileAccess.Write, FileShare.Read)) {
                    while (0 != (read = stream.Read(buffer, 0, buffer.Length)))
                    {
                        fs.Write(buffer, 0, read);
                    }
                }
            }

            if (file.compressed_mime == "application/x-gzip")
            {
                FileUtilities.GZUncompress(filename);
            }
        }
Exemplo n.º 3
0
        private void DownloadFile(DBWorkFile file, string directory)
        {
            string filename = Path.Combine(directory, file.filename);

            if (!Directory.Exists(directory))
            {
                Directory.CreateDirectory(directory);
            }

            using (WebClient web = new WebClient()) {
                web.Headers.Add("Accept-Encoding", "gzip");
                web.DownloadFile(CreateWebServiceDownloadUrl(file.id), filename);

                if (web.ResponseHeaders ["Content-Encoding"] == "gzip")
                {
                    FileUtilities.GZUncompress(filename);
                }
            }
        }
Exemplo n.º 4
0
 public Stream Download(DBWorkFile file)
 {
     return(new DBFileStream(DBFile_Extensions.Create(this, file.file_id), this));
 }
Exemplo n.º 5
0
        private static void Build(List <BuildInfoEntry> list)
        {
            string           temp_dir = null;
            List <Thread>    threads  = new List <Thread> ();
            List <BuildInfo> infos    = new List <BuildInfo> ();

            Logger.Log("Building {0} work items", list.Count);

            try {
                // Get the path of the temporary directory for this process
                using (Process p = Process.GetCurrentProcess()) {
                    temp_dir = Path.Combine(Path.Combine(Path.GetTempPath(), Configuration.ApplicationName), p.Id.ToString());
                    if (Directory.Exists(temp_dir))
                    {
                        // This should happen very rarely
                        Directory.Delete(temp_dir, true);
                    }
                    Directory.CreateDirectory(temp_dir);
                }

                for (int i = 0; i < list.Count; i++)
                {
                    BuildInfoEntry entry = list [i];
                    BuildInfo      info;

                    if (failed_hostlanes != null)
                    {
                        foreach (DBHostLane failed in failed_hostlanes)
                        {
                            if (failed.id == entry.HostLane.id)
                            {
                                Logger.Log("Skipping work, the hostlane {0} has failed work in this run, which has disabled any further work (in this run).", failed.id);
                                entry = null;
                                break;
                            }
                        }
                    }

                    if (entry == null)
                    {
                        continue;
                    }

                    info = new BuildInfo();

                    infos.Add(info);

                    info.lane     = entry.Lane;
                    info.revision = entry.Revision;

                    // download dependent files
                    if (entry.FilesToDownload != null)
                    {
                        for (int f = 0; f < entry.FilesToDownload.Count; f++)
                        {
                            DBWorkFile file           = entry.FilesToDownload [f];
                            DBLane     dependent_lane = entry.DependentLaneOfFiles [f];
                            WebService.DownloadFileSafe(file, Configuration.GetDependentDownloadDirectory(info.lane.id, dependent_lane.lane, info.revision.revision));
                        }
                    }

                    // Set revision-specific paths
                    info.BUILDER_DATA_INSTALL_DIR = Configuration.GetDataInstallDir(entry.Lane.id, entry.Revision.revision);
                    info.BUILDER_DATA_LOG_DIR     = Configuration.GetDataLogDir(entry.Lane.id, entry.Revision.revision);
                    info.BUILDER_DATA_SOURCE_DIR  = Configuration.GetDataSourceDir(entry.Lane.id, entry.Revision.revision);
                    info.environment_variables    = entry.EnvironmentVariables;

                    if (!Directory.Exists(info.BUILDER_DATA_SOURCE_DIR))
                    {
                        Directory.CreateDirectory(info.BUILDER_DATA_SOURCE_DIR);
                    }
                    if (!Directory.Exists(info.BUILDER_DATA_LOG_DIR))
                    {
                        Directory.CreateDirectory(info.BUILDER_DATA_LOG_DIR);
                    }

                    info.temp_dir = Path.Combine(temp_dir, i.ToString());
                    Directory.CreateDirectory(info.temp_dir);                      // this directory should not exist.

                    // Write all the files to the temporary directory
                    foreach (DBLanefile file in entry.LaneFiles)
                    {
                        File.WriteAllText(Path.Combine(info.temp_dir, file.name), Dos2Unix(file.contents));
                    }

                    info.command  = entry.Command;
                    info.host     = response.Host;
                    info.work     = entry.Work;
                    info.lane     = entry.Lane;
                    info.hostlane = entry.HostLane;
                    info.number   = i;
                    info.revision = entry.Revision;
                    info.host_being_worked_for = entry.Host.host;
                }

                threads.Clear();

                // Start worker threads.
                for (int i = 0; i < infos.Count; i++)
                {
                    threads.Add(new Thread(Build));
                    threads [i].Start(infos [i]);
                }

                // Wait until all threads have stopped.
                // Don't try to abort the threads after a certain time has passed
                // when threads are aborted they leave things in a pretty messed-up state,
                // and that pain is worse than the one caused if something hangs.
                for (int i = 0; i < threads.Count; i++)
                {
                    threads [i].Join();
                }

                Logger.Log("Finished building {0} work items", list.Count);
            } catch (Exception ex) {
                Logger.Log("Exception while building lane: {0}", ex);
            } finally {
                // clean up after us
                if (temp_dir != null && Directory.Exists(temp_dir))
                {
                    Directory.Delete(temp_dir, true);
                }
            }
        }