/// <nodoc /> public VsoClient(IIpcLogger logger, DropDaemon dropDaemon) { Contract.Requires(dropDaemon?.DropConfig != null); m_logger = logger; m_dropDaemon = dropDaemon; m_config = dropDaemon.DropConfig; m_cancellationSource = new CancellationTokenSource(); logger.Info("Using drop config: " + JsonConvert.SerializeObject(m_config)); Stats = new DropStatistics(); // instantiate drop client m_dropClient = new ReloadingDropServiceClient( logger: logger, clientConstructor: CreateDropServiceClient); m_nagleQueue = NagleQueue <AddFileItem> .Create( maxDegreeOfParallelism : m_config.MaxParallelUploads, batchSize : m_config.BatchSize, interval : m_config.NagleTime, processBatch : ProcessAddFilesAsync); if (m_config.ArtifactLogName != null) { DropAppTraceSource.SingleInstance.SetSourceLevel(System.Diagnostics.SourceLevels.Verbose); Tracer.AddFileTraceListener(Path.Combine(m_config.LogDir, m_config.ArtifactLogName)); } }
/// <nodoc /> public DropDaemon(IParser parser, DaemonConfig daemonConfig, DropConfig dropConfig, Task <IDropClient> dropClientTask, IIpcProvider rpcProvider = null, Client client = null) : base(parser, daemonConfig, !string.IsNullOrWhiteSpace(dropConfig.LogDir) ? new FileLogger(dropConfig.LogDir, LogFileName, daemonConfig.Moniker, dropConfig.Verbose, DropDLogPrefix) : daemonConfig.Logger, rpcProvider, client) { Contract.Requires(dropConfig != null); DropConfig = dropConfig; m_logger.Info("Using DropDaemon config: " + JsonConvert.SerializeObject(Config)); m_dropClientTask = dropClientTask ?? Task.Run(() => (IDropClient) new VsoClient(m_logger, dropConfig)); }
/// <nodoc /> public VsoClient(IIpcLogger logger, DropDaemon dropDaemon) { Contract.Requires(dropDaemon?.DropConfig != null); m_logger = logger; m_dropDaemon = dropDaemon; m_config = dropDaemon.DropConfig; m_cancellationSource = new CancellationTokenSource(); logger.Info("Using drop config: " + JsonConvert.SerializeObject(m_config)); Stats = new DropStatistics(); // instantiate drop client m_dropClient = new ReloadingDropServiceClient( logger: logger, clientConstructor: CreateDropServiceClient); // create dataflow blocks var groupingOptions = new GroupingDataflowBlockOptions { Greedy = true }; var actionOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = m_config.MaxParallelUploads }; var linkOptions = new DataflowLinkOptions { PropagateCompletion = true }; m_batchBlock = new BatchBlock <AddFileItem>(m_config.BatchSize, groupingOptions); m_bufferBlock = new BufferBlock <AddFileItem[]>(); // per http://blog.stephencleary.com/2012/11/async-producerconsumer-queue-using.html, good to have buffer when throttling m_actionBlock = new ActionBlock <AddFileItem[]>(ProcessAddFilesAsync, actionOptions); m_batchBlock.LinkTo(m_bufferBlock, linkOptions); m_bufferBlock.LinkTo(m_actionBlock, linkOptions); // create and set up timer for triggering the batch block TimeSpan timerInterval = m_config.NagleTime; m_batchTimer = new Timer(FlushBatchBlock, null, timerInterval, timerInterval); if (m_config.ArtifactLogName != null) { DropAppTraceSource.SingleInstance.SetSourceLevel(System.Diagnostics.SourceLevels.Verbose); Tracer.AddFileTraceListener(Path.Combine(m_config.LogDir, m_config.ArtifactLogName)); } }
/// <nodoc /> public Daemon(IParser parser, DaemonConfig daemonConfig, DropConfig dropConfig, Task <IDropClient> dropClientTask, IIpcProvider rpcProvider = null, Client client = null) { Contract.Requires(daemonConfig != null); Contract.Requires(dropConfig != null); Config = daemonConfig; DropConfig = dropConfig; m_parser = parser; ApiClient = client; m_logger = !string.IsNullOrWhiteSpace(dropConfig.LogDir) ? new FileLogger(dropConfig.LogDir, LogFileName, Config.Moniker, dropConfig.Verbose, DropDLogPrefix) : Config.Logger; m_logger.Info("Using DropDaemon config: " + JsonConvert.SerializeObject(Config)); rpcProvider = rpcProvider ?? IpcFactory.GetProvider(); m_server = rpcProvider.GetServer(Config.Moniker, Config); m_etwLogger = new BuildXLBasedCloudBuildLogger(Config.Logger, Config.EnableCloudBuildIntegration); m_dropClientTask = dropClientTask ?? Task.Run(() => (IDropClient) new VsoClient(m_logger, dropConfig)); }
internal static bool VerifyBuildManifestRequirements(ConfiguredCommand conf, DropConfig dropConfig) { if (dropConfig.EnableBuildManifestCreation == true) { List <string> missingFields = new List <string>(); if (string.IsNullOrEmpty(dropConfig.Repo)) { missingFields.Add("repo"); } if (string.IsNullOrEmpty(dropConfig.Branch)) { missingFields.Add("branch"); } if (string.IsNullOrEmpty(dropConfig.CommitId)) { missingFields.Add("commitId"); } if (string.IsNullOrEmpty(dropConfig.CloudBuildId)) { missingFields.Add("cloudBuildId"); } if (string.IsNullOrEmpty(dropConfig.BsiFileLocation)) { missingFields.Add("BsiFileLocation"); } if (missingFields.Count != 0) { conf.Logger.Error($"EnableBuildManifestCreation set to true, but the following required fields are missing: {string.Join(", ", missingFields)}"); return(false); } } return(true); }
/// <summary> /// Checks if provided <see cref="DropConfig"/> contains all required fields for Build Manifest generation and signing. /// </summary> /// <returns>True if dropConfig contains all required fields</returns> public static bool VerifyBuildManifestRequirements(DropConfig dropConfig, out string error) { if (dropConfig.GenerateSignedManifest == true) { List <string> missingFields = new List <string>(); if (string.IsNullOrEmpty(dropConfig.Repo)) { missingFields.Add("repo"); } if (string.IsNullOrEmpty(dropConfig.Branch)) { missingFields.Add("branch"); } if (string.IsNullOrEmpty(dropConfig.CommitId)) { missingFields.Add("commitId"); } if (string.IsNullOrEmpty(dropConfig.CloudBuildId)) { missingFields.Add("cloudBuildId"); } if (string.IsNullOrEmpty(dropConfig.BsiFileLocation)) { missingFields.Add("BsiFileLocation"); } if (string.IsNullOrEmpty(dropConfig.MakeCatToolPath)) { missingFields.Add("MakeCatToolPath"); } if (string.IsNullOrEmpty(dropConfig.EsrpManifestSignToolPath)) { missingFields.Add("EsrpManifestSignToolPath"); } if (missingFields.Count != 0) { error = $"GenerateSignedManifest = true, but the following required fields are missing: {string.Join(", ", missingFields)}"; return(false); } List <string> missingFiles = new List <string>(); if (!File.Exists(dropConfig.MakeCatToolPath)) { missingFiles.Add($"MakeCatTool does not exist at : '{dropConfig.MakeCatToolPath}'"); } if (!File.Exists(dropConfig.EsrpManifestSignToolPath)) { missingFiles.Add($"EsrpManifestSignTool does not exist at : '{dropConfig.EsrpManifestSignToolPath}'"); } if (missingFiles.Count != 0) { error = $"GenerateSignedManifest = true, but the the following file(s) are missing on disk: {string.Join(", ", missingFiles)}"; return(false); } } error = null; return(true); }