/// <nodoc />
public VsoClient(IIpcLogger logger, DropDaemon dropDaemon)
{
Contract.Requires(dropDaemon?.DropConfig != null);
m_logger = logger;
m_dropDaemon = dropDaemon;
m_config = dropDaemon.DropConfig;
m_cancellationSource = new CancellationTokenSource();
logger.Info("Using drop config: " + JsonConvert.SerializeObject(m_config));
Stats = new DropStatistics();
// instantiate drop client
m_dropClient = new ReloadingDropServiceClient(
logger: logger,
clientConstructor: CreateDropServiceClient);
m_nagleQueue = NagleQueue <AddFileItem> .Create(
maxDegreeOfParallelism : m_config.MaxParallelUploads,
batchSize : m_config.BatchSize,
interval : m_config.NagleTime,
processBatch : ProcessAddFilesAsync);
if (m_config.ArtifactLogName != null)
{
DropAppTraceSource.SingleInstance.SetSourceLevel(System.Diagnostics.SourceLevels.Verbose);
Tracer.AddFileTraceListener(Path.Combine(m_config.LogDir, m_config.ArtifactLogName));
}
}
/// <nodoc />
public DropDaemon(IParser parser, DaemonConfig daemonConfig, DropConfig dropConfig, Task <IDropClient> dropClientTask, IIpcProvider rpcProvider = null, Client client = null)
: base(parser,
daemonConfig,
!string.IsNullOrWhiteSpace(dropConfig.LogDir) ? new FileLogger(dropConfig.LogDir, LogFileName, daemonConfig.Moniker, dropConfig.Verbose, DropDLogPrefix) : daemonConfig.Logger,
rpcProvider,
client)
{
Contract.Requires(dropConfig != null);
DropConfig = dropConfig;
m_logger.Info("Using DropDaemon config: " + JsonConvert.SerializeObject(Config));
m_dropClientTask = dropClientTask ?? Task.Run(() => (IDropClient) new VsoClient(m_logger, dropConfig));
}
/// <nodoc />
public VsoClient(IIpcLogger logger, DropDaemon dropDaemon)
{
Contract.Requires(dropDaemon?.DropConfig != null);
m_logger = logger;
m_dropDaemon = dropDaemon;
m_config = dropDaemon.DropConfig;
m_cancellationSource = new CancellationTokenSource();
logger.Info("Using drop config: " + JsonConvert.SerializeObject(m_config));
Stats = new DropStatistics();
// instantiate drop client
m_dropClient = new ReloadingDropServiceClient(
logger: logger,
clientConstructor: CreateDropServiceClient);
// create dataflow blocks
var groupingOptions = new GroupingDataflowBlockOptions {
Greedy = true
};
var actionOptions = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = m_config.MaxParallelUploads
};
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
m_batchBlock = new BatchBlock <AddFileItem>(m_config.BatchSize, groupingOptions);
m_bufferBlock = new BufferBlock <AddFileItem[]>(); // per http://blog.stephencleary.com/2012/11/async-producerconsumer-queue-using.html, good to have buffer when throttling
m_actionBlock = new ActionBlock <AddFileItem[]>(ProcessAddFilesAsync, actionOptions);
m_batchBlock.LinkTo(m_bufferBlock, linkOptions);
m_bufferBlock.LinkTo(m_actionBlock, linkOptions);
// create and set up timer for triggering the batch block
TimeSpan timerInterval = m_config.NagleTime;
m_batchTimer = new Timer(FlushBatchBlock, null, timerInterval, timerInterval);
if (m_config.ArtifactLogName != null)
{
DropAppTraceSource.SingleInstance.SetSourceLevel(System.Diagnostics.SourceLevels.Verbose);
Tracer.AddFileTraceListener(Path.Combine(m_config.LogDir, m_config.ArtifactLogName));
}
}
/// <nodoc />
public Daemon(IParser parser, DaemonConfig daemonConfig, DropConfig dropConfig, Task <IDropClient> dropClientTask, IIpcProvider rpcProvider = null, Client client = null)
{
Contract.Requires(daemonConfig != null);
Contract.Requires(dropConfig != null);
Config = daemonConfig;
DropConfig = dropConfig;
m_parser = parser;
ApiClient = client;
m_logger = !string.IsNullOrWhiteSpace(dropConfig.LogDir) ? new FileLogger(dropConfig.LogDir, LogFileName, Config.Moniker, dropConfig.Verbose, DropDLogPrefix) : Config.Logger;
m_logger.Info("Using DropDaemon config: " + JsonConvert.SerializeObject(Config));
rpcProvider = rpcProvider ?? IpcFactory.GetProvider();
m_server = rpcProvider.GetServer(Config.Moniker, Config);
m_etwLogger = new BuildXLBasedCloudBuildLogger(Config.Logger, Config.EnableCloudBuildIntegration);
m_dropClientTask = dropClientTask ?? Task.Run(() => (IDropClient) new VsoClient(m_logger, dropConfig));
}
internal static bool VerifyBuildManifestRequirements(ConfiguredCommand conf, DropConfig dropConfig)
{
if (dropConfig.EnableBuildManifestCreation == true)
{
List <string> missingFields = new List <string>();
if (string.IsNullOrEmpty(dropConfig.Repo))
{
missingFields.Add("repo");
}
if (string.IsNullOrEmpty(dropConfig.Branch))
{
missingFields.Add("branch");
}
if (string.IsNullOrEmpty(dropConfig.CommitId))
{
missingFields.Add("commitId");
}
if (string.IsNullOrEmpty(dropConfig.CloudBuildId))
{
missingFields.Add("cloudBuildId");
}
if (string.IsNullOrEmpty(dropConfig.BsiFileLocation))
{
missingFields.Add("BsiFileLocation");
}
if (missingFields.Count != 0)
{
conf.Logger.Error($"EnableBuildManifestCreation set to true, but the following required fields are missing: {string.Join(", ", missingFields)}");
return(false);
}
}
return(true);
}
/// <summary>
/// Checks if provided <see cref="DropConfig"/> contains all required fields for Build Manifest generation and signing.
/// </summary>
/// <returns>True if dropConfig contains all required fields</returns>
public static bool VerifyBuildManifestRequirements(DropConfig dropConfig, out string error)
{
if (dropConfig.GenerateSignedManifest == true)
{
List <string> missingFields = new List <string>();
if (string.IsNullOrEmpty(dropConfig.Repo))
{
missingFields.Add("repo");
}
if (string.IsNullOrEmpty(dropConfig.Branch))
{
missingFields.Add("branch");
}
if (string.IsNullOrEmpty(dropConfig.CommitId))
{
missingFields.Add("commitId");
}
if (string.IsNullOrEmpty(dropConfig.CloudBuildId))
{
missingFields.Add("cloudBuildId");
}
if (string.IsNullOrEmpty(dropConfig.BsiFileLocation))
{
missingFields.Add("BsiFileLocation");
}
if (string.IsNullOrEmpty(dropConfig.MakeCatToolPath))
{
missingFields.Add("MakeCatToolPath");
}
if (string.IsNullOrEmpty(dropConfig.EsrpManifestSignToolPath))
{
missingFields.Add("EsrpManifestSignToolPath");
}
if (missingFields.Count != 0)
{
error = $"GenerateSignedManifest = true, but the following required fields are missing: {string.Join(", ", missingFields)}";
return(false);
}
List <string> missingFiles = new List <string>();
if (!File.Exists(dropConfig.MakeCatToolPath))
{
missingFiles.Add($"MakeCatTool does not exist at : '{dropConfig.MakeCatToolPath}'");
}
if (!File.Exists(dropConfig.EsrpManifestSignToolPath))
{
missingFiles.Add($"EsrpManifestSignTool does not exist at : '{dropConfig.EsrpManifestSignToolPath}'");
}
if (missingFiles.Count != 0)
{
error = $"GenerateSignedManifest = true, but the the following file(s) are missing on disk: {string.Join(", ", missingFiles)}";
return(false);
}
}
error = null;
return(true);
}