public static StageOutput<R, T> BinaryExpression<S1, S2, R, T>(this StageOutput<S1, T> stream1, StageOutput<S2, T> stream2,
Expression<Func<S1, int>> keyFunction1, Expression<Func<S2, int>> keyFunction2,
Expression<Func<IEnumerable<S1>, IEnumerable<S2>, IEnumerable<R>>> transformation, string name)
where T : Time<T>
{
return BinaryBufferingShard<S1, S2, R, T>.MakeStage(stream1, stream2, (i, v) => new Naiad.Frameworks.BinaryBufferingShard<S1, S2, R, T>(i, v, transformation), keyFunction1, keyFunction2, null, name);
}
public static StageOutput<Pair<K, X>, T> LocalReduce<A, X, R, S, K, I, T>(
this StageOutput<I, T> port, Func<I, K> key, Func<I, R> val, Func<A> factory, string name)
where A : IReducer<X, R, S>
where T : Time<T>
{
return LocalReduce<A, X, R, S, K, I, T>(port, key, val, factory, name, null, null);
}
public static StageOutput<Pair<K, S>, T> LocalCombine<A, X, R, S, K, T>(
this StageOutput<Pair<K, X>, T> port, Func<A> factory, string name)
where A : IReducer<X, R, S>
where T : Time<T>
{
return port.LocalCombine<A, X, R, S, K, T>(factory, name, null);
}
public static StageOutput<X, T> BroadcastReduce<A, X, T>(this StageOutput<X, T> port, Func<A> factory, string name)
where A : IReducer<X, X, X>
where X : Cloneable<X>
where T : Time<T>
{
return port.LocalReduce<A, X, X, X, T>(factory, name + "Reduce").Broadcast().LocalCombine<A, X, X, X, T>(factory, name + "Combine");
}
public static StageOutput<S, T> Concat<S, T>(this StageOutput<S, T> stream1, StageOutput<S, T> stream2)
where T : Time<T>
{
// test to see if they are partitioned properly, and if so maintain the information in output partitionedby information.
var partitionedBy = Naiad.CodeGeneration.ExpressionComparer.Instance.Equals(stream1.PartitionedBy, stream2.PartitionedBy) ? stream1.PartitionedBy : null;
return BinaryStreamingShard<S, S, S, T>.MakeStage(stream1, stream2, (i, v) => new Concat<S, T>(i, v), partitionedBy, partitionedBy, partitionedBy, "Concat");
}
public ProgressChannel(int producerPlacementCount,
ProgressUpdateConsumer consumerVertex,
StageOutput <Update, Empty> stream,
StageInput <Update, Empty> recvPort,
InternalController controller,
int channelId)
{
this.sendBundle = stream;
this.recvBundle = recvPort;
this.channelID = channelId;
var computation = sendBundle.ForStage.InternalComputation;
var recvFiber = this.recvBundle.GetPin(computation.Controller.Configuration.ProcessID);
this.mailbox = new Mailbox(recvFiber.Vertex.Scheduler.State(computation).PostOffice, consumerVertex, this.channelID, consumerVertex.VertexId, producerPlacementCount);
// recvFiber.Vertex.Scheduler.State(graphManager).PostOffice.RegisterMailbox(this.mailbox);
this.postboxes = new Dictionary <int, Fiber>();
foreach (VertexLocation loc in sendBundle.ForStage.Placement)
{
if (loc.ProcessId == controller.Configuration.ProcessID)
{
this.postboxes[loc.VertexId] = new Fiber(this.channelID, loc.VertexId, this.sendBundle.GetFiber(loc.VertexId), this.mailbox, controller);
}
}
if (controller.NetworkChannel != null)
{
controller.NetworkChannel.RegisterMailbox(this.mailbox);
}
Logging.Info("Allocated progress channel [{0}]: {1} -> {2}", this.channelID, sendBundle, recvBundle);
}
public static StageOutput<R, T> Broadcast<R, T>(this StageOutput<R, T> port)
where R : Cloneable<R>
where T : Time<T>
{
var controller = port.Context.Manager.Controller;
int threadCount = controller.DefaultPlacement.Count / controller.Configuration.Processes;
if (threadCount * controller.Configuration.Processes != controller.DefaultPlacement.Count)
{
throw new Exception("Uneven thread count?");
}
var processDests = controller.DefaultPlacement.Where(x => x.ThreadId == 0).Select(x => x.ShardId).ToArray();
var boutput = UnaryStreamingShard<R, Pair<int, R>, T>.MakeStage(port, (i, v) => new BroadcastSendShard<R, T>(i, v, processDests), null, null, "BroadcastProcessSend");
var collectable = boutput;
if (controller.DefaultPlacement.Where(x => x.ProcessId == controller.Configuration.ProcessID).Count() > 1)
{
var threadDests = controller.DefaultPlacement
.Where(x => x.ProcessId == controller.Configuration.ProcessID)
.Select(x => x.ShardId)
.ToArray();
collectable = UnaryStreamingShard<Pair<int, R>, Pair<int, R>, T>.MakeStage(boutput, (i, v) => new BroadcastForwardShard<R, T>(i, v, threadDests), x => x.s, null, "BroadcastShardSend");
}
return Naiad.Frameworks.Linq.ExtensionMethods.Select(collectable, x => x.t);
}
/// <summary>
/// Constructor
/// </summary>
public CentralizedProgressChannel(Stage <Runtime.Progress.ProgressUpdateCentralizer, Pointstamp> consumer,
StageOutput <Int64, Pointstamp> stream, StageInput <Int64, Pointstamp> recvPort,
InternalController controller,
int channelId)
{
this.consumer = consumer;
this.sendBundle = stream; // producer.Output;
this.recvBundle = recvPort; // consumer.Input;
this.postboxes = new Dictionary <int, Fiber>();
this.channelID = channelId;
// Get the shard id and process id of the single consumer
var consumerShardId = consumer.Placement.Single().VertexId;
var consumerProcessId = consumer.Placement.Single().ProcessId;
var graphManager = sendBundle.ForStage.InternalGraphManager;
if (debug)
{
Console.Error.WriteLine(" IncastChannel create ProcessId = {0}", graphManager.Controller.Configuration.ProcessID);
}
var myProcessId = graphManager.Controller.Configuration.ProcessID;
if (myProcessId == consumerProcessId)
{
if (debug)
{
Console.Error.WriteLine(" IncastChannel creating receive mailbox");
}
VertexInput <Int64, Pointstamp> recvFiber = this.recvBundle.GetPin(consumerProcessId);
this.mailbox = new Mailbox(recvFiber.Vertex.Scheduler.State(graphManager).PostOffice,
consumer.GetShard(consumerShardId), this.channelID, consumerShardId);
recvFiber.Vertex.Scheduler.State(graphManager).PostOffice.RegisterMailbox(this.mailbox);
if (controller.NetworkChannel != null)
{
controller.NetworkChannel.RegisterMailbox(this.mailbox);
}
}
foreach (VertexLocation loc in sendBundle.ForStage.Placement)
{
if (loc.ProcessId == sendBundle.ForStage.InternalGraphManager.Controller.Configuration.ProcessID)
{
if (debug)
{
Console.Error.WriteLine(" IncastChannel loc = {0}/{1}/{2}", loc.ProcessId, loc.VertexId, loc.ThreadId);
}
var postbox = new Fiber(this.channelID, loc.VertexId, this.sendBundle.GetFiber(loc.VertexId),
this.mailbox, controller, consumerShardId, consumerProcessId);
this.postboxes[loc.VertexId] = postbox;
}
}
Logging.Info("Allocated incast channel [{0}]: {1} -> {2}", this.channelID, sendBundle, recvBundle);
}
public static StageOutput<Pair<K, X>, T> LocalTimeReduce<A, X, R, S, K, I, T>(
this StageOutput<I, T> port, Func<I, K> key, Func<I, R> val, Func<A> factory, string name,
Expression<Func<I, int>> inPlacement, Expression<Func<Pair<K, X>, int>> outPlacement)
where A : IReducer<X, R, S>
where T : Time<T>
{
return UnaryStreamingShard<I, Pair<K, X>, T>.MakeStage(port, (i, v) => new LocalTimeKeyedReduceShard<A, X, R, S, K, I, T>(i, v, key, val, factory), inPlacement, outPlacement, name);
}
/// <summary>
/// Writes a new blank line
/// </summary>
protected void AOT_NewLine()
{
StageOutput.Add(LineOutColored.NewLine());
if (ShouldLogToConsoleRealTime)
{
Console.WriteLine();
}
}
/// <summary>
/// Writes the given text as normal output to both BuildStage output and to the console real time.
/// </summary>
/// <param name="text"></param>
protected void AOT_Normal(string text, Color textColor)
{
StageOutput.Add(LineOutColored.Normal(text, textColor));
if (ShouldLogToConsoleRealTime)
{
Console.WriteLine(text, textColor);
}
}
/// <summary>
/// Writes the given text as Informational output to both BuildStage output and to the console real time.
/// </summary>
/// <param name="text"></param>
protected void AOT_Info(string text)
{
StageOutput.Add(LineOutColored.Info(text));
if (ShouldLogToConsoleRealTime)
{
Console.WriteLine(text, Color.Cyan);
}
}
/// <summary>
/// Writes the given text as warning output to both BuildStage output and to the console real time.
/// </summary>
/// <param name="text"></param>
protected void AOT_Warning(string text)
{
StageOutput.Add(LineOutColored.Warning(text));
if (ShouldLogToConsoleRealTime)
{
Console.WriteLine(text, Color.Yellow);
}
}
/// <summary>
/// Writes the given text as Errored output to both BuildStage output and to the console real time.
/// </summary>
/// <param name="text"></param>
protected void AOT_Error(string text)
{
StageOutput.Add(LineOutColored.Error(text));
if (ShouldLogToConsoleRealTime)
{
Console.WriteLine(text, Color.Red);
}
}
/// <summary>
/// Writes the given text as Success output to both BuildStage output and to the console real time.
/// </summary>
/// <param name="text"></param>
public void AOT_Success(string text)
{
StageOutput.Add(LineOutColored.Success(text));
if (ShouldLogToConsoleRealTime)
{
Console.WriteLine(text, Color.Green);
}
}
internal static bool PartitionedEquivalently <S, T>(StageOutput <S, T> sender, StageInput <S, T> receiver)
where T : Time <T>
{
return(sender.ForStage.Placement.Equals(receiver.ForStage.Placement) &&
sender.PartitionedBy != null &&
receiver.PartitionedBy != null &&
Utilities.ExpressionComparer.Instance.Equals(sender.PartitionedBy, receiver.PartitionedBy));
}
public PostOfficeChannel(StageOutput <S, T> sendBundle, StageInput <S, T> recvBundle, Action <S[], int[], int> routingHashcodeFunction, NetworkChannel networkChannel, int channelId, Channel.Flags flags)
{
if ((flags & Channel.Flags.SpillOnRecv) == Channel.Flags.SpillOnRecv)
{
throw new Exception("SpillingLocalMailbox not currently supported");
}
this.sendBundle = sendBundle;
this.recvBundle = recvBundle;
this.channelID = channelId;
var computation = sendBundle.ForStage.InternalComputation;
// populate mailboxes; [proxy] destinations for sent messages.
this.mailboxes = new Mailbox <S, T> [recvBundle.ForStage.Placement.Count];
foreach (VertexLocation loc in recvBundle.ForStage.Placement)
{
if (loc.ProcessId == computation.Controller.Configuration.ProcessID)
{
var postOffice = this.recvBundle.GetPin(loc.VertexId).Vertex.Scheduler.State(computation).PostOffice;
var progressBuffer = new Runtime.Progress.ProgressUpdateBuffer <T>(this.ChannelId, this.recvBundle.GetPin(loc.VertexId).Vertex.Scheduler.State(computation).Producer);
LocalMailbox <S, T> localMailbox = new LocalMailbox <S, T>(postOffice, this.recvBundle.GetPin(loc.VertexId), channelID, loc.VertexId, progressBuffer);
this.mailboxes[loc.VertexId] = localMailbox;
postOffice.RegisterMailbox(localMailbox);
if (networkChannel != null)
{
networkChannel.RegisterMailbox(localMailbox);
}
}
else
{
this.mailboxes[loc.VertexId] = new RemoteMailbox <S, T>(this.channelID, loc.ProcessId, loc.VertexId, sendBundle.ForStage.InternalComputation);
}
}
// populate postboxes; collection points for each local worker.
this.postboxes = new Dictionary <int, Postbox <S, T> >();
foreach (VertexLocation location in sendBundle.ForStage.Placement)
{
if (location.ProcessId == sendBundle.ForStage.InternalComputation.Controller.Configuration.ProcessID)
{
var progressBuffer = new Runtime.Progress.ProgressUpdateBuffer <T>(this.ChannelId, this.sendBundle.GetFiber(location.VertexId).Vertex.Scheduler.State(computation).Producer);
// determine type of postbox to use.
if (routingHashcodeFunction == null)
{
this.postboxes[location.VertexId] = new NoHashCodePostbox <S, T>(this.channelID, this.sendBundle.GetFiber(location.VertexId), this.recvBundle, this.mailboxes, networkChannel, progressBuffer);
}
else
{
this.postboxes[location.VertexId] = new BufferingPostbox <S, T>(this.channelID, this.sendBundle.GetFiber(location.VertexId), this.recvBundle, this.mailboxes, routingHashcodeFunction, networkChannel, progressBuffer);
}
}
}
}
public static StageOutput<Pair<K, S>, T> Reduce<A, X, R, S, K, I, T>(
this StageOutput<I, T> port, Func<I, K> key, Func<I, R> val, Func<A> factory, string name)
where A : IReducer<X, R, S>
where T : Time<T>
{
return port.
LocalReduce<A, X, R, S, K, I, T>(key, val, factory, name + "Reduce", null, null).
LocalCombine<A, X, R, S, K, T>(factory, name + "Combine", x => x.s.GetHashCode());
}
public PostOfficeChannel(StageOutput <S, T> sendBundle, StageInput <S, T> recvBundle, Func <S, int> routingHashcodeFunction, NetworkChannel networkChannel, int channelId, Channel.Flags flags)
{
this.sendBundle = sendBundle;
this.recvBundle = recvBundle;
this.postboxes = new Dictionary <int, Postbox <S, T> >();
this.mailboxes = new Mailbox <S, T> [recvBundle.ForStage.Placement.Count];
this.channelID = channelId;
var computation = sendBundle.ForStage.InternalComputation;
foreach (VertexLocation loc in recvBundle.ForStage.Placement)
{
if (loc.ProcessId == computation.Controller.Configuration.ProcessID)
{
var postOffice = this.recvBundle.GetPin(loc.VertexId).Vertex.Scheduler.State(computation).PostOffice;
LocalMailbox <S, T> localMailbox;
var progressBuffer = new Runtime.Progress.ProgressUpdateBuffer <T>(this.ChannelId, this.recvBundle.GetPin(loc.VertexId).Vertex.Scheduler.State(computation).Producer);
if ((flags & Channel.Flags.SpillOnRecv) == Channel.Flags.SpillOnRecv)
{
//localMailbox = new SpillingLocalMailbox<S, T>(postOffice, this.recvBundle.GetPin(loc.VertexId), channelID, loc.VertexId, progressBuffer);
throw new Exception("SpillingLocalMailbox not currently supported");
}
else
{
localMailbox = new LegacyLocalMailbox <S, T>(postOffice, this.recvBundle.GetPin(loc.VertexId), channelID, loc.VertexId, progressBuffer);
}
this.mailboxes[loc.VertexId] = localMailbox;
postOffice.RegisterMailbox(localMailbox);
if (networkChannel != null)
{
networkChannel.RegisterMailbox(localMailbox);
}
}
else
{
this.mailboxes[loc.VertexId] = new RemoteMailbox <S, T>(this.channelID, loc.ProcessId, loc.VertexId, sendBundle.ForStage.InternalComputation);
}
}
foreach (VertexLocation loc in sendBundle.ForStage.Placement)
{
if (loc.ProcessId == sendBundle.ForStage.InternalComputation.Controller.Configuration.ProcessID)
{
var progressBuffer = new Runtime.Progress.ProgressUpdateBuffer <T>(this.ChannelId, this.sendBundle.GetFiber(loc.VertexId).Vertex.Scheduler.State(computation).Producer);
this.postboxes[loc.VertexId] = new Postbox <S, T>(this.channelID, this.sendBundle.GetFiber(loc.VertexId), this.recvBundle, this.mailboxes, routingHashcodeFunction, networkChannel, progressBuffer);
}
}
}
/// <summary>
/// Run Restore Process
/// </summary>
/// <returns></returns>
protected override StageCompletionStatusEnum ExecuteProcess()
{
DotNetRestoreSettings settings = new DotNetRestoreSettings();
settings.ProjectFile = CISession.Solution;
settings.Verbosity = DotNetVerbosity.Minimal;
(BlockingCollection <ILineOut> outputs, int exitCode) = DotNetRestore(settings);
StageOutput.AddRange(outputs);
ControlFlow.Assert(exitCode == 0, "Process DotNetRestore failed");
return(StageCompletionStatusEnum.Success);
}
public static StageOutput<Pair<K, S>, T> LocalTimeCombine<A, X, R, S, K, T>(
this StageOutput<Pair<K, X>, T> port, Func<A> factory, string name,
Expression<Func<Pair<K, S>, int>> outPlacement)
where A : IReducer<X, R, S>
where T : Time<T>
{
Expression<Func<Pair<K, X>, int>> inPlacement = null;
if (outPlacement != null)
{
inPlacement = x => x.s.GetHashCode();
}
return UnaryStreamingShard<Pair<K, X>, Pair<K, S>, T>.MakeStage(port, (i, v) => new LocalTimeKeyedCombineShard<A, X, R, S, K, T>(i, v, factory), inPlacement, outPlacement, name);
}
/// <summary>
/// Constructor
/// </summary>
public CentralizedProgressChannel(Stage <ProgressUpdateCentralizer, Empty> consumer,
StageOutput <Update, Empty> stream, StageInput <Update, Empty> recvPort,
InternalController controller,
int channelId)
{
this.consumer = consumer;
this.sendBundle = stream; // producer.Output;
this.recvBundle = recvPort; // consumer.Input;
this.postboxes = new Dictionary <int, Fiber>();
this.channelID = channelId;
// Get the vertex id and process id of the single consumer
var consumerVertexId = consumer.Placement.Single().VertexId;
var consumerProcessId = consumer.Placement.Single().ProcessId;
var computation = sendBundle.ForStage.InternalComputation;
var myProcessId = computation.Controller.Configuration.ProcessID;
if (myProcessId == consumerProcessId)
{
VertexInput <Update, Empty> recvFiber = this.recvBundle.GetPin(consumerProcessId);
this.mailbox = new Mailbox(recvFiber.Vertex.Scheduler.State(computation).PostOffice,
consumer.GetVertex(consumerVertexId), this.channelID, consumerVertexId);
//recvFiber.Vertex.Scheduler.State(computation).PostOffice.RegisterMailbox(this.mailbox);
if (controller.NetworkChannel != null)
{
controller.NetworkChannel.RegisterMailbox(this.mailbox);
}
}
foreach (VertexLocation loc in sendBundle.ForStage.Placement)
{
if (loc.ProcessId == sendBundle.ForStage.InternalComputation.Controller.Configuration.ProcessID)
{
var postbox = new Fiber(this.channelID, loc.VertexId, this.sendBundle.GetFiber(loc.VertexId),
this.mailbox, controller, consumerVertexId, consumerProcessId);
this.postboxes[loc.VertexId] = postbox;
}
}
Logging.Info("Allocated CentralizedProgressChannel [{0}]: {1} -> {2}", this.channelID, sendBundle, recvBundle);
NaiadTracing.Trace.ChannelInfo(ChannelId, SourceStage.StageId, DestinationStage.StageId, true, true);
}
public PipelineChannel(StageOutput <S, T> sender, StageInput <S, T> receiver, int channelId)
{
this.sender = sender;
this.receiver = receiver;
this.channelId = channelId;
this.subChannels = new Dictionary <int, Fiber>();
foreach (VertexLocation loc in sender.ForStage.Placement)
{
if (loc.ProcessId == sender.ForStage.InternalComputation.Controller.Configuration.ProcessID)
{
this.subChannels[loc.VertexId] = new Fiber(this, receiver.GetPin(loc.VertexId), loc.VertexId);
}
}
}
/// <summary>
/// Writes the given text as Errored output to both BuildStage output and to the console real time.
/// </summary>
/// <param name="text"></param>
protected void AOT_Error(Exception exception)
{
string exceptionSeparator = "************************ [ Exception Encountered ] ************************";
int start = StageOutput.Count;
StageOutput.Add(LineOutColored.Error(exceptionSeparator));
StageOutput.Add(LineOutColored.Error(exception.Message));
StageOutput.Add(LineOutColored.Error(exceptionSeparator));
StageOutput.Add(LineOutColored.NewLine());
StageOutput.Add(LineOutColored.Error(exception.ToString()));
StageOutput.Add(LineOutColored.NewLine());
if (ShouldLogToConsoleRealTime)
{
Print_StageOutput(start);
}
}
public static StageOutput<S, T> Iterate<S, T>(this StageOutput<S, T> input, Func<Naiad.Dataflow.Iteration.Context<T>, StageOutput<S, IterationIn<T>>, StageOutput<S, IterationIn<T>>> function, Expression<Func<S, int>> initialIteration, Expression<Func<S, int>> partitionedBy, int iterations, string name)
where T : Time<T>
{
var helper = new Naiad.Dataflow.Iteration.Context<T>(input.Context, name);
var delayed = helper.Delay<S>(partitionedBy, iterations);
var ingress = Naiad.Dataflow.PartitionBy.ExtensionMethods.PartitionBy(helper.EnterLoop(input, initialIteration.Compile()), partitionedBy);
var loopHead = Naiad.Frameworks.Linq.ExtensionMethods.Concat(ingress, delayed.Output);
var loopTail = function(helper, loopHead);
delayed.Input = loopTail;
return helper.ExitLoop(loopTail);
//return helper.ExitLoop(loopTail, iterations);
}
/// <summary>
/// Run the Angular Build
/// </summary>
/// <returns></returns>
protected override StageCompletionStatusEnum ExecuteProcess()
{
string command = "ng";
string ngArgs = "build";
if (CISession.SkipAngularBuild)
{
return(StageCompletionStatusEnum.Skipped);
}
CompletionStatus = StageCompletionStatusEnum.InProcess;
foreach (AngularProject project in CISession.SlugCIConfigObj.AngularProjects)
{
AddOutputText("Project: " + project.Name, OutputType.Std);
AbsolutePath angularProjectPath = CISession.AngularDirectory / project.Name;
IProcess process = ProcessTasks.StartProcess(command, ngArgs, angularProjectPath, customLogger: AngularLogger);
process.AssertWaitForExit();
StageOutput.AddRange(process.Output);
if (process.ExitCode != 0)
{
SetInprocessStageStatus(StageCompletionStatusEnum.Failure);
project.Results.CompileSuccess = false;
}
else
{
SetInprocessStageStatus(StageCompletionStatusEnum.Success);
project.Results.CompileSuccess = true;
}
}
if (CompletionStatus == StageCompletionStatusEnum.InProcess)
{
CompletionStatus = StageCompletionStatusEnum.Success;
}
return(CompletionStatus);
}
/// <summary>
/// Run Compile process
/// </summary>
/// <returns></returns>
protected override StageCompletionStatusEnum ExecuteProcess()
{
DotNetBuildSettings dotNetBuildSettings = new DotNetBuildSettings()
{
ProjectFile = CISession.Solution,
NoRestore = true,
PropertiesInternal = new Dictionary <string, object>(),
};
dotNetBuildSettings = dotNetBuildSettings.SetProjectFile(CISession.Solution)
.SetConfiguration(CISession.CompileConfig)
.SetVerbosity(DotNetVerbosity.Minimal)
.EnableNoRestore()
.SetAssemblyVersion(CISession.VersionInfo.AssemblyVersion)
.SetVersion(CISession.VersionInfo.AssemblyVersion)
.SetInformationalVersion(CISession.VersionInfo.InformationalVersion)
.SetFileVersion(CISession.VersionInfo.FileVersion);
(BlockingCollection <ILineOut> outputs, int exitCode) = DotNetBuild(dotNetBuildSettings);
StageOutput.AddRange(outputs);
ControlFlow.Assert(exitCode == 0, "Process DotNetBuild failed");
AOT_NewLine();
AOT_Info("Compilation Status:");
foreach (SlugCIProject project in CISession.SlugCIConfigObj.Projects)
{
foreach (ILineOut lineOut in outputs)
{
if (lineOut.Text.StartsWith(" " + project.Name + " -> "))
{
AOT_Success("Compile Success: " + lineOut.Text);
project.Results.CompileSuccess = true;
break;
}
}
}
return(StageCompletionStatusEnum.Success);
}
/// <summary>
/// Run the typewriter publishing steps
/// </summary>
/// <returns></returns>
protected override StageCompletionStatusEnum ExecuteProcess()
{
string command = "npm";
string npmArgs = "run publishTW";
CompletionStatus = StageCompletionStatusEnum.InProcess;
foreach (SlugCIProject project in CISession.Projects)
{
AOT_Normal("Project: " + project.Name, Color.Magenta);
AOT_Normal(" --> HasTypeWriterScripts: " + project.HasTypeWriterScripts, Color.Magenta);
if (!project.HasTypeWriterScripts)
{
continue;
}
AbsolutePath scriptsFolder = project.VSProject.Directory / "_scripts";
IProcess process = ProcessTasks.StartProcess(command, npmArgs, scriptsFolder, customLogger: NPMLogger);
process.AssertWaitForExit();
StageOutput.AddRange(process.Output);
if (process.ExitCode != 0)
{
SetInprocessStageStatus(StageCompletionStatusEnum.Failure);
}
else
{
SetInprocessStageStatus(StageCompletionStatusEnum.Success);
}
}
if (CompletionStatus == StageCompletionStatusEnum.InProcess)
{
CompletionStatus = StageCompletionStatusEnum.Success;
}
return(CompletionStatus);
}
/// <summary>
/// Run Unit Test Runner Process
/// </summary>
/// <returns></returns>
protected override StageCompletionStatusEnum ExecuteProcess()
{
if (CISession.SkipTests)
{
return(StageCompletionStatusEnum.Skipped);
}
FileSystemTasks.EnsureExistingDirectory(CISession.CoveragePath);
DotNetTestSettings settings = new DotNetTestSettings()
{
ProjectFile = CISession.Solution,
Configuration = CISession.CompileConfig,
NoRestore = true,
NoBuild = true,
Verbosity = DotNetVerbosity.Minimal,
ProcessLogOutput = true,
ResultsDirectory = CISession.TestOutputPath,
ProcessArgumentConfigurator = arguments => arguments.Add("/p:CollectCoverage={0}", true)
.Add("", false)
.Add("/p:CoverletOutput={0}/", CISession.CoveragePath)
.Add("/p:CoverletOutputFormat={0}", "cobertura")
.Add("/p:Threshold={0}", CISession.SlugCIConfigObj.CodeCoverageThreshold)
.Add("/p:SkipAutoProps={0}", true)
.Add("/p:ExcludeByAttribute={0}",
"\"Obsolete%2cGeneratedCodeAttribute%2cCompilerGeneratedAttribute\"")
.Add("/p:UseSourceLink={0}", true)
};
(BlockingCollection <ILineOut> outputs, int exitCode) = DotNetTest(settings);
StageOutput.AddRange(outputs);
if (exitCode > 0 && CISession.FailedUnitTestsOkay)
{
AOT_Warning("One or more unit tests failed. HOWEVER, the Failed Unit Tests Okay flag was set, so this is not stopping the CI process");
return(StageCompletionStatusEnum.Warning);
}
return(StageCompletionStatusEnum.Success);
}
/// <summary>
/// Publishes a Nuget package to a nuget site.
/// </summary>
private void Publish_Nuget()
{
DotNetNuGetPushSettings settings = new DotNetNuGetPushSettings()
{
Source = CISession.NugetRepoURL,
ApiKey = CISession.NugetAPIKey,
SkipDuplicate = true,
};
IReadOnlyCollection <AbsolutePath> nugetPackages = CISession.OutputDirectory.GlobFiles("*.nupkg");
foreach (AbsolutePath nugetPackage in nugetPackages)
{
if (nugetPackage.ToString().EndsWith("symbols.nupkg"))
{
continue;
}
bool pushedSuccessfully = false;
StageCompletionStatusEnum stepStatus = StageCompletionStatusEnum.NotStarted;
try
{
settings.TargetPath = nugetPackage;
(BlockingCollection <ILineOut> nugetOutput, int exitCode) = DotNetNuGetPush(settings);
StageOutput.AddRange(nugetOutput);
ControlFlow.Assert(exitCode == 0, "Process DotNetBuild failed");
if (nugetOutput.Count > 0)
{
// Look for skipped message.
foreach (ILineOut outputLine in nugetOutput)
{
if (outputLine.Text.Contains("already exists at feed"))
{
stepStatus = StageCompletionStatusEnum.Warning;
string msg = @"A nuget package <" +
Path.GetFileName(nugetPackage) +
"> with this name and version already exists. " +
"Assuming this is due to you re-running the publish after a prior error that occurred after the push to Nuget was successful. " +
"Will carry on as though this push was successful. " +
"Otherwise, if this should have been a new update, then you will need to make another commit and re-publish";
Logger.Warn(msg);
}
else if (outputLine.Text.Contains("package was pushed"))
{
pushedSuccessfully = true;
stepStatus = StageCompletionStatusEnum.Success;
}
}
}
}
catch (ProcessException pe)
{
stepStatus = StageCompletionStatusEnum.Failure;
if (!CISession.NugetRepoURL.Contains("nuget.org"))
{
Logger.Warn(
"The nuget Push process threw an error. Since you are using a service other than Nuget this may be a service outage with the site or it might mean the version of the library you are pushing already exists. You will need to perform a manual check to determine which it is.");
}
else
{
throw;
}
}
if (pushedSuccessfully)
{
string fileName = Path.GetFileName(nugetPackage);
fileName = fileName.TrimEnd(".symbols.nupkg");
fileName = fileName.TrimEnd(".nupkg");
fileName = fileName.TrimEnd("." + CISession.VersionInfo.SemVersionAsString);
fileName = fileName.ToLower();
// Loop thru projects looking for that assembly name
foreach (SlugCIProject project in CISession.Projects)
{
if (project.AssemblyName.ToLower() == fileName || project.PackageId.ToLower() == fileName)
{
// TODO - Remove - not needed anylonger.
// For Tool Deployed projects, we need to copy the current version out to the deploy folder
/*
* if ( project.Deploy == SlugCIDeployMethod.Tool ) {
* AbsolutePath deployFile = CISession.DeployCopyPath / project.Name / CISession.PublishTarget.ToString() / "Version.json";
* ToolVersionJSON toolVersionJSON = new ToolVersionJSON() {ToolVersion = CISession.VersionInfo.SemVersionAsString};
* string json = JsonSerializer.Serialize<ToolVersionJSON>(toolVersionJSON, ToolVersionJSON.SerializerOptions());
* File.WriteAllText(deployFile, json);
* }
*/
project.Results.PublishedSuccess = true;
break;
}
}
}
// Set stage status based upon Step Status
SetInprocessStageStatus(stepStatus);
}
}