public static IEnumerable <SarifLog> Split(this SarifLog sarifLog, SplittingStrategy splittingStrategy)
{
PartitionFunction <string> partitionFunction = null;
switch (splittingStrategy)
{
case SplittingStrategy.PerResult:
{
partitionFunction = (result) => result.RuleId;
break;
}
case SplittingStrategy.PerRun:
{
foreach (Run run in sarifLog.Runs)
{
run.SetRunOnResults();
}
partitionFunction = (result) => result.Run.GetHashCode().ToString();
break;
}
default:
{
throw new NotImplementedException($"SplittingStrategy: {splittingStrategy}");
}
}
var partitioningVisitor = new PartitioningVisitor <string>(partitionFunction, deepClone: false);
partitioningVisitor.VisitSarifLog(sarifLog);
return(partitioningVisitor.GetPartitionLogs().Values);
}
public CrossValidation(Machine machine, Features features, Labels labels, SplittingStrategy splitting_strategy, Evaluation evaluation_criterium) : this(modshogunPINVOKE.new_CrossValidation__SWIG_1(Machine.getCPtr(machine), Features.getCPtr(features), Labels.getCPtr(labels), SplittingStrategy.getCPtr(splitting_strategy), Evaluation.getCPtr(evaluation_criterium)), true)
{
if (modshogunPINVOKE.SWIGPendingException.Pending)
{
throw modshogunPINVOKE.SWIGPendingException.Retrieve();
}
}
internal static HandleRef getCPtr(SplittingStrategy obj)
{
return((obj == null) ? new HandleRef(null, IntPtr.Zero) : obj.swigCPtr);
}
public virtual void FileWorkItems(SarifLog sarifLog)
{
sarifLog = sarifLog ?? throw new ArgumentNullException(nameof(sarifLog));
this.FilingClient.Connect(this.FilingContext.PersonalAccessToken).Wait();
OptionallyEmittedData optionallyEmittedData = this.FilingContext.DataToRemove;
if (optionallyEmittedData != OptionallyEmittedData.None)
{
var dataRemovingVisitor = new RemoveOptionalDataVisitor(optionallyEmittedData);
dataRemovingVisitor.Visit(sarifLog);
}
optionallyEmittedData = this.FilingContext.DataToInsert;
if (optionallyEmittedData != OptionallyEmittedData.None)
{
var dataInsertingVisitor = new InsertOptionalDataVisitor(optionallyEmittedData);
dataInsertingVisitor.Visit(sarifLog);
}
SplittingStrategy splittingStrategy = this.FilingContext.SplittingStrategy;
if (splittingStrategy == SplittingStrategy.None)
{
FileWorkItemsHelper(sarifLog, this.FilingContext, this.FilingClient);
return;
}
for (int runIndex = 0; runIndex < sarifLog.Runs?.Count; ++runIndex)
{
if (sarifLog.Runs[runIndex]?.Results?.Count > 0)
{
IList <SarifLog> logsToProcess = new List <SarifLog>(new SarifLog[] { sarifLog });
if (splittingStrategy != SplittingStrategy.PerRun)
{
SplittingVisitor visitor;
switch (splittingStrategy)
{
case SplittingStrategy.PerRunPerRule:
visitor = new PerRunPerRuleSplittingVisitor();
break;
case SplittingStrategy.PerRunPerTarget:
visitor = new PerRunPerTargetSplittingVisitor();
break;
case SplittingStrategy.PerRunPerTargetPerRule:
visitor = new PerRunPerTargetPerRuleSplittingVisitor();
break;
// TODO: Implement PerResult and PerRun splittings strategies
//
// https://github.com/microsoft/sarif-sdk/issues/1763
// https://github.com/microsoft/sarif-sdk/issues/1762
//
case SplittingStrategy.PerResult:
case SplittingStrategy.PerRun:
default:
throw new ArgumentOutOfRangeException($"SplittingStrategy: {splittingStrategy}");
}
visitor.VisitRun(sarifLog.Runs[runIndex]);
logsToProcess = visitor.SplitSarifLogs;
}
for (int splitFileIndex = 0; splitFileIndex < logsToProcess.Count; splitFileIndex++)
{
SarifLog splitLog = logsToProcess[splitFileIndex];
FileWorkItemsHelper(splitLog, this.FilingContext, this.FilingClient);
}
}
}
}
public virtual IReadOnlyList <SarifLog> SplitLogFile(SarifLog sarifLog)
{
IList <SarifLog> logsToProcess;
using (Logger.BeginScopeContext(nameof(SplitLogFile)))
{
sarifLog = sarifLog ?? throw new ArgumentNullException(nameof(sarifLog));
sarifLog.SetProperty("guid", Guid.NewGuid());
this.FilingResult = FilingResult.None;
this.FiledWorkItems = new List <WorkItemModel>();
sarifLog = sarifLog ?? throw new ArgumentNullException(nameof(sarifLog));
Logger.LogInformation("Connecting to filing client: {accountOrOrganization}", this.FilingClient.AccountOrOrganization);
this.FilingClient.Connect(this.FilingContext.PersonalAccessToken).Wait();
OptionallyEmittedData optionallyEmittedData = this.FilingContext.DataToRemove;
if (optionallyEmittedData != OptionallyEmittedData.None)
{
Logger.LogDebug("Removing optional data.");
var dataRemovingVisitor = new RemoveOptionalDataVisitor(optionallyEmittedData);
dataRemovingVisitor.Visit(sarifLog);
}
optionallyEmittedData = this.FilingContext.DataToInsert;
if (optionallyEmittedData != OptionallyEmittedData.None)
{
Logger.LogDebug("Inserting optional data.");
var dataInsertingVisitor = new InsertOptionalDataVisitor(optionallyEmittedData);
dataInsertingVisitor.Visit(sarifLog);
}
using (Logger.BeginScopeContext("Splitting visitor"))
{
SplittingStrategy splittingStrategy = this.FilingContext.SplittingStrategy;
Logger.LogInformation($"Splitting strategy - {splittingStrategy}");
if (splittingStrategy == SplittingStrategy.None)
{
return(new[] { sarifLog });
}
PartitionFunction <string> partitionFunction = null;
Stopwatch splittingStopwatch = Stopwatch.StartNew();
switch (splittingStrategy)
{
case SplittingStrategy.PerRun:
{
partitionFunction = (result) => result.ShouldBeFiled() ? "Include" : null;
break;
}
case SplittingStrategy.PerResult:
{
partitionFunction = (result) => result.ShouldBeFiled() ? Guid.NewGuid().ToString() : null;
break;
}
case SplittingStrategy.PerRunPerOrgPerEntityTypePerPartialFingerprint:
{
partitionFunction = (result) => result.ShouldBeFiled() ? result.GetFingerprintSplittingStrategyId() : null;
break;
}
case SplittingStrategy.PerRunPerOrgPerEntityTypePerRepositoryPerPartialFingerprint:
{
partitionFunction = (result) => result.ShouldBeFiled() ? result.GetPerRepositoryFingerprintSplittingStrategyId() : null;
break;
}
default:
{
throw new ArgumentOutOfRangeException($"SplittingStrategy: {splittingStrategy}");
}
}
Logger.LogDebug("Begin splitting logs");
var partitioningVisitor = new PartitioningVisitor <string>(partitionFunction, deepClone: false);
partitioningVisitor.VisitSarifLog(sarifLog);
Logger.LogDebug("Begin retrieving split logs");
logsToProcess = new List <SarifLog>(partitioningVisitor.GetPartitionLogs().Values);
Logger.LogDebug("End retrieving split logs");
var logsToProcessMetrics = new Dictionary <string, object>
{
{ "splittingStrategy", splittingStrategy },
{ "logsToProcessCount", logsToProcess.Count },
{ "splittingDurationInMilliseconds", splittingStopwatch.ElapsedMilliseconds },
};
this.Logger.LogMetrics(EventIds.LogsToProcessMetrics, logsToProcessMetrics);
splittingStopwatch.Stop();
}
}
if (logsToProcess != null && !this.FilingContext.ShouldFileUnchanged)
{
// Remove any logs that do not contain at least one result with a New or None baselinestate.
logsToProcess = logsToProcess.Where(log => log?.Runs?.Any(run => run.Results?.Any(result => result.BaselineState == BaselineState.New || result.BaselineState == BaselineState.None) == true) == true).ToList();
}
return(logsToProcess.ToArray());
}
public virtual SarifLog FileWorkItems(SarifLog sarifLog)
{
sarifLog = sarifLog ?? throw new ArgumentNullException(nameof(sarifLog));
sarifLog.SetProperty("guid", Guid.NewGuid());
using (Logger.BeginScope(nameof(FileWorkItems)))
{
this.FilingResult = FilingResult.None;
this.FiledWorkItems = new List<WorkItemModel>();
sarifLog = sarifLog ?? throw new ArgumentNullException(nameof(sarifLog));
Logger.LogInformation("Connecting to filing client: {accountOrOrganization}", this.FilingClient.AccountOrOrganization);
this.FilingClient.Connect(this.FilingContext.PersonalAccessToken).Wait();
OptionallyEmittedData optionallyEmittedData = this.FilingContext.DataToRemove;
if (optionallyEmittedData != OptionallyEmittedData.None)
{
var dataRemovingVisitor = new RemoveOptionalDataVisitor(optionallyEmittedData);
dataRemovingVisitor.Visit(sarifLog);
}
optionallyEmittedData = this.FilingContext.DataToInsert;
if (optionallyEmittedData != OptionallyEmittedData.None)
{
var dataInsertingVisitor = new InsertOptionalDataVisitor(optionallyEmittedData);
dataInsertingVisitor.Visit(sarifLog);
}
SplittingStrategy splittingStrategy = this.FilingContext.SplittingStrategy;
if (splittingStrategy == SplittingStrategy.None)
{
FileWorkItemsHelper(sarifLog, this.FilingContext, this.FilingClient);
return sarifLog;
}
IList<SarifLog> logsToProcess;
PartitionFunction<string> partitionFunction = null;
Stopwatch splittingStopwatch = Stopwatch.StartNew();
switch (splittingStrategy)
{
case SplittingStrategy.PerRun:
{
partitionFunction = (result) => result.ShouldBeFiled() ? "Include" : null;
break;
}
case SplittingStrategy.PerResult:
{
partitionFunction = (result) => result.ShouldBeFiled() ? Guid.NewGuid().ToString() : null;
break;
}
default:
{
throw new ArgumentOutOfRangeException($"SplittingStrategy: {splittingStrategy}");
}
}
var partitioningVisitor = new PartitioningVisitor<string>(partitionFunction, deepClone: false);
partitioningVisitor.VisitSarifLog(sarifLog);
logsToProcess = new List<SarifLog>(partitioningVisitor.GetPartitionLogs().Values);
var logsToProcessMetrics = new Dictionary<string, object>
{
{ "splittingStrategy", splittingStrategy },
{ "logsToProcessCount", logsToProcess.Count },
{ "splittingDurationInMilliseconds", splittingStopwatch.ElapsedMilliseconds },
};
this.Logger.LogMetrics(EventIds.LogsToProcessMetrics, logsToProcessMetrics);
splittingStopwatch.Stop();
for (int splitFileIndex = 0; splitFileIndex < logsToProcess.Count; splitFileIndex++)
{
SarifLog splitLog = logsToProcess[splitFileIndex];
FileWorkItemsHelper(splitLog, this.FilingContext, this.FilingClient);
}
}
return sarifLog;
}
private IEnumerable <SarifLog> SplitLogs(SplittingStrategy perRule)
{
throw new NotImplementedException();
}
internal static HandleRef getCPtr(SplittingStrategy obj) {
return (obj == null) ? new HandleRef(null, IntPtr.Zero) : obj.swigCPtr;
}
public CrossValidation(Machine machine, Features features, Labels labels, SplittingStrategy splitting_strategy, Evaluation evaluation_criterium) : this(modshogunPINVOKE.new_CrossValidation__SWIG_1(Machine.getCPtr(machine), Features.getCPtr(features), Labels.getCPtr(labels), SplittingStrategy.getCPtr(splitting_strategy), Evaluation.getCPtr(evaluation_criterium)), true) {
if (modshogunPINVOKE.SWIGPendingException.Pending) throw modshogunPINVOKE.SWIGPendingException.Retrieve();
}
