protected static void AssertPipTypeCount(Pip[] pips, PipType pipType, int expected)
{
XAssert.AreEqual(
expected,
pips.Where(p => p.PipType == pipType).Count(),
$"Number of <{pipType}> pips doesn't match");
}
/// <inheritdoc />
public override string ToString()
{
var builder = new StringBuilder();
builder.AppendLine();
builder.AppendLine($" Serialized pips: {PipsSerialized}");
builder.AppendLine($" Deserialized pips: {PipsDeserialized}");
for (int i = 0; i < m_pips.Length; ++i)
{
PipType pipType = (PipType)i;
builder.AppendLine($" {pipType.ToString()}: {m_pips[i]}");
if (pipType == PipType.Process)
{
for (int j = 0; j < m_serviceKinds.Length; ++j)
{
ServicePipKind servicePipKind = (ServicePipKind)j;
if (servicePipKind != ServicePipKind.None)
{
builder.AppendLine($" {servicePipKind.ToString()}: {m_serviceKinds[j]}");
}
}
}
}
return(builder.ToString());
}
/// <summary>
/// Gets all pips of a certain type.
/// </summary>
/// <returns>Returns list of all pips of certain type, empty if no such pips exist.</returns>
public IEnumerable <IMessage> GetAllPipsByType(PipType pipType)
{
Contract.Requires(Accessor != null, "XldbDataStore is not initialized");
var storedPips = new List <IMessage>();
if (!m_pipParserDictionary.TryGetValue(pipType, out var parser))
{
Contract.Assert(false, "No parser found for PipId");
}
// Empty key will match all pips in the prefix search, and then we grab only the ones that match the type we want
var pipIdKey = new PipIdKey();
var maybeFound = Accessor.Use(database =>
{
foreach (var kvp in database.PrefixSearch(pipIdKey.ToByteArray(), PipColumnFamilyName))
{
var pipKey = PipIdKey.Parser.ParseFrom(kvp.Key);
if (pipKey.PipType == pipType)
{
storedPips.Add(parser.ParseFrom(kvp.Value));
}
}
});
if (!maybeFound.Succeeded)
{
maybeFound.Failure.Throw();
}
return(storedPips);
}
private static string ConvertTypeToKind(PipType pipType)
{
switch (pipType)
{
case PipType.WriteFile:
return("writeFile");
case PipType.CopyFile:
return("copyFile");
case PipType.Process:
return("process");
case PipType.SealDirectory:
return("sealDirectory");
case PipType.Ipc:
return("ipc");
case PipType.Value:
return("value");
case PipType.SpecFile:
return("specFile");
case PipType.Module:
return("module");
case PipType.HashSourceFile:
return("hashFile");
default:
return("unknownPip");
}
}
/// <summary>
/// Gets the pip stored based on the semistable hash
/// </summary>
/// <returns>Returns null if no such pip is found</returns>
public IMessage GetPipBySemiStableHash(long semiStableHash, out PipType pipType)
{
Contract.Requires(Accessor != null, "XldbDataStore is not initialized");
IMessage foundPip = null;
var pipSemistableHashKey = new PipSemistableHashKey()
{
SemiStableHash = semiStableHash
};
PipType outPipType = 0;
var maybeFound = Accessor.Use(database =>
{
if (database.TryGetValue(pipSemistableHashKey.ToByteArray(), out var pipValueSemistableHash, PipColumnFamilyName))
{
foundPip = GetPipByPipId(PipIdValue.Parser.ParseFrom(pipValueSemistableHash).PipId, out outPipType);
}
});
if (!maybeFound.Succeeded)
{
maybeFound.Failure.Throw();
}
pipType = outPipType;
return(foundPip);
}
internal static MutablePipState Deserialize(BuildXLReader reader, PipType pipType, long semiStableHash, PageableStoreId storeId)
{
ServiceInfo serviceInfo = reader.ReadNullable(ServiceInfo.InternalDeserialize);
int options = reader.ReadInt32();
return(new ProcessMutablePipState(pipType, semiStableHash, storeId, serviceInfo, (Process.Options)options));
}
/// <summary>
/// Atomically transitions <see cref="State" /> to <paramref name="targetState" />.
/// The current state must not change during the attempted transition (i.e., synchronization may be required).
/// If provided, <paramref name="onCommit"/> is called upon committing to transition to the next state but
/// before the state could possibly transition further.
/// </summary>
/// <remarks>
/// This method is effectively not thread safe, since one must somehow know that a transition to
/// <paramref name="targetState" /> is valid
/// and that the current state will not change during this call (e.g. via some collaborating lock).
/// </remarks>
/// <returns>The prior pip state</returns>
public PipState Transition(PipState targetState, PipType pipType, Action <PipState, PipState, PipType> onCommit = null)
{
// This Requires is Static (not at runtime) so we avoid loading State twice (it can change behind our backs).
// Below we do a final (authoritative) load of State and then runtime-verify a valid transition.
Contract.RequiresDebug(State.CanTransitionTo(targetState));
Contract.Requires(pipType < PipType.Max || onCommit == null);
PipState presentState = State;
if (!State.CanTransitionTo(targetState))
{
Contract.Assume(false, I($"Transition failure (not a valid transition): {presentState:G} -> {targetState:G}"));
}
Contract.Assume(((int)presentState & PreCommitStateBit) == 0);
bool transitioned = TryTransitionInternal(presentState, targetState, pipType, onCommit);
if (!transitioned)
{
Contract.Assume(
false,
I($"Failed to transition a pip from {presentState:G} to {targetState:G} due to an unexpected intervening state change (current state is now {State:G})"));
}
return(presentState);
}
internal RunnablePip(
LoggingContext phaseLoggingContext,
PipId pipId,
PipType type,
int priority,
Func <RunnablePip, Task> executionFunc,
IPipExecutionEnvironment environment,
int maxRetryLimitForStoppedWorker,
Pip pip = null)
{
Contract.Requires(phaseLoggingContext != null);
Contract.Requires(environment != null);
PipId = pipId;
PipType = type;
Priority = priority;
OperationContext = OperationContext.CreateUntracked(phaseLoggingContext);
m_executionFunc = executionFunc;
Environment = environment;
Transition(PipExecutionStep.Start);
ScheduleTime = DateTime.UtcNow;
Performance = new RunnablePipPerformanceInfo(ScheduleTime);
m_pip = pip;
MaxRetryLimitForStoppedWorker = maxRetryLimitForStoppedWorker;
}
/// <summary>
/// Prints out information about the PipTypes of a collection
/// </summary>
public void PrintPipCountData(List <PipReference> pipList)
{
Dictionary <PipType, int> pipDictionary = new Dictionary <PipType, int>();
foreach (PipReference pip in pipList)
{
PipType type = pip.PipType;
if (!pipDictionary.ContainsKey(type))
{
pipDictionary[type] = 1;
}
else
{
pipDictionary[type]++;
}
}
Console.WriteLine($"\nNumber of Pips found matching filters: '{pipList.Count}'");
Console.WriteLine($"Types of Pips and counts");
foreach (var pair in pipDictionary)
{
Console.WriteLine($"{pair.Key}: {pair.Value}");
}
Console.WriteLine();
}
public SealDirectoryMutablePipState(PipType piptype, long semiStableHash, PageableStoreId storeId, SealDirectoryKind sealDirectoryKind, ReadOnlyArray <StringId> patterns, bool isComposite, bool scrub)
: base(piptype, semiStableHash, storeId)
{
SealDirectoryKind = sealDirectoryKind;
Patterns = patterns;
IsComposite = isComposite;
Scrub = scrub;
}
/// <summary>
/// Records that a pip is currently in <paramref name="state"/> and will have future transitions to count
/// (but no prior transitions have been counted).
/// </summary>
public void AccumulateInitialStateBulk(PipState state, PipType pipType, int count)
{
long prior = Interlocked.CompareExchange(
ref m_countersMatrix[(int)state - PipStateRange.MinValue, (int)state - PipStateRange.MinValue, (int)pipType],
count,
0);
Contract.Assume(prior == 0);
}
/// <summary>
/// Construct a PipEntry.
/// </summary>
public PipEntry(
long semiStableHash,
NameId name,
PipType type)
{
SemiStableHash = semiStableHash;
Name = name;
PipType = type;
}
internal static MutablePipState Deserialize(BuildXLReader reader, PipType pipType, long semiStableHash, PageableStoreId storeId)
{
var sealDirectoryKind = (SealDirectoryKind)reader.ReadByte();
var patterns = reader.ReadReadOnlyArray(reader1 => reader1.ReadStringId());
var isComposite = reader.ReadBoolean();
var scrub = reader.ReadBoolean();
return(new SealDirectoryMutablePipState(pipType, semiStableHash, storeId, sealDirectoryKind, patterns, isComposite, scrub));
}
internal CopyMutablePipState(
PipType pipType,
long semiStableHash,
PageableStoreId storeId,
bool keepOutputWritable)
: base(pipType, semiStableHash, storeId)
{
m_keepOutputWritable = keepOutputWritable;
}
/// <summary>
/// Add a pip, when you are sure it's not already in the table.
/// </summary>
public PipId Add(long semiStableHash, string name, PipType pipType)
{
PipEntry entry = new PipEntry(
semiStableHash,
NameTableBuilder.GetOrAdd(name),
pipType);
return(((PipTable)ValueTable).Add(entry));
}
internal ProcessMutablePipState(
PipType pipType,
long semiStableHash,
PageableStoreId storeId,
ServiceInfo serviceInfo,
Process.Options processOptions)
: base(pipType, semiStableHash, storeId)
{
ServiceInfo = serviceInfo;
ProcessOptions = processOptions;
}
internal static MutablePipState Deserialize(BuildXLReader reader, PipType pipType, long semiStableHash, PageableStoreId storeId)
{
ServiceInfo serviceInfo = reader.ReadNullable(ServiceInfo.InternalDeserialize);
int options = reader.ReadInt32();
RewritePolicy rewritePolicy = (RewritePolicy)reader.ReadByte();
AbsolutePath executablePath = reader.ReadAbsolutePath();
int priority = reader.ReadInt32();
int preserveOutputTrustLevel = reader.ReadInt32();
return(new ProcessMutablePipState(pipType, semiStableHash, storeId, serviceInfo, (Process.Options)options, rewritePolicy, executablePath, priority, preserveOutputTrustLevel));
}
/// <summary>
/// See <see cref="PipRuntimeInfo.TryTransition" /> ; additionally accumulates counters for pips in the source and target
/// states.
/// </summary>
public static bool TryTransition(
this PipRuntimeInfo pipRuntimeInfo,
PipStateCounters counters,
PipType pipType,
PipState assumedPresentState,
PipState targetState)
{
Contract.Requires(pipRuntimeInfo != null);
Contract.Requires(counters != null);
return(pipRuntimeInfo.TryTransition(assumedPresentState, targetState, pipType, counters.OnPipStateTransitionCommit));
}
internal ProcessMutablePipState(
PipType pipType,
long semiStableHash,
PageableStoreId storeId,
ServiceInfo serviceInfo,
Process.Options processOptions,
int priority,
int?preserveOutputsTrustLevel = null)
: base(pipType, semiStableHash, storeId)
{
ServiceInfo = serviceInfo;
ProcessOptions = processOptions;
Priority = priority;
PreserveOutputTrustLevel = preserveOutputsTrustLevel ?? 0;
}
/// <summary>
/// Indicates whether pips of the given type can produce outputs
/// </summary>
public static bool CanProduceOutputs(PipType pipType)
{
switch (pipType)
{
case PipType.WriteFile:
case PipType.CopyFile:
case PipType.Process:
case PipType.Ipc:
case PipType.SealDirectory:
return(true);
default:
return(false);
}
}
internal ProcessMutablePipState(
PipType pipType,
long semiStableHash,
PageableStoreId storeId,
ServiceInfo serviceInfo,
Process.Options processOptions,
RewritePolicy rewritePolicy,
AbsolutePath executablePath,
int priority,
int?preserveOutputsTrustLevel = null)
: base(pipType, semiStableHash, storeId)
{
ServiceInfo = serviceInfo;
ProcessOptions = processOptions;
RewritePolicy = rewritePolicy;
ExecutablePath = executablePath;
Priority = priority;
PreserveOutputTrustLevel = preserveOutputsTrustLevel ?? 0;
}
private bool IsSupportedPipType(PipType pipType)
{
switch (pipType)
{
case PipType.WriteFile:
case PipType.CopyFile:
case PipType.Process:
case PipType.Ipc:
case PipType.SealDirectory:
return(true);
case PipType.Value:
case PipType.SpecFile:
case PipType.Module:
case PipType.HashSourceFile:
return(false);
default:
throw Contract.AssertFailure($"Unknown pip type: {pipType}");
}
}
/// <summary>
/// Gets the pip stored based on the pip id
/// </summary>
/// <returns>Returns null if no such pip is found</returns>
public IMessage GetPipByPipId(uint pipId, out PipType pipType)
{
Contract.Requires(Accessor != null, "XldbDataStore is not initialized");
IMessage foundPip = null;
var pipIdKey = new PipIdKey()
{
PipId = pipId
};
PipType outPipType = 0;
var maybeFound = Accessor.Use(database =>
{
foreach (var kvp in database.PrefixSearch(pipIdKey.ToByteArray(), PipColumnFamilyName))
{
var pipKey = PipIdKey.Parser.ParseFrom(kvp.Key);
if (m_pipParserDictionary.TryGetValue(pipKey.PipType, out var parser))
{
foundPip = parser.ParseFrom(kvp.Value);
outPipType = pipKey.PipType;
}
else
{
Contract.Assert(false, "No parser found for PipId");
}
}
});
if (!maybeFound.Succeeded)
{
maybeFound.Failure.Throw();
}
pipType = outPipType;
return(foundPip);
}
internal ProcessMutablePipState(
PipType pipType,
long semiStableHash,
PageableStoreId storeId,
ServiceInfo serviceInfo,
Process.Options processOptions,
RewritePolicy rewritePolicy,
AbsolutePath executablePath,
int priority,
ModuleId moduleId,
int preserveOutputsTrustLevel = 0,
bool isSucceedFast = false)
: base(pipType, semiStableHash, storeId)
{
ServiceInfo = serviceInfo;
ProcessOptions = processOptions;
RewritePolicy = rewritePolicy;
ExecutablePath = executablePath;
Priority = priority;
PreserveOutputTrustLevel = preserveOutputsTrustLevel;
ModuleId = moduleId;
IsSucceedFast = isSucceedFast;
}
public Card(SuitType suit, PipType pip, int imgIdx)
{
Suit = suit;
Pip = pip;
ImageIndex = imgIdx;
}
private bool TryTransitionInternal(PipState assumedPresentState, PipState targetState, PipType pipType, Action <PipState, PipState, PipType> onCommit)
{
Contract.Requires(((int)assumedPresentState & PreCommitStateBit) == 0, "PipState values should have the high bit cleared");
// First we see if we can transition from assumedPresentState to a pre-commit version of it (note return value of State does
// not change as a result of this). If we win, then we have rights to actually commit and visibly transition thereafter.
// The end result is that {pre commit, onCommit(), actually transition} is atomic with respect to other concurrent calls.
int preCommitStateValue = ((int)assumedPresentState) | PreCommitStateBit;
int observedPresentStateValue = Interlocked.CompareExchange(ref m_state, preCommitStateValue, (int)assumedPresentState);
if (observedPresentStateValue == (int)assumedPresentState)
{
if (onCommit != null)
{
onCommit(assumedPresentState, targetState, pipType);
}
Volatile.Write(ref m_state, (int)targetState);
return(true);
}
else
{
return(false);
}
}
/// <summary>
/// Atomically transitions <see cref="State" /> from <paramref name="assumedPresentState" /> to
/// <paramref name="targetState" />.
/// Returns a bool indicating if the transition succeeded (if false, the <paramref name="assumedPresentState" /> was
/// mismatched with the current state).
/// If provided, <paramref name="onCommit"/> is called upon committing to transition to the next state but
/// before the state could possibly transition further.
/// </summary>
/// <remarks>
/// This method is thread safe.
/// </remarks>
public bool TryTransition(PipState assumedPresentState, PipState targetState, PipType pipType = PipType.Max, Action <PipState, PipState, PipType> onCommit = null)
{
Contract.Requires(assumedPresentState.CanTransitionTo(targetState));
Contract.Requires(((int)assumedPresentState & PreCommitStateBit) == 0, "PipState values should have the high bit cleared");
Contract.Requires(pipType < PipType.Max || onCommit == null);
return(TryTransitionInternal(assumedPresentState, targetState, pipType, onCommit));
}
/// <summary>
/// /// Constructor used for deserialization
/// </summary>
protected MutablePipState(PipType piptype, long semiStableHash, PageableStoreId storeId)
{
PipType = piptype;
SemiStableHash = semiStableHash;
StoreId = storeId;
}
internal static MutablePipState Deserialize(BuildXLReader reader, PipType pipType, long semiStableHash, PageableStoreId storeId)
{
bool keepOutputWritable = reader.ReadBoolean();
return(new CopyMutablePipState(pipType, semiStableHash, storeId, keepOutputWritable));
}
public bool IsValidNode(PipType pipType)
{
return(pipType == PipType.Process || pipType == PipType.CopyFile || pipType == PipType.WriteFile);
}
/// <summary>
/// Returns true if this is a MetaPip
/// </summary>
public static bool IsMetaPip(this PipType pipType)
{
return(pipType == PipType.Value || pipType == PipType.SpecFile || pipType == PipType.Module);
}
