/// <summary>
/// The constructor obtains the state information and the
/// callback delegate.
/// </summary>
internal TaskExecutionState
(
TaskExecutionMode howToExecuteTask,
Lookup lookupForInference,
Lookup lookupForExecution,
XmlElement taskXmlNode,
ITaskHost hostObject,
string projectFileOfTaskNode,
string parentProjectFullFileName,
string executionDirectory,
int handleId,
BuildEventContext buildEventContext
)
{
ErrorUtilities.VerifyThrow(taskXmlNode != null, "Must have task node");
this.howToExecuteTask = howToExecuteTask;
this.lookupForInference = lookupForInference;
this.lookupForExecution = lookupForExecution;
this.hostObject = hostObject;
this.projectFileOfTaskNode = projectFileOfTaskNode;
this.parentProjectFullFileName = parentProjectFullFileName;
this.executionDirectory = executionDirectory;
this.handleId = handleId;
this.buildEventContext = buildEventContext;
this.taskXmlNode = taskXmlNode;
}
/// <summary>
/// Creates an instance of this class using the given bucket data.
/// </summary>
/// <param name="itemNames">Item types being batched on: null indicates no batching is occurring</param>
/// <param name="itemMetadata">Hashtable of item metadata values: null indicates no batching is occurring</param>
internal ItemBucket
(
ICollection itemNames,
Dictionary<string, string> itemMetadata,
Lookup lookup,
int bucketSequenceNumber
)
{
ErrorUtilities.VerifyThrow(lookup != null, "Need lookup.");
// Create our own lookup just for this bucket
this.lookup = lookup.Clone();
// Push down the items, so that item changes in this batch are not visible to parallel batches
this.lookup.EnterScope();
// Add empty item groups for each of the item names, so that (unless items are added to this bucket) there are
// no item types visible in this bucket among the item types being batched on
if (itemNames != null)
{
foreach (string name in itemNames)
{
this.lookup.PopulateWithItems(name, new BuildItemGroup());
}
}
this.expander = new Expander(this.lookup.ReadOnlyLookup, itemMetadata);
this.bucketSequenceNumber = bucketSequenceNumber;
}
/// <summary>
/// Determines how many times the batchable object needs to be executed (each execution is termed a "batch"), and prepares
/// buckets of items to pass to the object in each batch.
/// </summary>
/// <returns>ArrayList containing ItemBucket objects, each one representing an execution batch.</returns>
internal static ArrayList PrepareBatchingBuckets
(
XmlNode parentNode,
List<string> batchableObjectParameters,
Lookup lookup
)
{
return PrepareBatchingBuckets(parentNode, batchableObjectParameters, lookup, null);
}
/// <summary>
/// Called to execute a task within a target. This method instantiates the task, sets its parameters, and executes it.
/// </summary>
internal void ExecuteTask(Lookup lookup)
{
ErrorUtilities.VerifyThrow(lookup != null, "Need to specify lookup.");
if ((conditionAttribute != null)
&& !Utilities.EvaluateCondition(conditionAttribute.Value, conditionAttribute, new Expander(lookup.ReadOnlyLookup), null, ParserOptions.AllowPropertiesAndItemLists, loggingServices, buildEventContext))
{
return;
}
// For these tasks, "execution" occurs the same whether we are asked to execute tasks
// or merely asked to infer outputs
switch (backingType)
{
case BackingType.PropertyGroup:
ExecutePropertyGroup(lookup);
break;
case BackingType.ItemGroup:
ExecuteItemGroup(lookup);
break;
}
}
/// <summary>
/// Determines how many times the batchable object needs to be executed (each execution is termed a "batch"), and prepares
/// buckets of items to pass to the object in each batch.
/// </summary>
/// <param name="parentNode"></param>
/// <param name="batchableObjectParameters"></param>
/// <param name="lookup"></param>
/// <param name="implicitBatchableItemType">Any item type that can be considered an implicit input to this batchable object.
/// This is useful for items inside targets, where the item name is plainly an item type that's an "input" to the object.</param>
/// <returns>ArrayList containing ItemBucket objects, each one representing an execution batch.</returns>
internal static ArrayList PrepareBatchingBuckets(
XmlNode parentNode,
List<string> batchableObjectParameters,
Lookup lookup,
string implicitBatchableItemType
)
{
ErrorUtilities.VerifyThrow(parentNode != null, "Need the XML node that represents the batchable object.");
ErrorUtilities.VerifyThrow(batchableObjectParameters != null, "Need the parameters of the batchable object to determine if it can be batched.");
ErrorUtilities.VerifyThrow(lookup != null, "Need to specify the lookup.");
ItemsAndMetadataPair pair = ExpressionShredder.GetReferencedItemNamesAndMetadata(batchableObjectParameters);
// All the @(itemname) item list references in the tag, including transforms, etc.
// The keys in the hashtable are the item names, and the values are all String.Empty (not used).
Hashtable consumedItemReferences = pair.Items;
// All the %(itemname.metadataname) references in the tag (not counting those embedded
// inside item transforms), and note that the itemname portion is optional.
// The keys in the returned hash table are the qualified metadata names (e.g. "EmbeddedResource.Culture"
// or just "Culture"). The values are MetadataReference structs, which simply split out the item
// name (possibly null) and the actual metadata name.
Dictionary<string, MetadataReference> consumedMetadataReferences = pair.Metadata;
ArrayList buckets = null;
if (consumedMetadataReferences?.Count > 0)
{
// Add any item types that we were explicitly told to assume.
if (implicitBatchableItemType != null)
{
consumedItemReferences = Utilities.CreateTableIfNecessary(consumedItemReferences);
consumedItemReferences[implicitBatchableItemType] = String.Empty;
}
// This method goes through all the item list references and figures out which ones
// will be participating in batching, and which ones won't. We get back a hashtable
// where the key is the item name that will be participating in batching. The values
// are all String.Empty (not used). This method may return additional item names
// that weren't represented in "consumedItemReferences"... this would happen if there
// were qualified metadata references in the consumedMetadataReferences table, such as
// %(EmbeddedResource.Culture).
Hashtable itemListsToBeBatched = GetItemListsToBeBatched(parentNode, consumedMetadataReferences, consumedItemReferences, lookup);
// At this point, if there were any metadata references in the tag, but no item
// references to batch on, we've got a problem because we can't figure out which
// item lists the user wants us to batch.
if (itemListsToBeBatched.Count == 0)
{
foreach (string unqualifiedMetadataName in consumedMetadataReferences.Keys)
{
// Of course, since this throws an exception, there's no way we're ever going
// to really loop here... it's just that the foreach is the only way I can
// figure out how to get data out of the hashtable without knowing any of the
// keys!
ProjectErrorUtilities.VerifyThrowInvalidProject(false,
parentNode, "CannotReferenceItemMetadataWithoutItemName", unqualifiedMetadataName);
}
}
else
{
// If the batchable object consumes item metadata as well as items to be batched,
// we need to partition the items consumed by the object.
buckets = BucketConsumedItems(parentNode, lookup, itemListsToBeBatched, consumedMetadataReferences);
}
}
// if the batchable object does not consume any item metadata or items, or if the item lists it consumes are all
// empty, then the object does not need to be batched
if ((buckets == null) || (buckets.Count == 0))
{
// create a default bucket that references the project items and properties -- this way we always have a bucket
buckets = new ArrayList(1);
buckets.Add(new ItemBucket(null, null, lookup, buckets.Count));
}
return buckets;
}
internal ReadOnlyLookup(Hashtable items, BuildPropertyGroup properties)
{
// Lookup only needs ItemDefinitionLibrary to mark new items with it.
// Since we're a read-only lookup, we don't need a real one.
this.lookup = new Lookup(items, properties, new ItemDefinitionLibrary(null));
}
/// <summary>
/// Of all the item lists that are referenced in this batchable object, which ones should we
/// batch on, and which ones should we just pass in wholesale to every invocation of the
/// target/task?
///
/// Rule #1. If the user has referenced any *qualified* item metadata such as %(EmbeddedResource.Culture),
/// then that item list "EmbeddedResource" will definitely get batched.
///
/// Rule #2. For all the unqualified item metadata such as %(Culture), we make sure that
/// every single item in every single item list being passed into the task contains a value
/// for that metadata. If not, it's an error. If so, we batch all of those item lists.
///
/// All other item lists will not be batched, and instead will be passed in wholesale to all buckets.
/// </summary>
/// <returns>Hashtable containing the item names that should be batched.</returns>
private static Hashtable GetItemListsToBeBatched
(
XmlNode parentNode,
Dictionary<string, MetadataReference> consumedMetadataReferences, // Key is [string] potentially qualified metadata name
// Value is [struct MetadataReference]
Hashtable consumedItemReferenceNames, // Key is [string] item name.
// Value is always String.Empty (unused).
Lookup lookup
)
{
// The keys in this hashtable are the names of the items that we will batch on.
// The values are always String.Empty (not used).
Hashtable itemListsToBeBatched = new Hashtable(StringComparer.OrdinalIgnoreCase);
// Loop through all the metadata references and find the ones that are qualified
// with an item name.
foreach (MetadataReference consumedMetadataReference in consumedMetadataReferences.Values)
{
if (consumedMetadataReference.itemName != null)
{
// Rule #1. Qualified metadata reference.
// For metadata references that are qualified with an item name
// (e.g., %(EmbeddedResource.Culture) ), we add that item name to the list of
// consumed item names, even if the item name wasn't otherwise referenced via
// @(...) syntax, and even if every item in the list doesn't necessary contain
// a value for this metadata. This is the special power that you get by qualifying
// the metadata reference with an item name.
itemListsToBeBatched[consumedMetadataReference.itemName] = String.Empty;
// Also add this qualified item to the consumed item references list, because
// %(EmbeddedResource.Culture) effectively means that @(EmbeddedResource) is
// being consumed, even though we may not see literally "@(EmbeddedResource)"
// in the tag anywhere. Adding it to this list allows us (down below in this
// method) to check that every item in this list has a value for each
// unqualified metadata reference.
consumedItemReferenceNames = Utilities.CreateTableIfNecessary(consumedItemReferenceNames);
consumedItemReferenceNames[consumedMetadataReference.itemName] = String.Empty;
}
}
// Loop through all the metadata references and find the ones that are unqualified.
foreach (MetadataReference consumedMetadataReference in consumedMetadataReferences.Values)
{
if (consumedMetadataReference.itemName == null)
{
// Rule #2. Unqualified metadata reference.
// For metadata references that are unqualified, every single consumed item
// must contain a value for that metadata. If any item doesn't, it's an error
// to use unqualified metadata.
if (consumedItemReferenceNames != null)
{
foreach (string consumedItemName in consumedItemReferenceNames.Keys)
{
// Loop through all the items in the item list.
BuildItemGroup items = lookup.GetItems(consumedItemName);
if (items != null)
{
// Loop through all the items in the BuildItemGroup.
foreach (BuildItem item in items)
{
ProjectErrorUtilities.VerifyThrowInvalidProject(
item.HasMetadata(consumedMetadataReference.metadataName),
parentNode, "ItemDoesNotContainValueForUnqualifiedMetadata",
item.Include, consumedItemName, consumedMetadataReference.metadataName);
}
}
// This item list passes the test of having every single item containing
// a value for this metadata. Therefore, add this item list to the batching list.
// Also, to save doing lookup.GetItems again, put the items in the table as the value.
itemListsToBeBatched[consumedItemName] = items;
}
}
}
}
return itemListsToBeBatched;
}
/// <summary>
/// Since we could not derrive from TaskEngine and have no Interface, we need to overide the method in here and
/// replace the calls when testing the class because of the calls to TaskEngine. If at a future time we get a mock task
/// engine, Interface or a non sealed TaskEngine these methods can disappear.
/// </summary>
/// <returns></returns>
virtual internal bool TaskEngineExecuteTask(
TaskEngine taskEngine,
TaskExecutionMode howTaskShouldBeExecuted,
Lookup lookup
)
{
return taskEngine.ExecuteTask
(
howTaskShouldBeExecuted,
lookup
);
}
private void InitializeForRunningTargetBatches()
{
// Make sure the <target> node has been given to us.
ErrorUtilities.VerifyThrow(targetElement != null,
"Need an XML node representing the <target> element.");
// Make sure this really is the <target> node.
ProjectXmlUtilities.VerifyThrowElementName(targetElement, XMakeElements.target);
overallSuccess = true;
projectContent = new Lookup(parentProject.evaluatedItemsByName, parentProject.evaluatedItems, parentProject.evaluatedProperties, parentProject.ItemDefinitionLibrary);
// If we need to use the task thread - ie, we encounter a non-intrinsic task - we will need to make sure
// the task thread only sees clones of the project items and properties. We insert a scope to allow us to
// do that later. See comment in InitializeForRunningFirstNonIntrinsicTask()
placeholderForClonedProjectContent = projectContent.EnterScope();
buckets = BatchingEngine.PrepareBatchingBuckets(targetElement, targetParameters, projectContent);
currentBucket = 0;
// Initialize the first bucket
InitializeForRunningSingleTargetBatch();
}
internal ReadOnlyLookup(Lookup lookup)
{
this.lookup = lookup;
}
private void CreateLookupAndEnterScope()
{
lookupPassedBetweenThreads = LookupHelpers.CreateEmptyLookup();
lookupPassedBetweenThreads.EnterScope();
}
private void GatherTaskItemOutputs(Lookup lookup, TaskOutput taskOutputSpecification, string itemName, string propertyName, object outputs)
{
// if the task has generated outputs (if it didn't, don't do anything)
if (outputs != null)
{
ITaskItem[] taskItemOutputs = (outputs is ITaskItem[])
? (ITaskItem[])outputs
: new ITaskItem[] { (ITaskItem)outputs };
if (taskOutputSpecification.IsItemVector)
{
ErrorUtilities.VerifyThrow((itemName != null) && (itemName.Length > 0), "Need item type.");
foreach (ITaskItem output in taskItemOutputs)
{
// if individual items in the array are null, ignore them
if (output != null)
{
lookup.AddNewItem(new BuildItem(itemName, output));
}
}
}
else
{
Debug.Assert(taskOutputSpecification.IsProperty);
ErrorUtilities.VerifyThrow((propertyName != null) && (propertyName.Length > 0), "Need property name.");
// to store an ITaskItem array in a property, join all the item-specs with semi-colons to make the
// property value, and ignore/discard the attributes on the ITaskItems
StringBuilder joinedOutputs = new StringBuilder();
foreach (ITaskItem output in taskItemOutputs)
{
// if individual items in the array are null, ignore them
if (output != null)
{
if (joinedOutputs.Length > 0)
{
joinedOutputs.Append(';');
}
joinedOutputs.Append(EscapingUtilities.Escape(output.ItemSpec));
}
}
lookup.SetProperty(new BuildProperty(propertyName, joinedOutputs.ToString(), PropertyType.OutputProperty));
}
}
}
/// <summary>
/// Execute a PropertyGroup element, including each child property
/// </summary>
private void ExecutePropertyGroup(Lookup lookup)
{
foreach (BuildProperty property in backingPropertyGroup)
{
ArrayList buckets = null;
try
{
// Find all the metadata references in order to create buckets
List<string> parameterValues = new List<string>();
GetBatchableValuesFromProperty(parameterValues, property);
buckets = BatchingEngine.PrepareBatchingBuckets(taskNodeXmlElement, parameterValues, lookup);
// "Execute" each bucket
foreach (ItemBucket bucket in buckets)
{
if (Utilities.EvaluateCondition(property.Condition, property.ConditionAttribute,
bucket.Expander, null, ParserOptions.AllowAll, loggingServices, buildEventContext))
{
// Check for a reserved name now, so it fails right here instead of later when the property eventually reaches
// the outer scope.
ProjectErrorUtilities.VerifyThrowInvalidProject(!ReservedPropertyNames.IsReservedProperty(property.Name), property.PropertyElement,
"CannotModifyReservedProperty", property.Name);
property.Evaluate(bucket.Expander);
bucket.Lookup.SetProperty(property);
}
}
}
finally
{
if (buckets != null)
{
// Propagate the property changes to the bucket above
foreach (ItemBucket bucket in buckets)
{
bucket.Lookup.LeaveScope();
}
}
}
}
}
/// <summary>
/// Execute an ItemGroup element, including each child item expression
/// </summary>
private void ExecuteItemGroup(Lookup lookup)
{
foreach (BuildItemGroupChildXml child in backingBuildItemGroupChildren)
{
ArrayList buckets = null;
try
{
List<string> parameterValues = new List<string>();
GetBatchableValuesFromBuildItemGroupChild(parameterValues, child);
buckets = BatchingEngine.PrepareBatchingBuckets(taskNodeXmlElement, parameterValues, lookup, child.Name);
// "Execute" each bucket
foreach (ItemBucket bucket in buckets)
{
// Gather the outputs, but don't make them visible to other buckets
switch (child.ChildType)
{
case ChildType.BuildItemAdd:
// It's an item -- we're "adding" items to the world
ExecuteAdd(child, bucket);
break;
case ChildType.BuildItemRemove:
// It's a remove -- we're "removing" items from the world
ExecuteRemove(child, bucket);
break;
case ChildType.BuildItemModify:
// It's a modify -- changing existing items
ExecuteModify(child, bucket);
break;
}
}
}
finally
{
if (buckets != null)
{
// Propagate the item changes to the bucket above
foreach (ItemBucket bucket in buckets)
{
bucket.Lookup.LeaveScope();
}
}
}
}
}
/// <summary>
/// Determines how many times the batchable object needs to be executed (each execution is termed a "batch"), and prepares
/// buckets of items to pass to the object in each batch.
/// </summary>
/// <param name="parentNode"></param>
/// <param name="batchableObjectParameters"></param>
/// <param name="lookup"></param>
/// <param name="implicitBatchableItemType">Any item type that can be considered an implicit input to this batchable object.
/// This is useful for items inside targets, where the item name is plainly an item type that's an "input" to the object.</param>
/// <returns>ArrayList containing ItemBucket objects, each one representing an execution batch.</returns>
internal static ArrayList PrepareBatchingBuckets
(
XmlNode parentNode,
List<string> batchableObjectParameters,
Lookup lookup,
string implicitBatchableItemType
)
{
ErrorUtilities.VerifyThrow(parentNode != null, "Need the XML node that represents the batchable object.");
ErrorUtilities.VerifyThrow(batchableObjectParameters != null, "Need the parameters of the batchable object to determine if it can be batched.");
ErrorUtilities.VerifyThrow(lookup != null, "Need to specify the lookup.");
ItemsAndMetadataPair pair = ExpressionShredder.GetReferencedItemNamesAndMetadata(batchableObjectParameters);
// All the @(itemname) item list references in the tag, including transforms, etc.
// The keys in the hashtable are the item names, and the values are all String.Empty (not used).
Hashtable consumedItemReferences = pair.Items;
// All the %(itemname.metadataname) references in the tag (not counting those embedded
// inside item transforms), and note that the itemname portion is optional.
// The keys in the returned hash table are the qualified metadata names (e.g. "EmbeddedResource.Culture"
// or just "Culture"). The values are MetadataReference structs, which simply split out the item
// name (possibly null) and the actual metadata name.
Dictionary<string, MetadataReference> consumedMetadataReferences = pair.Metadata;
ArrayList buckets = null;
if (consumedMetadataReferences != null && consumedMetadataReferences.Count > 0)
{
// Add any item types that we were explicitly told to assume.
if (implicitBatchableItemType != null)
{
consumedItemReferences = Utilities.CreateTableIfNecessary(consumedItemReferences);
consumedItemReferences[implicitBatchableItemType] = String.Empty;
}
// This method goes through all the item list references and figures out which ones
// will be participating in batching, and which ones won't. We get back a hashtable
// where the key is the item name that will be participating in batching. The values
// are all String.Empty (not used). This method may return additional item names
// that weren't represented in "consumedItemReferences"... this would happen if there
// were qualified metadata references in the consumedMetadataReferences table, such as
// %(EmbeddedResource.Culture).
Hashtable itemListsToBeBatched = GetItemListsToBeBatched(parentNode, consumedMetadataReferences, consumedItemReferences, lookup);
// At this point, if there were any metadata references in the tag, but no item
// references to batch on, we've got a problem because we can't figure out which
// item lists the user wants us to batch.
if (itemListsToBeBatched.Count == 0)
{
foreach (string unqualifiedMetadataName in consumedMetadataReferences.Keys)
{
// Of course, since this throws an exception, there's no way we're ever going
// to really loop here... it's just that the foreach is the only way I can
// figure out how to get data out of the hashtable without knowing any of the
// keys!
ProjectErrorUtilities.VerifyThrowInvalidProject(false,
parentNode, "CannotReferenceItemMetadataWithoutItemName", unqualifiedMetadataName);
}
}
else
{
// If the batchable object consumes item metadata as well as items to be batched,
// we need to partition the items consumed by the object.
buckets = BucketConsumedItems(parentNode, lookup, itemListsToBeBatched, consumedMetadataReferences);
}
}
// if the batchable object does not consume any item metadata or items, or if the item lists it consumes are all
// empty, then the object does not need to be batched
if ((buckets == null) || (buckets.Count == 0))
{
// create a default bucket that references the project items and properties -- this way we always have a bucket
buckets = new ArrayList(1);
buckets.Add(new ItemBucket(null, null, lookup, buckets.Count));
}
return buckets;
}
/// <summary>
/// Partitions the items consumed by the batchable object into buckets, where each bucket contains a set of items that
/// have the same value set on all item metadata consumed by the object.
/// </summary>
/// <remarks>
/// PERF NOTE: Given n items and m batching metadata that produce l buckets, it is usually the case that n > l > m,
/// because a batchable object typically uses one or two item metadata to control batching, and only has a handful of
/// buckets. The number of buckets is typically only large if a batchable object is using single-item batching
/// (where l == n). Any algorithm devised for bucketing therefore, should try to minimize n and l in its complexity
/// equation. The algorithm below has a complexity of O(n*lg(l)*m/2) in its comparisons, and is effectively O(n) when
/// l is small, and O(n*lg(n)) in the worst case as l -> n. However, note that the comparison complexity is not the
/// same as the operational complexity for this algorithm. The operational complexity of this algorithm is actually
/// O(n*m + n*lg(l)*m/2 + n*l/2 + n + l), which is effectively O(n^2) in the worst case. The additional complexity comes
/// from the array and metadata operations that are performed. However, those operations are extremely cheap compared
/// to the comparison operations, which dominate the time spent in this method.
/// </remarks>
/// <returns>ArrayList containing ItemBucket objects (can be empty), each one representing an execution batch.</returns>
private static ArrayList BucketConsumedItems
(
XmlNode parentNode,
Lookup lookup,
Hashtable itemListsToBeBatched,
Dictionary<string, MetadataReference> consumedMetadataReferences
)
{
ErrorUtilities.VerifyThrow(itemListsToBeBatched.Count > 0, "Need item types consumed by the batchable object.");
ErrorUtilities.VerifyThrow(consumedMetadataReferences.Count > 0, "Need item metadata consumed by the batchable object.");
ArrayList buckets = new ArrayList();
// Get and iterate through the list of item names that we're supposed to batch on.
foreach (DictionaryEntry entry in itemListsToBeBatched)
{
string itemName = (string)entry.Key;
// Use the previously-fetched items, if possible
BuildItemGroup items;
if (entry.Value is BuildItemGroup)
{
items = (BuildItemGroup)entry.Value;
}
else
{
items = lookup.GetItems(itemName);
}
if (items != null)
{
foreach (BuildItem item in items)
{
// Get this item's values for all the metadata consumed by the batchable object.
Dictionary<string, string> itemMetadataValues = GetItemMetadataValues(parentNode, item, consumedMetadataReferences);
// put the metadata into a dummy bucket we can use for searching
ItemBucket dummyBucket = ItemBucket.GetDummyBucketForComparisons(itemMetadataValues);
// look through all previously created buckets to find a bucket whose items have the same values as
// this item for all metadata consumed by the batchable object
int matchingBucketIndex = buckets.BinarySearch(dummyBucket);
ItemBucket matchingBucket = (matchingBucketIndex >= 0)
? (ItemBucket)buckets[matchingBucketIndex]
: null;
// If we didn't find a bucket that matches this item, create a new one, adding
// this item to the bucket.
if (null == matchingBucket)
{
matchingBucket = new ItemBucket(itemListsToBeBatched.Keys, itemMetadataValues, lookup, buckets.Count);
// make sure to put the new bucket into the appropriate location
// in the sorted list as indicated by the binary search
// NOTE: observe the ~ operator (bitwise complement) in front of
// the index -- see MSDN for more information on the return value
// from the ArrayList.BinarySearch() method
buckets.Insert(~matchingBucketIndex, matchingBucket);
}
// We already have a bucket for this type of item, so add this item to
// the bucket.
matchingBucket.AddItem(item);
}
}
}
// Put the buckets back in the order in which they were discovered, so that the first
// item declared in the project file ends up in the first batch passed into the target/task.
ArrayList orderedBuckets = ArrayList.Repeat(null, buckets.Count);
foreach (ItemBucket bucket in buckets)
{
orderedBuckets[bucket.BucketSequenceNumber] = bucket;
}
return orderedBuckets;
}
internal static Lookup CreateLookup(Hashtable items)
{
ItemDefinitionLibrary itemDefinitionLibrary = CreateEmptyEvaluatedItemDefinitionLibrary();
Lookup lookup = new Lookup(items, null, itemDefinitionLibrary);
return lookup;
}
/// <summary>
/// Uses the given task output specification to (statically) infer the task's outputs.
/// </summary>
/// <param name="taskOutputSpecification"></param>
/// <param name="taskParameterName"></param>
/// <param name="itemName">can be null</param>
/// <param name="propertyName">can be null</param>
/// <param name="bucket"></param>
private void InferTaskOutputs
(
Lookup lookup,
TaskOutput taskOutputSpecification,
string taskParameterName,
string itemName,
string propertyName,
ItemBucket bucket
)
{
// if the task has a value set for the output parameter, expand all embedded properties and item metadata in it
XmlAttribute taskParameterAttribute = null;
// Lookup attribute name needs to be case-insensitive
// DevDiv bugs: 33981
foreach (XmlAttribute taskNodeAttribute in taskNode.Attributes)
{
if (String.Compare(taskNodeAttribute.Name, taskParameterName, StringComparison.OrdinalIgnoreCase) == 0)
{
taskParameterAttribute = taskNodeAttribute;
break;
}
}
if (taskParameterAttribute != null)
{
if (taskOutputSpecification.IsItemVector)
{
// This is an output item.
ErrorUtilities.VerifyThrow((itemName != null) && (itemName.Length > 0), "Need item type.");
// Expand only with properties first, so that expressions like Include="@(foo)" will transfer the metadata of the "foo" items as well, not just their item specs.
Expander propertyAndMetadataExpander = new Expander(bucket.Expander, ExpanderOptions.ExpandPropertiesAndMetadata);
List<string> outputItemSpecs = propertyAndMetadataExpander.ExpandAllIntoStringListLeaveEscaped(taskParameterAttribute);
foreach (string outputItemSpec in outputItemSpecs)
{
BuildItemGroup items = bucket.Expander.ExpandSingleItemListExpressionIntoItemsLeaveEscaped(outputItemSpec, taskParameterAttribute);
// if the output item-spec is an item vector, get the items in it
if (items != null)
{
foreach (BuildItem item in items)
{
// we want to preserve the attributes on the item
BuildItem clonedItem = item.VirtualClone();
// but we do need to change the item type
clonedItem.Name = itemName;
lookup.AddNewItem(clonedItem);
}
}
else
{
// if the output item-spec is not an item vector, accept it as-is
lookup.AddNewItem(new BuildItem(itemName, outputItemSpec));
}
}
}
else
{
// This is an output property.
Debug.Assert(taskOutputSpecification.IsProperty);
ErrorUtilities.VerifyThrow((propertyName != null) && (propertyName.Length > 0), "Need property name.");
string taskParameterValue = bucket.Expander.ExpandAllIntoString(taskParameterAttribute);
if (taskParameterValue.Length > 0)
{
lookup.SetProperty(new BuildProperty(propertyName, taskParameterValue, PropertyType.OutputProperty));
}
}
}
}
internal static Lookup CreateLookup(BuildPropertyGroup properties, Hashtable items)
{
ItemDefinitionLibrary itemDefinitionLibrary = CreateEmptyEvaluatedItemDefinitionLibrary();
Lookup lookup = new Lookup(items, properties, itemDefinitionLibrary);
return lookup;
}
/// <summary>
/// Called to execute a task within a target. This method instantiates the task, sets its parameters, and executes it.
/// </summary>
/// <returns>true, if successful</returns>
internal bool ExecuteTask(TaskExecutionMode howToExecuteTask, Lookup lookup)
{
ErrorUtilities.VerifyThrow(lookup != null, "Need to specify items available to task.");
bool taskExecutedSuccessfully = false;
EngineProxy engineProxy = null;
ArrayList buckets = null;
try
{
engineProxy = new EngineProxy(parentModule, handleId, parentProjectFullFileName, projectFileOfTaskNode, loggingServices, buildEventContext);
List<string> taskParameterValues = CreateListOfParameterValues();
buckets = BatchingEngine.PrepareBatchingBuckets(taskNode, taskParameterValues, lookup);
lookupHash = null;
// Only create a hash table if there are more than one bucket as this is the only time a property can be overridden
if (buckets.Count > 1)
{
lookupHash = Utilities.CreateTableIfNecessary((Hashtable)null);
}
// Loop through each of the batch buckets and execute them one at a time
for (int i=0; i < buckets.Count; i++)
{
// Execute the batch bucket, pass in which bucket we are executing so that we know when to get a new taskId for the bucket.
taskExecutedSuccessfully = ExecuteBucket(engineProxy, (ItemBucket)buckets[i], i, howToExecuteTask);
if (!taskExecutedSuccessfully)
{
break;
}
}
}
finally
{
// Remove the AssemblyResolve handler in the default AppDomain, we are done with the task.
if (resolver != null)
{
resolver.RemoveHandler();
}
if (engineProxy != null)
{
engineProxy.MarkAsInActive();
}
// Now all task batches are done, apply all item adds to the outer
// target batch; we do this even if the task wasn't found (in that case,
// no items or properties will have been added to the scope)
if (buckets != null)
{
foreach (ItemBucket bucket in buckets)
{
bucket.Lookup.LeaveScope();
}
}
}
return taskExecutedSuccessfully;
}
internal static Lookup CreateEmptyLookup()
{
ItemDefinitionLibrary itemDefinitionLibrary = CreateEmptyEvaluatedItemDefinitionLibrary();
Lookup lookup = new Lookup(null, null, itemDefinitionLibrary);
return lookup;
}
/// <summary>
/// Uses the given task output specification to grab the task's outputs using .NET reflection.
/// </summary>
/// <remarks>
/// This method is "internal" for unit-testing purposes only.
/// </remarks>
/// <param name="taskOutputSpecification"></param>
/// <param name="taskParameterName"></param>
/// <param name="itemName">can be null</param>
/// <param name="propertyName">can be null</param>
/// <param name="task"></param>
/// <returns>true, if successful</returns>
internal bool GatherGeneratedTaskOutputs
(
Lookup lookup,
TaskOutput taskOutputSpecification,
string taskParameterName,
string itemName,
string propertyName,
ITask task)
{
ErrorUtilities.VerifyThrow(task != null, "Need instantiated task to retrieve outputs from.");
bool gatheredGeneratedOutputsSuccessfully = true;
try
{
PropertyInfo parameter = TaskClass.GetProperty(taskParameterName);
// flag an error if we find a parameter that has no .NET property equivalent
ProjectErrorUtilities.VerifyThrowInvalidProject(parameter != null,
taskOutputSpecification.TaskParameterAttribute,
"UnexpectedTaskOutputAttribute", taskParameterName, TaskName);
// output parameters must have their corresponding .NET properties marked with the Output attribute
ProjectErrorUtilities.VerifyThrowInvalidProject(TaskClass.GetNamesOfPropertiesWithOutputAttribute().ContainsKey(taskParameterName),
taskOutputSpecification.TaskParameterAttribute,
"UnmarkedOutputTaskParameter", parameter.Name, TaskName);
// grab the outputs from the task's designated output parameter (which is a .NET property)
object outputs = parameter.GetValue(task, null);
Type type = parameter.PropertyType;
// don't use the C# "is" operator as it always returns false if the object is null
if (
typeof(ITaskItem[]).IsAssignableFrom(type) || /* ITaskItem array or derived type, or */
typeof(ITaskItem).IsAssignableFrom(type) /* ITaskItem or derived type */
)
{
GatherTaskItemOutputs(lookup, taskOutputSpecification, itemName, propertyName, outputs);
}
// don't use the C# "is" operator as it always returns false if the object is null
else if (
(type.IsArray && type.GetElementType().IsValueType) || /* array of value types, or */
(type == typeof(string[])) || /* string array, or */
(type.IsValueType) || /* value type, or */
(type == typeof(string)) /* string */
)
{
GatherArrayStringAndValueOutputs(lookup, taskOutputSpecification, itemName, propertyName, parameter, outputs);
}
else
{
ProjectErrorUtilities.VerifyThrowInvalidProject(false, taskOutputSpecification.TaskParameterAttribute,
"UnsupportedTaskParameterTypeError", parameter.PropertyType, parameter.Name, TaskName);
}
}
// handle invalid TaskItems in task outputs
catch (InvalidOperationException e)
{
loggingServices.LogError(buildEventContext, Utilities.CreateBuildEventFileInfo(taskOutputSpecification.TaskParameterAttribute, projectFileOfTaskNode),
"InvalidTaskItemsInTaskOutputs", TaskName, taskParameterName, e.Message);
gatheredGeneratedOutputsSuccessfully = false;
}
// handle any exception thrown by the task's getter
catch (TargetInvocationException e)
{
// Exception thrown by the called code itself
// Log the stack, so the task vendor can fix their code
// Log the task line number, whatever the value of ContinueOnError;
// because this will be a hard error anyway.
loggingServices.LogFatalTaskError(buildEventContext, e.InnerException,
CreateBuildEventFileInfoForTask(),
TaskName);
// We do not recover from a task exception while getting outputs,
// so do not merely set gatheredGeneratedOutputsSuccessfully = false; here
ProjectErrorUtilities.VerifyThrowInvalidProject(false, taskOutputSpecification.TaskParameterAttribute,
"FailedToRetrieveTaskOutputs", TaskName, taskParameterName, e.InnerException.Message);
}
catch (Exception e) // Catching Exception, but rethrowing unless it's a well-known exception.
{
if (ExceptionHandling.NotExpectedReflectionException(e))
throw;
ProjectErrorUtilities.VerifyThrowInvalidProject(false, taskOutputSpecification.TaskParameterAttribute,
"FailedToRetrieveTaskOutputs", TaskName, taskParameterName, e.Message);
}
return gatheredGeneratedOutputsSuccessfully;
}
internal ReadOnlyLookup(Hashtable items, BuildPropertyGroup properties)
{
// Lookup only needs ItemDefinitionLibrary to mark new items with it.
// Since we're a read-only lookup, we don't need a real one.
this.lookup = new Lookup(items, properties, new ItemDefinitionLibrary(null));
}
private void ExecuteTask(IntrinsicTask task, Lookup lookup)
{
if (lookup == null)
{
lookup = LookupHelpers.CreateEmptyLookup();
}
task.ExecuteTask(lookup);
}
private void InitializeForRunningSingleTargetBatch()
{
// Verify that the target is in the right state
ErrorUtilities.VerifyThrow(inProgressBuildState == InProgressBuildState.RunningTasks, "Wrong state");
// Check if the current task number is valid
ErrorUtilities.VerifyThrow(currentBucket < buckets.Count, "No buckets left");
Hashtable changedTargetInputs = null;
Hashtable upToDateTargetInputs = null;
howToBuild = DependencyAnalysisResult.FullBuild;
ItemBucket bucket = (ItemBucket)buckets[currentBucket];
// For the first batch of a target use the targets original targetID. for each batch after the first one use a uniqueId to identity the target in the batch
if (currentBucket != 0)
{
targetBuildEventContext = new BuildEventContext(targetBuildEventContext.NodeId, parentEngine.GetNextTargetId(), targetBuildEventContext.ProjectContextId, targetBuildEventContext.TaskId);
}
// Flag the start of the target.
parentEngine.LoggingServices.LogTargetStarted(
targetBuildEventContext,
targetClass.Name,
this.parentProject.FullFileName,
targetClass.ProjectFileOfTargetElement);
loggedTargetStart = true;
// Figure out how we should build the target
TargetDependencyAnalyzer dependencyAnalyzer = new TargetDependencyAnalyzer(parentProject.ProjectDirectory, targetClass, parentEngine.LoggingServices, targetBuildEventContext);
howToBuild = dependencyAnalyzer.PerformDependencyAnalysis(bucket, out changedTargetInputs, out upToDateTargetInputs);
targetBuildSuccessful = true;
exitBatchDueToError = false;
// If we need to build the target - initialize the data structure for
// running the tasks
if ((howToBuild != DependencyAnalysisResult.SkipNoInputs) &&
(howToBuild != DependencyAnalysisResult.SkipNoOutputs))
{
// Within each target batch items are divided into lookup and execution; they must be
// kept separate: enforce this by cloning and entering scope
lookupForInference = bucket.Lookup;
lookupForExecution = bucket.Lookup.Clone();
lookupForInference.EnterScope();
lookupForExecution.EnterScope();
// if we're doing an incremental build, we need to effectively run the task twice -- once
// to infer the outputs for up-to-date input items, and once to actually execute the task;
// as a result we need separate sets of item and property collections to track changes
if (howToBuild == DependencyAnalysisResult.IncrementalBuild)
{
// subset the relevant items to those that are up-to-date
foreach (DictionaryEntry upToDateTargetInputsEntry in upToDateTargetInputs)
{
lookupForInference.PopulateWithItems((string)upToDateTargetInputsEntry.Key, (BuildItemGroup)upToDateTargetInputsEntry.Value);
}
// subset the relevant items to those that have changed
foreach (DictionaryEntry changedTargetInputsEntry in changedTargetInputs)
{
lookupForExecution.PopulateWithItems((string)changedTargetInputsEntry.Key, (BuildItemGroup)changedTargetInputsEntry.Value);
}
}
projectFileOfTaskNode = XmlUtilities.GetXmlNodeFile(targetElement, parentProject.FullFileName);
// count the tasks in the target
currentTask = 0;
skippedNodeCount = 0;
}
else
{
currentTask = targetElement.ChildNodes.Count;
}
}
/// <summary>
/// Mock out and override the method inside of TaskExecutionState which makes the calls to the task engine.
/// </summary>
internal override bool TaskEngineExecuteTask(TaskEngine taskEngine, TaskExecutionMode howToExecuteTask, Lookup lookup)
{
return targetInferenceSuccessful;
}
private void GatherArrayStringAndValueOutputs(Lookup lookup, TaskOutput taskOutputSpecification, string itemName, string propertyName, PropertyInfo parameter, object outputs)
{
// if the task has generated outputs (if it didn't, don't do anything)
if (outputs != null)
{
Array convertibleOutputs = (parameter.PropertyType.IsArray)
? (Array)outputs
: new object[] { outputs };
if (taskOutputSpecification.IsItemVector)
{
ErrorUtilities.VerifyThrow((itemName != null) && (itemName.Length > 0), "Need item type.");
// to store the outputs as items, use the string representations of the outputs as item-specs
foreach (object output in convertibleOutputs)
{
// if individual outputs in the array are null, ignore them
if (output != null)
{
string stringValueFromTask = (string)Convert.ChangeType(output, typeof(string), CultureInfo.InvariantCulture);
// attempting to put an empty string into an item is a no-op (bug #444501).
if (stringValueFromTask.Length > 0)
{
lookup.AddNewItem(new BuildItem(itemName, EscapingUtilities.Escape(stringValueFromTask)));
}
}
}
}
else
{
Debug.Assert(taskOutputSpecification.IsProperty);
ErrorUtilities.VerifyThrow((propertyName != null) && (propertyName.Length > 0), "Need property name.");
// to store an object array in a property, join all the string representations of the objects with
// semi-colons to make the property value
StringBuilder joinedOutputs = new StringBuilder();
foreach (object output in convertibleOutputs)
{
// if individual outputs in the array are null, ignore them
if (output != null)
{
if (joinedOutputs.Length > 0)
{
joinedOutputs.Append(';');
}
string stringValueFromTask = (string)Convert.ChangeType(output, typeof(string), CultureInfo.InvariantCulture);
joinedOutputs.Append(EscapingUtilities.Escape(stringValueFromTask));
}
}
lookup.SetProperty(new BuildProperty(propertyName, joinedOutputs.ToString(), PropertyType.OutputProperty));
}
}
}
/// <summary>
/// Copy constructor (called via Clone() - clearer semantics)
/// </summary>
private Lookup(Lookup that)
{
// Add the same tables from the original
foreach (LookupEntry entry in that.lookupEntries)
{
this.lookupEntries.AddLast(entry);
}
this.projectItems = that.projectItems;
this.itemDefinitionLibrary = that.itemDefinitionLibrary;
// Clones need to share an (item)clone table; the batching engine asks for items from the lookup,
// then populates buckets with them, which have clone lookups.
this.cloneTable = that.cloneTable;
}
internal TaskExecutionStateHelper
(
TaskExecutionMode howToExecuteTask,
Lookup lookupForInference,
Lookup lookupForExecution,
XmlElement taskXmlNode,
ITaskHost hostObject,
string projectFileOfTaskNode,
string parentProjectFullFileName,
string executionDirectory,
int nodeProxyId
)
: base(howToExecuteTask,
lookupForInference,
lookupForExecution,
taskXmlNode,
hostObject,
projectFileOfTaskNode,
parentProjectFullFileName,
executionDirectory,
nodeProxyId,
null
)
{
// Dont need to do anything
}
/// <summary>
/// Partitions the items consumed by the batchable object into buckets, where each bucket contains a set of items that
/// have the same value set on all item metadata consumed by the object.
/// </summary>
/// <remarks>
/// PERF NOTE: Given n items and m batching metadata that produce l buckets, it is usually the case that n > l > m,
/// because a batchable object typically uses one or two item metadata to control batching, and only has a handful of
/// buckets. The number of buckets is typically only large if a batchable object is using single-item batching
/// (where l == n). Any algorithm devised for bucketing therefore, should try to minimize n and l in its complexity
/// equation. The algorithm below has a complexity of O(n*lg(l)*m/2) in its comparisons, and is effectively O(n) when
/// l is small, and O(n*lg(n)) in the worst case as l -> n. However, note that the comparison complexity is not the
/// same as the operational complexity for this algorithm. The operational complexity of this algorithm is actually
/// O(n*m + n*lg(l)*m/2 + n*l/2 + n + l), which is effectively O(n^2) in the worst case. The additional complexity comes
/// from the array and metadata operations that are performed. However, those operations are extremely cheap compared
/// to the comparison operations, which dominate the time spent in this method.
/// </remarks>
/// <returns>ArrayList containing ItemBucket objects (can be empty), each one representing an execution batch.</returns>
private static ArrayList BucketConsumedItems(
XmlNode parentNode,
Lookup lookup,
Hashtable itemListsToBeBatched,
Dictionary<string, MetadataReference> consumedMetadataReferences
)
{
ErrorUtilities.VerifyThrow(itemListsToBeBatched.Count > 0, "Need item types consumed by the batchable object.");
ErrorUtilities.VerifyThrow(consumedMetadataReferences.Count > 0, "Need item metadata consumed by the batchable object.");
ArrayList buckets = new ArrayList();
// Get and iterate through the list of item names that we're supposed to batch on.
foreach (DictionaryEntry entry in itemListsToBeBatched)
{
string itemName = (string)entry.Key;
// Use the previously-fetched items, if possible
BuildItemGroup items;
if (entry.Value is BuildItemGroup)
{
items = (BuildItemGroup)entry.Value;
}
else
{
items = lookup.GetItems(itemName);
}
if (items != null)
{
foreach (BuildItem item in items)
{
// Get this item's values for all the metadata consumed by the batchable object.
Dictionary<string, string> itemMetadataValues = GetItemMetadataValues(parentNode, item, consumedMetadataReferences);
// put the metadata into a dummy bucket we can use for searching
ItemBucket dummyBucket = ItemBucket.GetDummyBucketForComparisons(itemMetadataValues);
// look through all previously created buckets to find a bucket whose items have the same values as
// this item for all metadata consumed by the batchable object
int matchingBucketIndex = buckets.BinarySearch(dummyBucket);
ItemBucket matchingBucket = (matchingBucketIndex >= 0)
? (ItemBucket)buckets[matchingBucketIndex]
: null;
// If we didn't find a bucket that matches this item, create a new one, adding
// this item to the bucket.
if (matchingBucket == null)
{
matchingBucket = new ItemBucket(itemListsToBeBatched.Keys, itemMetadataValues, lookup, buckets.Count);
// make sure to put the new bucket into the appropriate location
// in the sorted list as indicated by the binary search
// NOTE: observe the ~ operator (bitwise complement) in front of
// the index -- see MSDN for more information on the return value
// from the ArrayList.BinarySearch() method
buckets.Insert(~matchingBucketIndex, matchingBucket);
}
// We already have a bucket for this type of item, so add this item to
// the bucket.
matchingBucket.AddItem(item);
}
}
}
// Put the buckets back in the order in which they were discovered, so that the first
// item declared in the project file ends up in the first batch passed into the target/task.
ArrayList orderedBuckets = ArrayList.Repeat(null, buckets.Count);
foreach (ItemBucket bucket in buckets)
{
orderedBuckets[bucket.BucketSequenceNumber] = bucket;
}
return orderedBuckets;
}
/// <summary>
/// Of all the item lists that are referenced in this batchable object, which ones should we
/// batch on, and which ones should we just pass in wholesale to every invocation of the
/// target/task?
///
/// Rule #1. If the user has referenced any *qualified* item metadata such as %(EmbeddedResource.Culture),
/// then that item list "EmbeddedResource" will definitely get batched.
///
/// Rule #2. For all the unqualified item metadata such as %(Culture), we make sure that
/// every single item in every single item list being passed into the task contains a value
/// for that metadata. If not, it's an error. If so, we batch all of those item lists.
///
/// All other item lists will not be batched, and instead will be passed in wholesale to all buckets.
/// </summary>
/// <returns>Hashtable containing the item names that should be batched.</returns>
private static Hashtable GetItemListsToBeBatched(
XmlNode parentNode,
Dictionary<string, MetadataReference> consumedMetadataReferences, // Key is [string] potentially qualified metadata name
// Value is [struct MetadataReference]
Hashtable consumedItemReferenceNames, // Key is [string] item name.
// Value is always String.Empty (unused).
Lookup lookup
)
{
// The keys in this hashtable are the names of the items that we will batch on.
// The values are always String.Empty (not used).
Hashtable itemListsToBeBatched = new Hashtable(StringComparer.OrdinalIgnoreCase);
// Loop through all the metadata references and find the ones that are qualified
// with an item name.
foreach (MetadataReference consumedMetadataReference in consumedMetadataReferences.Values)
{
if (consumedMetadataReference.itemName != null)
{
// Rule #1. Qualified metadata reference.
// For metadata references that are qualified with an item name
// (e.g., %(EmbeddedResource.Culture) ), we add that item name to the list of
// consumed item names, even if the item name wasn't otherwise referenced via
// @(...) syntax, and even if every item in the list doesn't necessary contain
// a value for this metadata. This is the special power that you get by qualifying
// the metadata reference with an item name.
itemListsToBeBatched[consumedMetadataReference.itemName] = String.Empty;
// Also add this qualified item to the consumed item references list, because
// %(EmbeddedResource.Culture) effectively means that @(EmbeddedResource) is
// being consumed, even though we may not see literally "@(EmbeddedResource)"
// in the tag anywhere. Adding it to this list allows us (down below in this
// method) to check that every item in this list has a value for each
// unqualified metadata reference.
consumedItemReferenceNames = Utilities.CreateTableIfNecessary(consumedItemReferenceNames);
consumedItemReferenceNames[consumedMetadataReference.itemName] = String.Empty;
}
}
// Loop through all the metadata references and find the ones that are unqualified.
foreach (MetadataReference consumedMetadataReference in consumedMetadataReferences.Values)
{
if (consumedMetadataReference.itemName == null)
{
// Rule #2. Unqualified metadata reference.
// For metadata references that are unqualified, every single consumed item
// must contain a value for that metadata. If any item doesn't, it's an error
// to use unqualified metadata.
if (consumedItemReferenceNames != null)
{
foreach (string consumedItemName in consumedItemReferenceNames.Keys)
{
// Loop through all the items in the item list.
BuildItemGroup items = lookup.GetItems(consumedItemName);
if (items != null)
{
// Loop through all the items in the BuildItemGroup.
foreach (BuildItem item in items)
{
ProjectErrorUtilities.VerifyThrowInvalidProject(
item.HasMetadata(consumedMetadataReference.metadataName),
parentNode, "ItemDoesNotContainValueForUnqualifiedMetadata",
item.Include, consumedItemName, consumedMetadataReference.metadataName);
}
}
// This item list passes the test of having every single item containing
// a value for this metadata. Therefore, add this item list to the batching list.
// Also, to save doing lookup.GetItems again, put the items in the table as the value.
itemListsToBeBatched[consumedItemName] = items;
}
}
}
}
return itemListsToBeBatched;
}
internal ReadOnlyLookup(Lookup lookup)
{
this.lookup = lookup;
}
