/// <summary> /// The constructor obtains the state information and the /// callback delegate. /// </summary> internal TaskExecutionState ( TaskExecutionMode howToExecuteTask, Lookup lookupForInference, Lookup lookupForExecution, XmlElement taskXmlNode, ITaskHost hostObject, string projectFileOfTaskNode, string parentProjectFullFileName, string executionDirectory, int handleId, BuildEventContext buildEventContext ) { ErrorUtilities.VerifyThrow(taskXmlNode != null, "Must have task node"); this.howToExecuteTask = howToExecuteTask; this.lookupForInference = lookupForInference; this.lookupForExecution = lookupForExecution; this.hostObject = hostObject; this.projectFileOfTaskNode = projectFileOfTaskNode; this.parentProjectFullFileName = parentProjectFullFileName; this.executionDirectory = executionDirectory; this.handleId = handleId; this.buildEventContext = buildEventContext; this.taskXmlNode = taskXmlNode; }
/// <summary> /// Creates an instance of this class using the given bucket data. /// </summary> /// <param name="itemNames">Item types being batched on: null indicates no batching is occurring</param> /// <param name="itemMetadata">Hashtable of item metadata values: null indicates no batching is occurring</param> internal ItemBucket ( ICollection itemNames, Dictionary<string, string> itemMetadata, Lookup lookup, int bucketSequenceNumber ) { ErrorUtilities.VerifyThrow(lookup != null, "Need lookup."); // Create our own lookup just for this bucket this.lookup = lookup.Clone(); // Push down the items, so that item changes in this batch are not visible to parallel batches this.lookup.EnterScope(); // Add empty item groups for each of the item names, so that (unless items are added to this bucket) there are // no item types visible in this bucket among the item types being batched on if (itemNames != null) { foreach (string name in itemNames) { this.lookup.PopulateWithItems(name, new BuildItemGroup()); } } this.expander = new Expander(this.lookup.ReadOnlyLookup, itemMetadata); this.bucketSequenceNumber = bucketSequenceNumber; }
/// <summary> /// Determines how many times the batchable object needs to be executed (each execution is termed a "batch"), and prepares /// buckets of items to pass to the object in each batch. /// </summary> /// <returns>ArrayList containing ItemBucket objects, each one representing an execution batch.</returns> internal static ArrayList PrepareBatchingBuckets ( XmlNode parentNode, List<string> batchableObjectParameters, Lookup lookup ) { return PrepareBatchingBuckets(parentNode, batchableObjectParameters, lookup, null); }
/// <summary> /// Called to execute a task within a target. This method instantiates the task, sets its parameters, and executes it. /// </summary> internal void ExecuteTask(Lookup lookup) { ErrorUtilities.VerifyThrow(lookup != null, "Need to specify lookup."); if ((conditionAttribute != null) && !Utilities.EvaluateCondition(conditionAttribute.Value, conditionAttribute, new Expander(lookup.ReadOnlyLookup), null, ParserOptions.AllowPropertiesAndItemLists, loggingServices, buildEventContext)) { return; } // For these tasks, "execution" occurs the same whether we are asked to execute tasks // or merely asked to infer outputs switch (backingType) { case BackingType.PropertyGroup: ExecutePropertyGroup(lookup); break; case BackingType.ItemGroup: ExecuteItemGroup(lookup); break; } }
/// <summary> /// Determines how many times the batchable object needs to be executed (each execution is termed a "batch"), and prepares /// buckets of items to pass to the object in each batch. /// </summary> /// <param name="parentNode"></param> /// <param name="batchableObjectParameters"></param> /// <param name="lookup"></param> /// <param name="implicitBatchableItemType">Any item type that can be considered an implicit input to this batchable object. /// This is useful for items inside targets, where the item name is plainly an item type that's an "input" to the object.</param> /// <returns>ArrayList containing ItemBucket objects, each one representing an execution batch.</returns> internal static ArrayList PrepareBatchingBuckets( XmlNode parentNode, List<string> batchableObjectParameters, Lookup lookup, string implicitBatchableItemType ) { ErrorUtilities.VerifyThrow(parentNode != null, "Need the XML node that represents the batchable object."); ErrorUtilities.VerifyThrow(batchableObjectParameters != null, "Need the parameters of the batchable object to determine if it can be batched."); ErrorUtilities.VerifyThrow(lookup != null, "Need to specify the lookup."); ItemsAndMetadataPair pair = ExpressionShredder.GetReferencedItemNamesAndMetadata(batchableObjectParameters); // All the @(itemname) item list references in the tag, including transforms, etc. // The keys in the hashtable are the item names, and the values are all String.Empty (not used). Hashtable consumedItemReferences = pair.Items; // All the %(itemname.metadataname) references in the tag (not counting those embedded // inside item transforms), and note that the itemname portion is optional. // The keys in the returned hash table are the qualified metadata names (e.g. "EmbeddedResource.Culture" // or just "Culture"). The values are MetadataReference structs, which simply split out the item // name (possibly null) and the actual metadata name. Dictionary<string, MetadataReference> consumedMetadataReferences = pair.Metadata; ArrayList buckets = null; if (consumedMetadataReferences?.Count > 0) { // Add any item types that we were explicitly told to assume. if (implicitBatchableItemType != null) { consumedItemReferences = Utilities.CreateTableIfNecessary(consumedItemReferences); consumedItemReferences[implicitBatchableItemType] = String.Empty; } // This method goes through all the item list references and figures out which ones // will be participating in batching, and which ones won't. We get back a hashtable // where the key is the item name that will be participating in batching. The values // are all String.Empty (not used). This method may return additional item names // that weren't represented in "consumedItemReferences"... this would happen if there // were qualified metadata references in the consumedMetadataReferences table, such as // %(EmbeddedResource.Culture). Hashtable itemListsToBeBatched = GetItemListsToBeBatched(parentNode, consumedMetadataReferences, consumedItemReferences, lookup); // At this point, if there were any metadata references in the tag, but no item // references to batch on, we've got a problem because we can't figure out which // item lists the user wants us to batch. if (itemListsToBeBatched.Count == 0) { foreach (string unqualifiedMetadataName in consumedMetadataReferences.Keys) { // Of course, since this throws an exception, there's no way we're ever going // to really loop here... it's just that the foreach is the only way I can // figure out how to get data out of the hashtable without knowing any of the // keys! ProjectErrorUtilities.VerifyThrowInvalidProject(false, parentNode, "CannotReferenceItemMetadataWithoutItemName", unqualifiedMetadataName); } } else { // If the batchable object consumes item metadata as well as items to be batched, // we need to partition the items consumed by the object. buckets = BucketConsumedItems(parentNode, lookup, itemListsToBeBatched, consumedMetadataReferences); } } // if the batchable object does not consume any item metadata or items, or if the item lists it consumes are all // empty, then the object does not need to be batched if ((buckets == null) || (buckets.Count == 0)) { // create a default bucket that references the project items and properties -- this way we always have a bucket buckets = new ArrayList(1); buckets.Add(new ItemBucket(null, null, lookup, buckets.Count)); } return buckets; }
internal ReadOnlyLookup(Hashtable items, BuildPropertyGroup properties) { // Lookup only needs ItemDefinitionLibrary to mark new items with it. // Since we're a read-only lookup, we don't need a real one. this.lookup = new Lookup(items, properties, new ItemDefinitionLibrary(null)); }
/// <summary> /// Of all the item lists that are referenced in this batchable object, which ones should we /// batch on, and which ones should we just pass in wholesale to every invocation of the /// target/task? /// /// Rule #1. If the user has referenced any *qualified* item metadata such as %(EmbeddedResource.Culture), /// then that item list "EmbeddedResource" will definitely get batched. /// /// Rule #2. For all the unqualified item metadata such as %(Culture), we make sure that /// every single item in every single item list being passed into the task contains a value /// for that metadata. If not, it's an error. If so, we batch all of those item lists. /// /// All other item lists will not be batched, and instead will be passed in wholesale to all buckets. /// </summary> /// <returns>Hashtable containing the item names that should be batched.</returns> private static Hashtable GetItemListsToBeBatched ( XmlNode parentNode, Dictionary<string, MetadataReference> consumedMetadataReferences, // Key is [string] potentially qualified metadata name // Value is [struct MetadataReference] Hashtable consumedItemReferenceNames, // Key is [string] item name. // Value is always String.Empty (unused). Lookup lookup ) { // The keys in this hashtable are the names of the items that we will batch on. // The values are always String.Empty (not used). Hashtable itemListsToBeBatched = new Hashtable(StringComparer.OrdinalIgnoreCase); // Loop through all the metadata references and find the ones that are qualified // with an item name. foreach (MetadataReference consumedMetadataReference in consumedMetadataReferences.Values) { if (consumedMetadataReference.itemName != null) { // Rule #1. Qualified metadata reference. // For metadata references that are qualified with an item name // (e.g., %(EmbeddedResource.Culture) ), we add that item name to the list of // consumed item names, even if the item name wasn't otherwise referenced via // @(...) syntax, and even if every item in the list doesn't necessary contain // a value for this metadata. This is the special power that you get by qualifying // the metadata reference with an item name. itemListsToBeBatched[consumedMetadataReference.itemName] = String.Empty; // Also add this qualified item to the consumed item references list, because // %(EmbeddedResource.Culture) effectively means that @(EmbeddedResource) is // being consumed, even though we may not see literally "@(EmbeddedResource)" // in the tag anywhere. Adding it to this list allows us (down below in this // method) to check that every item in this list has a value for each // unqualified metadata reference. consumedItemReferenceNames = Utilities.CreateTableIfNecessary(consumedItemReferenceNames); consumedItemReferenceNames[consumedMetadataReference.itemName] = String.Empty; } } // Loop through all the metadata references and find the ones that are unqualified. foreach (MetadataReference consumedMetadataReference in consumedMetadataReferences.Values) { if (consumedMetadataReference.itemName == null) { // Rule #2. Unqualified metadata reference. // For metadata references that are unqualified, every single consumed item // must contain a value for that metadata. If any item doesn't, it's an error // to use unqualified metadata. if (consumedItemReferenceNames != null) { foreach (string consumedItemName in consumedItemReferenceNames.Keys) { // Loop through all the items in the item list. BuildItemGroup items = lookup.GetItems(consumedItemName); if (items != null) { // Loop through all the items in the BuildItemGroup. foreach (BuildItem item in items) { ProjectErrorUtilities.VerifyThrowInvalidProject( item.HasMetadata(consumedMetadataReference.metadataName), parentNode, "ItemDoesNotContainValueForUnqualifiedMetadata", item.Include, consumedItemName, consumedMetadataReference.metadataName); } } // This item list passes the test of having every single item containing // a value for this metadata. Therefore, add this item list to the batching list. // Also, to save doing lookup.GetItems again, put the items in the table as the value. itemListsToBeBatched[consumedItemName] = items; } } } } return itemListsToBeBatched; }
/// <summary> /// Since we could not derrive from TaskEngine and have no Interface, we need to overide the method in here and /// replace the calls when testing the class because of the calls to TaskEngine. If at a future time we get a mock task /// engine, Interface or a non sealed TaskEngine these methods can disappear. /// </summary> /// <returns></returns> virtual internal bool TaskEngineExecuteTask( TaskEngine taskEngine, TaskExecutionMode howTaskShouldBeExecuted, Lookup lookup ) { return taskEngine.ExecuteTask ( howTaskShouldBeExecuted, lookup ); }
private void InitializeForRunningTargetBatches() { // Make sure the <target> node has been given to us. ErrorUtilities.VerifyThrow(targetElement != null, "Need an XML node representing the <target> element."); // Make sure this really is the <target> node. ProjectXmlUtilities.VerifyThrowElementName(targetElement, XMakeElements.target); overallSuccess = true; projectContent = new Lookup(parentProject.evaluatedItemsByName, parentProject.evaluatedItems, parentProject.evaluatedProperties, parentProject.ItemDefinitionLibrary); // If we need to use the task thread - ie, we encounter a non-intrinsic task - we will need to make sure // the task thread only sees clones of the project items and properties. We insert a scope to allow us to // do that later. See comment in InitializeForRunningFirstNonIntrinsicTask() placeholderForClonedProjectContent = projectContent.EnterScope(); buckets = BatchingEngine.PrepareBatchingBuckets(targetElement, targetParameters, projectContent); currentBucket = 0; // Initialize the first bucket InitializeForRunningSingleTargetBatch(); }
internal ReadOnlyLookup(Lookup lookup) { this.lookup = lookup; }
private void CreateLookupAndEnterScope() { lookupPassedBetweenThreads = LookupHelpers.CreateEmptyLookup(); lookupPassedBetweenThreads.EnterScope(); }
private void GatherTaskItemOutputs(Lookup lookup, TaskOutput taskOutputSpecification, string itemName, string propertyName, object outputs) { // if the task has generated outputs (if it didn't, don't do anything) if (outputs != null) { ITaskItem[] taskItemOutputs = (outputs is ITaskItem[]) ? (ITaskItem[])outputs : new ITaskItem[] { (ITaskItem)outputs }; if (taskOutputSpecification.IsItemVector) { ErrorUtilities.VerifyThrow((itemName != null) && (itemName.Length > 0), "Need item type."); foreach (ITaskItem output in taskItemOutputs) { // if individual items in the array are null, ignore them if (output != null) { lookup.AddNewItem(new BuildItem(itemName, output)); } } } else { Debug.Assert(taskOutputSpecification.IsProperty); ErrorUtilities.VerifyThrow((propertyName != null) && (propertyName.Length > 0), "Need property name."); // to store an ITaskItem array in a property, join all the item-specs with semi-colons to make the // property value, and ignore/discard the attributes on the ITaskItems StringBuilder joinedOutputs = new StringBuilder(); foreach (ITaskItem output in taskItemOutputs) { // if individual items in the array are null, ignore them if (output != null) { if (joinedOutputs.Length > 0) { joinedOutputs.Append(';'); } joinedOutputs.Append(EscapingUtilities.Escape(output.ItemSpec)); } } lookup.SetProperty(new BuildProperty(propertyName, joinedOutputs.ToString(), PropertyType.OutputProperty)); } } }
/// <summary> /// Execute a PropertyGroup element, including each child property /// </summary> private void ExecutePropertyGroup(Lookup lookup) { foreach (BuildProperty property in backingPropertyGroup) { ArrayList buckets = null; try { // Find all the metadata references in order to create buckets List<string> parameterValues = new List<string>(); GetBatchableValuesFromProperty(parameterValues, property); buckets = BatchingEngine.PrepareBatchingBuckets(taskNodeXmlElement, parameterValues, lookup); // "Execute" each bucket foreach (ItemBucket bucket in buckets) { if (Utilities.EvaluateCondition(property.Condition, property.ConditionAttribute, bucket.Expander, null, ParserOptions.AllowAll, loggingServices, buildEventContext)) { // Check for a reserved name now, so it fails right here instead of later when the property eventually reaches // the outer scope. ProjectErrorUtilities.VerifyThrowInvalidProject(!ReservedPropertyNames.IsReservedProperty(property.Name), property.PropertyElement, "CannotModifyReservedProperty", property.Name); property.Evaluate(bucket.Expander); bucket.Lookup.SetProperty(property); } } } finally { if (buckets != null) { // Propagate the property changes to the bucket above foreach (ItemBucket bucket in buckets) { bucket.Lookup.LeaveScope(); } } } } }
/// <summary> /// Execute an ItemGroup element, including each child item expression /// </summary> private void ExecuteItemGroup(Lookup lookup) { foreach (BuildItemGroupChildXml child in backingBuildItemGroupChildren) { ArrayList buckets = null; try { List<string> parameterValues = new List<string>(); GetBatchableValuesFromBuildItemGroupChild(parameterValues, child); buckets = BatchingEngine.PrepareBatchingBuckets(taskNodeXmlElement, parameterValues, lookup, child.Name); // "Execute" each bucket foreach (ItemBucket bucket in buckets) { // Gather the outputs, but don't make them visible to other buckets switch (child.ChildType) { case ChildType.BuildItemAdd: // It's an item -- we're "adding" items to the world ExecuteAdd(child, bucket); break; case ChildType.BuildItemRemove: // It's a remove -- we're "removing" items from the world ExecuteRemove(child, bucket); break; case ChildType.BuildItemModify: // It's a modify -- changing existing items ExecuteModify(child, bucket); break; } } } finally { if (buckets != null) { // Propagate the item changes to the bucket above foreach (ItemBucket bucket in buckets) { bucket.Lookup.LeaveScope(); } } } } }
/// <summary> /// Determines how many times the batchable object needs to be executed (each execution is termed a "batch"), and prepares /// buckets of items to pass to the object in each batch. /// </summary> /// <param name="parentNode"></param> /// <param name="batchableObjectParameters"></param> /// <param name="lookup"></param> /// <param name="implicitBatchableItemType">Any item type that can be considered an implicit input to this batchable object. /// This is useful for items inside targets, where the item name is plainly an item type that's an "input" to the object.</param> /// <returns>ArrayList containing ItemBucket objects, each one representing an execution batch.</returns> internal static ArrayList PrepareBatchingBuckets ( XmlNode parentNode, List<string> batchableObjectParameters, Lookup lookup, string implicitBatchableItemType ) { ErrorUtilities.VerifyThrow(parentNode != null, "Need the XML node that represents the batchable object."); ErrorUtilities.VerifyThrow(batchableObjectParameters != null, "Need the parameters of the batchable object to determine if it can be batched."); ErrorUtilities.VerifyThrow(lookup != null, "Need to specify the lookup."); ItemsAndMetadataPair pair = ExpressionShredder.GetReferencedItemNamesAndMetadata(batchableObjectParameters); // All the @(itemname) item list references in the tag, including transforms, etc. // The keys in the hashtable are the item names, and the values are all String.Empty (not used). Hashtable consumedItemReferences = pair.Items; // All the %(itemname.metadataname) references in the tag (not counting those embedded // inside item transforms), and note that the itemname portion is optional. // The keys in the returned hash table are the qualified metadata names (e.g. "EmbeddedResource.Culture" // or just "Culture"). The values are MetadataReference structs, which simply split out the item // name (possibly null) and the actual metadata name. Dictionary<string, MetadataReference> consumedMetadataReferences = pair.Metadata; ArrayList buckets = null; if (consumedMetadataReferences != null && consumedMetadataReferences.Count > 0) { // Add any item types that we were explicitly told to assume. if (implicitBatchableItemType != null) { consumedItemReferences = Utilities.CreateTableIfNecessary(consumedItemReferences); consumedItemReferences[implicitBatchableItemType] = String.Empty; } // This method goes through all the item list references and figures out which ones // will be participating in batching, and which ones won't. We get back a hashtable // where the key is the item name that will be participating in batching. The values // are all String.Empty (not used). This method may return additional item names // that weren't represented in "consumedItemReferences"... this would happen if there // were qualified metadata references in the consumedMetadataReferences table, such as // %(EmbeddedResource.Culture). Hashtable itemListsToBeBatched = GetItemListsToBeBatched(parentNode, consumedMetadataReferences, consumedItemReferences, lookup); // At this point, if there were any metadata references in the tag, but no item // references to batch on, we've got a problem because we can't figure out which // item lists the user wants us to batch. if (itemListsToBeBatched.Count == 0) { foreach (string unqualifiedMetadataName in consumedMetadataReferences.Keys) { // Of course, since this throws an exception, there's no way we're ever going // to really loop here... it's just that the foreach is the only way I can // figure out how to get data out of the hashtable without knowing any of the // keys! ProjectErrorUtilities.VerifyThrowInvalidProject(false, parentNode, "CannotReferenceItemMetadataWithoutItemName", unqualifiedMetadataName); } } else { // If the batchable object consumes item metadata as well as items to be batched, // we need to partition the items consumed by the object. buckets = BucketConsumedItems(parentNode, lookup, itemListsToBeBatched, consumedMetadataReferences); } } // if the batchable object does not consume any item metadata or items, or if the item lists it consumes are all // empty, then the object does not need to be batched if ((buckets == null) || (buckets.Count == 0)) { // create a default bucket that references the project items and properties -- this way we always have a bucket buckets = new ArrayList(1); buckets.Add(new ItemBucket(null, null, lookup, buckets.Count)); } return buckets; }
/// <summary> /// Partitions the items consumed by the batchable object into buckets, where each bucket contains a set of items that /// have the same value set on all item metadata consumed by the object. /// </summary> /// <remarks> /// PERF NOTE: Given n items and m batching metadata that produce l buckets, it is usually the case that n > l > m, /// because a batchable object typically uses one or two item metadata to control batching, and only has a handful of /// buckets. The number of buckets is typically only large if a batchable object is using single-item batching /// (where l == n). Any algorithm devised for bucketing therefore, should try to minimize n and l in its complexity /// equation. The algorithm below has a complexity of O(n*lg(l)*m/2) in its comparisons, and is effectively O(n) when /// l is small, and O(n*lg(n)) in the worst case as l -> n. However, note that the comparison complexity is not the /// same as the operational complexity for this algorithm. The operational complexity of this algorithm is actually /// O(n*m + n*lg(l)*m/2 + n*l/2 + n + l), which is effectively O(n^2) in the worst case. The additional complexity comes /// from the array and metadata operations that are performed. However, those operations are extremely cheap compared /// to the comparison operations, which dominate the time spent in this method. /// </remarks> /// <returns>ArrayList containing ItemBucket objects (can be empty), each one representing an execution batch.</returns> private static ArrayList BucketConsumedItems ( XmlNode parentNode, Lookup lookup, Hashtable itemListsToBeBatched, Dictionary<string, MetadataReference> consumedMetadataReferences ) { ErrorUtilities.VerifyThrow(itemListsToBeBatched.Count > 0, "Need item types consumed by the batchable object."); ErrorUtilities.VerifyThrow(consumedMetadataReferences.Count > 0, "Need item metadata consumed by the batchable object."); ArrayList buckets = new ArrayList(); // Get and iterate through the list of item names that we're supposed to batch on. foreach (DictionaryEntry entry in itemListsToBeBatched) { string itemName = (string)entry.Key; // Use the previously-fetched items, if possible BuildItemGroup items; if (entry.Value is BuildItemGroup) { items = (BuildItemGroup)entry.Value; } else { items = lookup.GetItems(itemName); } if (items != null) { foreach (BuildItem item in items) { // Get this item's values for all the metadata consumed by the batchable object. Dictionary<string, string> itemMetadataValues = GetItemMetadataValues(parentNode, item, consumedMetadataReferences); // put the metadata into a dummy bucket we can use for searching ItemBucket dummyBucket = ItemBucket.GetDummyBucketForComparisons(itemMetadataValues); // look through all previously created buckets to find a bucket whose items have the same values as // this item for all metadata consumed by the batchable object int matchingBucketIndex = buckets.BinarySearch(dummyBucket); ItemBucket matchingBucket = (matchingBucketIndex >= 0) ? (ItemBucket)buckets[matchingBucketIndex] : null; // If we didn't find a bucket that matches this item, create a new one, adding // this item to the bucket. if (null == matchingBucket) { matchingBucket = new ItemBucket(itemListsToBeBatched.Keys, itemMetadataValues, lookup, buckets.Count); // make sure to put the new bucket into the appropriate location // in the sorted list as indicated by the binary search // NOTE: observe the ~ operator (bitwise complement) in front of // the index -- see MSDN for more information on the return value // from the ArrayList.BinarySearch() method buckets.Insert(~matchingBucketIndex, matchingBucket); } // We already have a bucket for this type of item, so add this item to // the bucket. matchingBucket.AddItem(item); } } } // Put the buckets back in the order in which they were discovered, so that the first // item declared in the project file ends up in the first batch passed into the target/task. ArrayList orderedBuckets = ArrayList.Repeat(null, buckets.Count); foreach (ItemBucket bucket in buckets) { orderedBuckets[bucket.BucketSequenceNumber] = bucket; } return orderedBuckets; }
internal static Lookup CreateLookup(Hashtable items) { ItemDefinitionLibrary itemDefinitionLibrary = CreateEmptyEvaluatedItemDefinitionLibrary(); Lookup lookup = new Lookup(items, null, itemDefinitionLibrary); return lookup; }
/// <summary> /// Uses the given task output specification to (statically) infer the task's outputs. /// </summary> /// <param name="taskOutputSpecification"></param> /// <param name="taskParameterName"></param> /// <param name="itemName">can be null</param> /// <param name="propertyName">can be null</param> /// <param name="bucket"></param> private void InferTaskOutputs ( Lookup lookup, TaskOutput taskOutputSpecification, string taskParameterName, string itemName, string propertyName, ItemBucket bucket ) { // if the task has a value set for the output parameter, expand all embedded properties and item metadata in it XmlAttribute taskParameterAttribute = null; // Lookup attribute name needs to be case-insensitive // DevDiv bugs: 33981 foreach (XmlAttribute taskNodeAttribute in taskNode.Attributes) { if (String.Compare(taskNodeAttribute.Name, taskParameterName, StringComparison.OrdinalIgnoreCase) == 0) { taskParameterAttribute = taskNodeAttribute; break; } } if (taskParameterAttribute != null) { if (taskOutputSpecification.IsItemVector) { // This is an output item. ErrorUtilities.VerifyThrow((itemName != null) && (itemName.Length > 0), "Need item type."); // Expand only with properties first, so that expressions like Include="@(foo)" will transfer the metadata of the "foo" items as well, not just their item specs. Expander propertyAndMetadataExpander = new Expander(bucket.Expander, ExpanderOptions.ExpandPropertiesAndMetadata); List<string> outputItemSpecs = propertyAndMetadataExpander.ExpandAllIntoStringListLeaveEscaped(taskParameterAttribute); foreach (string outputItemSpec in outputItemSpecs) { BuildItemGroup items = bucket.Expander.ExpandSingleItemListExpressionIntoItemsLeaveEscaped(outputItemSpec, taskParameterAttribute); // if the output item-spec is an item vector, get the items in it if (items != null) { foreach (BuildItem item in items) { // we want to preserve the attributes on the item BuildItem clonedItem = item.VirtualClone(); // but we do need to change the item type clonedItem.Name = itemName; lookup.AddNewItem(clonedItem); } } else { // if the output item-spec is not an item vector, accept it as-is lookup.AddNewItem(new BuildItem(itemName, outputItemSpec)); } } } else { // This is an output property. Debug.Assert(taskOutputSpecification.IsProperty); ErrorUtilities.VerifyThrow((propertyName != null) && (propertyName.Length > 0), "Need property name."); string taskParameterValue = bucket.Expander.ExpandAllIntoString(taskParameterAttribute); if (taskParameterValue.Length > 0) { lookup.SetProperty(new BuildProperty(propertyName, taskParameterValue, PropertyType.OutputProperty)); } } } }
internal static Lookup CreateLookup(BuildPropertyGroup properties, Hashtable items) { ItemDefinitionLibrary itemDefinitionLibrary = CreateEmptyEvaluatedItemDefinitionLibrary(); Lookup lookup = new Lookup(items, properties, itemDefinitionLibrary); return lookup; }
/// <summary> /// Called to execute a task within a target. This method instantiates the task, sets its parameters, and executes it. /// </summary> /// <returns>true, if successful</returns> internal bool ExecuteTask(TaskExecutionMode howToExecuteTask, Lookup lookup) { ErrorUtilities.VerifyThrow(lookup != null, "Need to specify items available to task."); bool taskExecutedSuccessfully = false; EngineProxy engineProxy = null; ArrayList buckets = null; try { engineProxy = new EngineProxy(parentModule, handleId, parentProjectFullFileName, projectFileOfTaskNode, loggingServices, buildEventContext); List<string> taskParameterValues = CreateListOfParameterValues(); buckets = BatchingEngine.PrepareBatchingBuckets(taskNode, taskParameterValues, lookup); lookupHash = null; // Only create a hash table if there are more than one bucket as this is the only time a property can be overridden if (buckets.Count > 1) { lookupHash = Utilities.CreateTableIfNecessary((Hashtable)null); } // Loop through each of the batch buckets and execute them one at a time for (int i=0; i < buckets.Count; i++) { // Execute the batch bucket, pass in which bucket we are executing so that we know when to get a new taskId for the bucket. taskExecutedSuccessfully = ExecuteBucket(engineProxy, (ItemBucket)buckets[i], i, howToExecuteTask); if (!taskExecutedSuccessfully) { break; } } } finally { // Remove the AssemblyResolve handler in the default AppDomain, we are done with the task. if (resolver != null) { resolver.RemoveHandler(); } if (engineProxy != null) { engineProxy.MarkAsInActive(); } // Now all task batches are done, apply all item adds to the outer // target batch; we do this even if the task wasn't found (in that case, // no items or properties will have been added to the scope) if (buckets != null) { foreach (ItemBucket bucket in buckets) { bucket.Lookup.LeaveScope(); } } } return taskExecutedSuccessfully; }
internal static Lookup CreateEmptyLookup() { ItemDefinitionLibrary itemDefinitionLibrary = CreateEmptyEvaluatedItemDefinitionLibrary(); Lookup lookup = new Lookup(null, null, itemDefinitionLibrary); return lookup; }
/// <summary> /// Uses the given task output specification to grab the task's outputs using .NET reflection. /// </summary> /// <remarks> /// This method is "internal" for unit-testing purposes only. /// </remarks> /// <param name="taskOutputSpecification"></param> /// <param name="taskParameterName"></param> /// <param name="itemName">can be null</param> /// <param name="propertyName">can be null</param> /// <param name="task"></param> /// <returns>true, if successful</returns> internal bool GatherGeneratedTaskOutputs ( Lookup lookup, TaskOutput taskOutputSpecification, string taskParameterName, string itemName, string propertyName, ITask task) { ErrorUtilities.VerifyThrow(task != null, "Need instantiated task to retrieve outputs from."); bool gatheredGeneratedOutputsSuccessfully = true; try { PropertyInfo parameter = TaskClass.GetProperty(taskParameterName); // flag an error if we find a parameter that has no .NET property equivalent ProjectErrorUtilities.VerifyThrowInvalidProject(parameter != null, taskOutputSpecification.TaskParameterAttribute, "UnexpectedTaskOutputAttribute", taskParameterName, TaskName); // output parameters must have their corresponding .NET properties marked with the Output attribute ProjectErrorUtilities.VerifyThrowInvalidProject(TaskClass.GetNamesOfPropertiesWithOutputAttribute().ContainsKey(taskParameterName), taskOutputSpecification.TaskParameterAttribute, "UnmarkedOutputTaskParameter", parameter.Name, TaskName); // grab the outputs from the task's designated output parameter (which is a .NET property) object outputs = parameter.GetValue(task, null); Type type = parameter.PropertyType; // don't use the C# "is" operator as it always returns false if the object is null if ( typeof(ITaskItem[]).IsAssignableFrom(type) || /* ITaskItem array or derived type, or */ typeof(ITaskItem).IsAssignableFrom(type) /* ITaskItem or derived type */ ) { GatherTaskItemOutputs(lookup, taskOutputSpecification, itemName, propertyName, outputs); } // don't use the C# "is" operator as it always returns false if the object is null else if ( (type.IsArray && type.GetElementType().IsValueType) || /* array of value types, or */ (type == typeof(string[])) || /* string array, or */ (type.IsValueType) || /* value type, or */ (type == typeof(string)) /* string */ ) { GatherArrayStringAndValueOutputs(lookup, taskOutputSpecification, itemName, propertyName, parameter, outputs); } else { ProjectErrorUtilities.VerifyThrowInvalidProject(false, taskOutputSpecification.TaskParameterAttribute, "UnsupportedTaskParameterTypeError", parameter.PropertyType, parameter.Name, TaskName); } } // handle invalid TaskItems in task outputs catch (InvalidOperationException e) { loggingServices.LogError(buildEventContext, Utilities.CreateBuildEventFileInfo(taskOutputSpecification.TaskParameterAttribute, projectFileOfTaskNode), "InvalidTaskItemsInTaskOutputs", TaskName, taskParameterName, e.Message); gatheredGeneratedOutputsSuccessfully = false; } // handle any exception thrown by the task's getter catch (TargetInvocationException e) { // Exception thrown by the called code itself // Log the stack, so the task vendor can fix their code // Log the task line number, whatever the value of ContinueOnError; // because this will be a hard error anyway. loggingServices.LogFatalTaskError(buildEventContext, e.InnerException, CreateBuildEventFileInfoForTask(), TaskName); // We do not recover from a task exception while getting outputs, // so do not merely set gatheredGeneratedOutputsSuccessfully = false; here ProjectErrorUtilities.VerifyThrowInvalidProject(false, taskOutputSpecification.TaskParameterAttribute, "FailedToRetrieveTaskOutputs", TaskName, taskParameterName, e.InnerException.Message); } catch (Exception e) // Catching Exception, but rethrowing unless it's a well-known exception. { if (ExceptionHandling.NotExpectedReflectionException(e)) throw; ProjectErrorUtilities.VerifyThrowInvalidProject(false, taskOutputSpecification.TaskParameterAttribute, "FailedToRetrieveTaskOutputs", TaskName, taskParameterName, e.Message); } return gatheredGeneratedOutputsSuccessfully; }
private void ExecuteTask(IntrinsicTask task, Lookup lookup) { if (lookup == null) { lookup = LookupHelpers.CreateEmptyLookup(); } task.ExecuteTask(lookup); }
private void InitializeForRunningSingleTargetBatch() { // Verify that the target is in the right state ErrorUtilities.VerifyThrow(inProgressBuildState == InProgressBuildState.RunningTasks, "Wrong state"); // Check if the current task number is valid ErrorUtilities.VerifyThrow(currentBucket < buckets.Count, "No buckets left"); Hashtable changedTargetInputs = null; Hashtable upToDateTargetInputs = null; howToBuild = DependencyAnalysisResult.FullBuild; ItemBucket bucket = (ItemBucket)buckets[currentBucket]; // For the first batch of a target use the targets original targetID. for each batch after the first one use a uniqueId to identity the target in the batch if (currentBucket != 0) { targetBuildEventContext = new BuildEventContext(targetBuildEventContext.NodeId, parentEngine.GetNextTargetId(), targetBuildEventContext.ProjectContextId, targetBuildEventContext.TaskId); } // Flag the start of the target. parentEngine.LoggingServices.LogTargetStarted( targetBuildEventContext, targetClass.Name, this.parentProject.FullFileName, targetClass.ProjectFileOfTargetElement); loggedTargetStart = true; // Figure out how we should build the target TargetDependencyAnalyzer dependencyAnalyzer = new TargetDependencyAnalyzer(parentProject.ProjectDirectory, targetClass, parentEngine.LoggingServices, targetBuildEventContext); howToBuild = dependencyAnalyzer.PerformDependencyAnalysis(bucket, out changedTargetInputs, out upToDateTargetInputs); targetBuildSuccessful = true; exitBatchDueToError = false; // If we need to build the target - initialize the data structure for // running the tasks if ((howToBuild != DependencyAnalysisResult.SkipNoInputs) && (howToBuild != DependencyAnalysisResult.SkipNoOutputs)) { // Within each target batch items are divided into lookup and execution; they must be // kept separate: enforce this by cloning and entering scope lookupForInference = bucket.Lookup; lookupForExecution = bucket.Lookup.Clone(); lookupForInference.EnterScope(); lookupForExecution.EnterScope(); // if we're doing an incremental build, we need to effectively run the task twice -- once // to infer the outputs for up-to-date input items, and once to actually execute the task; // as a result we need separate sets of item and property collections to track changes if (howToBuild == DependencyAnalysisResult.IncrementalBuild) { // subset the relevant items to those that are up-to-date foreach (DictionaryEntry upToDateTargetInputsEntry in upToDateTargetInputs) { lookupForInference.PopulateWithItems((string)upToDateTargetInputsEntry.Key, (BuildItemGroup)upToDateTargetInputsEntry.Value); } // subset the relevant items to those that have changed foreach (DictionaryEntry changedTargetInputsEntry in changedTargetInputs) { lookupForExecution.PopulateWithItems((string)changedTargetInputsEntry.Key, (BuildItemGroup)changedTargetInputsEntry.Value); } } projectFileOfTaskNode = XmlUtilities.GetXmlNodeFile(targetElement, parentProject.FullFileName); // count the tasks in the target currentTask = 0; skippedNodeCount = 0; } else { currentTask = targetElement.ChildNodes.Count; } }
/// <summary> /// Mock out and override the method inside of TaskExecutionState which makes the calls to the task engine. /// </summary> internal override bool TaskEngineExecuteTask(TaskEngine taskEngine, TaskExecutionMode howToExecuteTask, Lookup lookup) { return targetInferenceSuccessful; }
private void GatherArrayStringAndValueOutputs(Lookup lookup, TaskOutput taskOutputSpecification, string itemName, string propertyName, PropertyInfo parameter, object outputs) { // if the task has generated outputs (if it didn't, don't do anything) if (outputs != null) { Array convertibleOutputs = (parameter.PropertyType.IsArray) ? (Array)outputs : new object[] { outputs }; if (taskOutputSpecification.IsItemVector) { ErrorUtilities.VerifyThrow((itemName != null) && (itemName.Length > 0), "Need item type."); // to store the outputs as items, use the string representations of the outputs as item-specs foreach (object output in convertibleOutputs) { // if individual outputs in the array are null, ignore them if (output != null) { string stringValueFromTask = (string)Convert.ChangeType(output, typeof(string), CultureInfo.InvariantCulture); // attempting to put an empty string into an item is a no-op (bug #444501). if (stringValueFromTask.Length > 0) { lookup.AddNewItem(new BuildItem(itemName, EscapingUtilities.Escape(stringValueFromTask))); } } } } else { Debug.Assert(taskOutputSpecification.IsProperty); ErrorUtilities.VerifyThrow((propertyName != null) && (propertyName.Length > 0), "Need property name."); // to store an object array in a property, join all the string representations of the objects with // semi-colons to make the property value StringBuilder joinedOutputs = new StringBuilder(); foreach (object output in convertibleOutputs) { // if individual outputs in the array are null, ignore them if (output != null) { if (joinedOutputs.Length > 0) { joinedOutputs.Append(';'); } string stringValueFromTask = (string)Convert.ChangeType(output, typeof(string), CultureInfo.InvariantCulture); joinedOutputs.Append(EscapingUtilities.Escape(stringValueFromTask)); } } lookup.SetProperty(new BuildProperty(propertyName, joinedOutputs.ToString(), PropertyType.OutputProperty)); } } }
/// <summary> /// Copy constructor (called via Clone() - clearer semantics) /// </summary> private Lookup(Lookup that) { // Add the same tables from the original foreach (LookupEntry entry in that.lookupEntries) { this.lookupEntries.AddLast(entry); } this.projectItems = that.projectItems; this.itemDefinitionLibrary = that.itemDefinitionLibrary; // Clones need to share an (item)clone table; the batching engine asks for items from the lookup, // then populates buckets with them, which have clone lookups. this.cloneTable = that.cloneTable; }
internal TaskExecutionStateHelper ( TaskExecutionMode howToExecuteTask, Lookup lookupForInference, Lookup lookupForExecution, XmlElement taskXmlNode, ITaskHost hostObject, string projectFileOfTaskNode, string parentProjectFullFileName, string executionDirectory, int nodeProxyId ) : base(howToExecuteTask, lookupForInference, lookupForExecution, taskXmlNode, hostObject, projectFileOfTaskNode, parentProjectFullFileName, executionDirectory, nodeProxyId, null ) { // Dont need to do anything }
/// <summary> /// Partitions the items consumed by the batchable object into buckets, where each bucket contains a set of items that /// have the same value set on all item metadata consumed by the object. /// </summary> /// <remarks> /// PERF NOTE: Given n items and m batching metadata that produce l buckets, it is usually the case that n > l > m, /// because a batchable object typically uses one or two item metadata to control batching, and only has a handful of /// buckets. The number of buckets is typically only large if a batchable object is using single-item batching /// (where l == n). Any algorithm devised for bucketing therefore, should try to minimize n and l in its complexity /// equation. The algorithm below has a complexity of O(n*lg(l)*m/2) in its comparisons, and is effectively O(n) when /// l is small, and O(n*lg(n)) in the worst case as l -> n. However, note that the comparison complexity is not the /// same as the operational complexity for this algorithm. The operational complexity of this algorithm is actually /// O(n*m + n*lg(l)*m/2 + n*l/2 + n + l), which is effectively O(n^2) in the worst case. The additional complexity comes /// from the array and metadata operations that are performed. However, those operations are extremely cheap compared /// to the comparison operations, which dominate the time spent in this method. /// </remarks> /// <returns>ArrayList containing ItemBucket objects (can be empty), each one representing an execution batch.</returns> private static ArrayList BucketConsumedItems( XmlNode parentNode, Lookup lookup, Hashtable itemListsToBeBatched, Dictionary<string, MetadataReference> consumedMetadataReferences ) { ErrorUtilities.VerifyThrow(itemListsToBeBatched.Count > 0, "Need item types consumed by the batchable object."); ErrorUtilities.VerifyThrow(consumedMetadataReferences.Count > 0, "Need item metadata consumed by the batchable object."); ArrayList buckets = new ArrayList(); // Get and iterate through the list of item names that we're supposed to batch on. foreach (DictionaryEntry entry in itemListsToBeBatched) { string itemName = (string)entry.Key; // Use the previously-fetched items, if possible BuildItemGroup items; if (entry.Value is BuildItemGroup) { items = (BuildItemGroup)entry.Value; } else { items = lookup.GetItems(itemName); } if (items != null) { foreach (BuildItem item in items) { // Get this item's values for all the metadata consumed by the batchable object. Dictionary<string, string> itemMetadataValues = GetItemMetadataValues(parentNode, item, consumedMetadataReferences); // put the metadata into a dummy bucket we can use for searching ItemBucket dummyBucket = ItemBucket.GetDummyBucketForComparisons(itemMetadataValues); // look through all previously created buckets to find a bucket whose items have the same values as // this item for all metadata consumed by the batchable object int matchingBucketIndex = buckets.BinarySearch(dummyBucket); ItemBucket matchingBucket = (matchingBucketIndex >= 0) ? (ItemBucket)buckets[matchingBucketIndex] : null; // If we didn't find a bucket that matches this item, create a new one, adding // this item to the bucket. if (matchingBucket == null) { matchingBucket = new ItemBucket(itemListsToBeBatched.Keys, itemMetadataValues, lookup, buckets.Count); // make sure to put the new bucket into the appropriate location // in the sorted list as indicated by the binary search // NOTE: observe the ~ operator (bitwise complement) in front of // the index -- see MSDN for more information on the return value // from the ArrayList.BinarySearch() method buckets.Insert(~matchingBucketIndex, matchingBucket); } // We already have a bucket for this type of item, so add this item to // the bucket. matchingBucket.AddItem(item); } } } // Put the buckets back in the order in which they were discovered, so that the first // item declared in the project file ends up in the first batch passed into the target/task. ArrayList orderedBuckets = ArrayList.Repeat(null, buckets.Count); foreach (ItemBucket bucket in buckets) { orderedBuckets[bucket.BucketSequenceNumber] = bucket; } return orderedBuckets; }
/// <summary> /// Of all the item lists that are referenced in this batchable object, which ones should we /// batch on, and which ones should we just pass in wholesale to every invocation of the /// target/task? /// /// Rule #1. If the user has referenced any *qualified* item metadata such as %(EmbeddedResource.Culture), /// then that item list "EmbeddedResource" will definitely get batched. /// /// Rule #2. For all the unqualified item metadata such as %(Culture), we make sure that /// every single item in every single item list being passed into the task contains a value /// for that metadata. If not, it's an error. If so, we batch all of those item lists. /// /// All other item lists will not be batched, and instead will be passed in wholesale to all buckets. /// </summary> /// <returns>Hashtable containing the item names that should be batched.</returns> private static Hashtable GetItemListsToBeBatched( XmlNode parentNode, Dictionary<string, MetadataReference> consumedMetadataReferences, // Key is [string] potentially qualified metadata name // Value is [struct MetadataReference] Hashtable consumedItemReferenceNames, // Key is [string] item name. // Value is always String.Empty (unused). Lookup lookup ) { // The keys in this hashtable are the names of the items that we will batch on. // The values are always String.Empty (not used). Hashtable itemListsToBeBatched = new Hashtable(StringComparer.OrdinalIgnoreCase); // Loop through all the metadata references and find the ones that are qualified // with an item name. foreach (MetadataReference consumedMetadataReference in consumedMetadataReferences.Values) { if (consumedMetadataReference.itemName != null) { // Rule #1. Qualified metadata reference. // For metadata references that are qualified with an item name // (e.g., %(EmbeddedResource.Culture) ), we add that item name to the list of // consumed item names, even if the item name wasn't otherwise referenced via // @(...) syntax, and even if every item in the list doesn't necessary contain // a value for this metadata. This is the special power that you get by qualifying // the metadata reference with an item name. itemListsToBeBatched[consumedMetadataReference.itemName] = String.Empty; // Also add this qualified item to the consumed item references list, because // %(EmbeddedResource.Culture) effectively means that @(EmbeddedResource) is // being consumed, even though we may not see literally "@(EmbeddedResource)" // in the tag anywhere. Adding it to this list allows us (down below in this // method) to check that every item in this list has a value for each // unqualified metadata reference. consumedItemReferenceNames = Utilities.CreateTableIfNecessary(consumedItemReferenceNames); consumedItemReferenceNames[consumedMetadataReference.itemName] = String.Empty; } } // Loop through all the metadata references and find the ones that are unqualified. foreach (MetadataReference consumedMetadataReference in consumedMetadataReferences.Values) { if (consumedMetadataReference.itemName == null) { // Rule #2. Unqualified metadata reference. // For metadata references that are unqualified, every single consumed item // must contain a value for that metadata. If any item doesn't, it's an error // to use unqualified metadata. if (consumedItemReferenceNames != null) { foreach (string consumedItemName in consumedItemReferenceNames.Keys) { // Loop through all the items in the item list. BuildItemGroup items = lookup.GetItems(consumedItemName); if (items != null) { // Loop through all the items in the BuildItemGroup. foreach (BuildItem item in items) { ProjectErrorUtilities.VerifyThrowInvalidProject( item.HasMetadata(consumedMetadataReference.metadataName), parentNode, "ItemDoesNotContainValueForUnqualifiedMetadata", item.Include, consumedItemName, consumedMetadataReference.metadataName); } } // This item list passes the test of having every single item containing // a value for this metadata. Therefore, add this item list to the batching list. // Also, to save doing lookup.GetItems again, put the items in the table as the value. itemListsToBeBatched[consumedItemName] = items; } } } } return itemListsToBeBatched; }