/// <summary> /// Of all the item lists that are referenced in this batchable object, which ones should we /// batch on, and which ones should we just pass in wholesale to every invocation of the /// target/task? /// /// Rule #1. If the user has referenced any *qualified* item metadata such as %(EmbeddedResource.Culture), /// then that item list "EmbeddedResource" will definitely get batched. /// /// Rule #2. For all the unqualified item metadata such as %(Culture), we make sure that /// every single item in every single item list being passed into the task contains a value /// for that metadata. If not, it's an error. If so, we batch all of those item lists. /// /// All other item lists will not be batched, and instead will be passed in wholesale to all buckets. /// </summary> /// <returns>Hashtable containing the item names that should be batched.</returns> private static Hashtable GetItemListsToBeBatched ( XmlNode parentNode, Dictionary <string, MetadataReference> consumedMetadataReferences, // Key is [string] potentially qualified metadata name // Value is [struct MetadataReference] Hashtable consumedItemReferenceNames, // Key is [string] item name. // Value is always String.Empty (unused). Lookup lookup ) { // The keys in this hashtable are the names of the items that we will batch on. // The values are always String.Empty (not used). Hashtable itemListsToBeBatched = new Hashtable(StringComparer.OrdinalIgnoreCase); // Loop through all the metadata references and find the ones that are qualified // with an item name. foreach (MetadataReference consumedMetadataReference in consumedMetadataReferences.Values) { if (consumedMetadataReference.itemName != null) { // Rule #1. Qualified metadata reference. // For metadata references that are qualified with an item name // (e.g., %(EmbeddedResource.Culture) ), we add that item name to the list of // consumed item names, even if the item name wasn't otherwise referenced via // @(...) syntax, and even if every item in the list doesn't necessary contain // a value for this metadata. This is the special power that you get by qualifying // the metadata reference with an item name. itemListsToBeBatched[consumedMetadataReference.itemName] = String.Empty; // Also add this qualified item to the consumed item references list, because // %(EmbeddedResource.Culture) effectively means that @(EmbeddedResource) is // being consumed, even though we may not see literally "@(EmbeddedResource)" // in the tag anywhere. Adding it to this list allows us (down below in this // method) to check that every item in this list has a value for each // unqualified metadata reference. consumedItemReferenceNames = Utilities.CreateTableIfNecessary(consumedItemReferenceNames); consumedItemReferenceNames[consumedMetadataReference.itemName] = String.Empty; } } // Loop through all the metadata references and find the ones that are unqualified. foreach (MetadataReference consumedMetadataReference in consumedMetadataReferences.Values) { if (consumedMetadataReference.itemName == null) { // Rule #2. Unqualified metadata reference. // For metadata references that are unqualified, every single consumed item // must contain a value for that metadata. If any item doesn't, it's an error // to use unqualified metadata. if (consumedItemReferenceNames != null) { foreach (string consumedItemName in consumedItemReferenceNames.Keys) { // Loop through all the items in the item list. BuildItemGroup items = lookup.GetItems(consumedItemName); if (items != null) { // Loop through all the items in the BuildItemGroup. foreach (BuildItem item in items) { ProjectErrorUtilities.VerifyThrowInvalidProject( item.HasMetadata(consumedMetadataReference.metadataName), parentNode, "ItemDoesNotContainValueForUnqualifiedMetadata", item.Include, consumedItemName, consumedMetadataReference.metadataName); } } // This item list passes the test of having every single item containing // a value for this metadata. Therefore, add this item list to the batching list. // Also, to save doing lookup.GetItems again, put the items in the table as the value. itemListsToBeBatched[consumedItemName] = items; } } } } return(itemListsToBeBatched); }
/// <summary> /// Partitions the items consumed by the batchable object into buckets, where each bucket contains a set of items that /// have the same value set on all item metadata consumed by the object. /// </summary> /// <remarks> /// PERF NOTE: Given n items and m batching metadata that produce l buckets, it is usually the case that n > l > m, /// because a batchable object typically uses one or two item metadata to control batching, and only has a handful of /// buckets. The number of buckets is typically only large if a batchable object is using single-item batching /// (where l == n). Any algorithm devised for bucketing therefore, should try to minimize n and l in its complexity /// equation. The algorithm below has a complexity of O(n*lg(l)*m/2) in its comparisons, and is effectively O(n) when /// l is small, and O(n*lg(n)) in the worst case as l -> n. However, note that the comparison complexity is not the /// same as the operational complexity for this algorithm. The operational complexity of this algorithm is actually /// O(n*m + n*lg(l)*m/2 + n*l/2 + n + l), which is effectively O(n^2) in the worst case. The additional complexity comes /// from the array and metadata operations that are performed. However, those operations are extremely cheap compared /// to the comparison operations, which dominate the time spent in this method. /// </remarks> /// <returns>ArrayList containing ItemBucket objects (can be empty), each one representing an execution batch.</returns> private static ArrayList BucketConsumedItems ( XmlNode parentNode, Lookup lookup, Hashtable itemListsToBeBatched, Dictionary <string, MetadataReference> consumedMetadataReferences ) { ErrorUtilities.VerifyThrow(itemListsToBeBatched.Count > 0, "Need item types consumed by the batchable object."); ErrorUtilities.VerifyThrow(consumedMetadataReferences.Count > 0, "Need item metadata consumed by the batchable object."); ArrayList buckets = new ArrayList(); // Get and iterate through the list of item names that we're supposed to batch on. foreach (DictionaryEntry entry in itemListsToBeBatched) { string itemName = (string)entry.Key; // Use the previously-fetched items, if possible BuildItemGroup items; if (entry.Value is BuildItemGroup) { items = (BuildItemGroup)entry.Value; } else { items = lookup.GetItems(itemName); } if (items != null) { foreach (BuildItem item in items) { // Get this item's values for all the metadata consumed by the batchable object. Dictionary <string, string> itemMetadataValues = GetItemMetadataValues(parentNode, item, consumedMetadataReferences); // put the metadata into a dummy bucket we can use for searching ItemBucket dummyBucket = ItemBucket.GetDummyBucketForComparisons(itemMetadataValues); // look through all previously created buckets to find a bucket whose items have the same values as // this item for all metadata consumed by the batchable object int matchingBucketIndex = buckets.BinarySearch(dummyBucket); ItemBucket matchingBucket = (matchingBucketIndex >= 0) ? (ItemBucket)buckets[matchingBucketIndex] : null; // If we didn't find a bucket that matches this item, create a new one, adding // this item to the bucket. if (null == matchingBucket) { matchingBucket = new ItemBucket(itemListsToBeBatched.Keys, itemMetadataValues, lookup, buckets.Count); // make sure to put the new bucket into the appropriate location // in the sorted list as indicated by the binary search // NOTE: observe the ~ operator (bitwise complement) in front of // the index -- see MSDN for more information on the return value // from the ArrayList.BinarySearch() method buckets.Insert(~matchingBucketIndex, matchingBucket); } // We already have a bucket for this type of item, so add this item to // the bucket. matchingBucket.AddItem(item); } } } // Put the buckets back in the order in which they were discovered, so that the first // item declared in the project file ends up in the first batch passed into the target/task. ArrayList orderedBuckets = ArrayList.Repeat(null, buckets.Count); foreach (ItemBucket bucket in buckets) { orderedBuckets[bucket.BucketSequenceNumber] = bucket; } return(orderedBuckets); }
/// <summary> /// Determines how many times the batchable object needs to be executed (each execution is termed a "batch"), and prepares /// buckets of items to pass to the object in each batch. /// </summary> /// <param name="parentNode"></param> /// <param name="batchableObjectParameters"></param> /// <param name="lookup"></param> /// <param name="implicitBatchableItemType">Any item type that can be considered an implicit input to this batchable object. /// This is useful for items inside targets, where the item name is plainly an item type that's an "input" to the object.</param> /// <returns>ArrayList containing ItemBucket objects, each one representing an execution batch.</returns> internal static ArrayList PrepareBatchingBuckets ( XmlNode parentNode, List <string> batchableObjectParameters, Lookup lookup, string implicitBatchableItemType ) { ErrorUtilities.VerifyThrow(parentNode != null, "Need the XML node that represents the batchable object."); ErrorUtilities.VerifyThrow(batchableObjectParameters != null, "Need the parameters of the batchable object to determine if it can be batched."); ErrorUtilities.VerifyThrow(lookup != null, "Need to specify the lookup."); ItemsAndMetadataPair pair = ExpressionShredder.GetReferencedItemNamesAndMetadata(batchableObjectParameters); // All the @(itemname) item list references in the tag, including transforms, etc. // The keys in the hashtable are the item names, and the values are all String.Empty (not used). Hashtable consumedItemReferences = pair.Items; // All the %(itemname.metadataname) references in the tag (not counting those embedded // inside item transforms), and note that the itemname portion is optional. // The keys in the returned hash table are the qualified metadata names (e.g. "EmbeddedResource.Culture" // or just "Culture"). The values are MetadataReference structs, which simply split out the item // name (possibly null) and the actual metadata name. Dictionary <string, MetadataReference> consumedMetadataReferences = pair.Metadata; ArrayList buckets = null; if (consumedMetadataReferences != null && consumedMetadataReferences.Count > 0) { // Add any item types that we were explicitly told to assume. if (implicitBatchableItemType != null) { consumedItemReferences = Utilities.CreateTableIfNecessary(consumedItemReferences); consumedItemReferences[implicitBatchableItemType] = String.Empty; } // This method goes through all the item list references and figures out which ones // will be participating in batching, and which ones won't. We get back a hashtable // where the key is the item name that will be participating in batching. The values // are all String.Empty (not used). This method may return additional item names // that weren't represented in "consumedItemReferences"... this would happen if there // were qualified metadata references in the consumedMetadataReferences table, such as // %(EmbeddedResource.Culture). Hashtable itemListsToBeBatched = GetItemListsToBeBatched(parentNode, consumedMetadataReferences, consumedItemReferences, lookup); // At this point, if there were any metadata references in the tag, but no item // references to batch on, we've got a problem because we can't figure out which // item lists the user wants us to batch. if (itemListsToBeBatched.Count == 0) { foreach (string unqualifiedMetadataName in consumedMetadataReferences.Keys) { // Of course, since this throws an exception, there's no way we're ever going // to really loop here... it's just that the foreach is the only way I can // figure out how to get data out of the hashtable without knowing any of the // keys! ProjectErrorUtilities.VerifyThrowInvalidProject(false, parentNode, "CannotReferenceItemMetadataWithoutItemName", unqualifiedMetadataName); } } else { // If the batchable object consumes item metadata as well as items to be batched, // we need to partition the items consumed by the object. buckets = BucketConsumedItems(parentNode, lookup, itemListsToBeBatched, consumedMetadataReferences); } } // if the batchable object does not consume any item metadata or items, or if the item lists it consumes are all // empty, then the object does not need to be batched if ((buckets == null) || (buckets.Count == 0)) { // create a default bucket that references the project items and properties -- this way we always have a bucket buckets = new ArrayList(1); buckets.Add(new ItemBucket(null, null, lookup, buckets.Count)); } return(buckets); }