Example #1
0
        private async Task PublishBlobsToAzureStorageNugetFeedAsync(
            List <BlobArtifactModel> blobsToPublish,
            IMaestroApi client,
            Maestro.Client.Models.Build buildInformation,
            FeedConfig feedConfig)
        {
            BlobAssetsBasePath = BlobAssetsBasePath.TrimEnd(
                Path.DirectorySeparatorChar,
                Path.AltDirectorySeparatorChar)
                                 + Path.DirectorySeparatorChar;

            var blobs = blobsToPublish
                        .Select(blob =>
            {
                var fileName = Path.GetFileName(blob.Id);
                return(new MSBuild.TaskItem($"{BlobAssetsBasePath}{fileName}", new Dictionary <string, string>
                {
                    { "RelativeBlobPath", blob.Id }
                }));
            })
                        .ToArray();

            var blobFeedAction = CreateBlobFeedAction(feedConfig);
            var pushOptions    = new PushOptions
            {
                AllowOverwrite = false,
                PassIfExistingItemIdentical = true
            };

            foreach (var blob in blobsToPublish)
            {
                var assetRecord = buildInformation.Assets
                                  .Where(a => a.Name.Equals(blob.Id))
                                  .SingleOrDefault();

                if (assetRecord == null)
                {
                    Log.LogError($"Asset with Id {blob.Id} isn't registered on the BAR Build with ID {BARBuildId}");
                    continue;
                }

                var assetWithLocations = await client.Assets.GetAssetAsync(assetRecord.Id);

                if (assetWithLocations?.Locations.Any(al => al.Location.Equals(feedConfig.TargetFeedURL, StringComparison.OrdinalIgnoreCase)) ?? false)
                {
                    Log.LogMessage($"Asset with Id {blob.Id} already has location {feedConfig.TargetFeedURL}");
                    continue;
                }

                await client.Assets.AddAssetLocationToAssetAsync(assetRecord.Id, AddAssetLocationToAssetAssetLocationType.Container, feedConfig.TargetFeedURL);
            }

            await blobFeedAction.PublishToFlatContainerAsync(blobs, maxClients : 8, pushOptions);
        }
Example #2
0
 public async Task PublishToFlatContainerAsync(IEnumerable <ITaskItem> taskItems, int maxClients,
                                               PushOptions pushOptions)
 {
     if (taskItems.Any())
     {
         using (var clientThrottle = new SemaphoreSlim(maxClients, maxClients))
         {
             await System.Threading.Tasks.Task.WhenAll(taskItems.Select(
                                                           item => { return(UploadAssetAsync(item, pushOptions, clientThrottle)); }
                                                           ));
         }
     }
 }
Example #3
0
        public async Task <bool> PushToFeedAsync(
            IEnumerable <string> items,
            PushOptions options)
        {
            if (IsSanityChecked(items))
            {
                if (CancellationToken.IsCancellationRequested)
                {
                    Log.LogError("Task PushToFeed cancelled");
                    CancellationToken.ThrowIfCancellationRequested();
                }

                await PushItemsToFeedAsync(items, options);
            }

            return(!Log.HasLoggedErrors);
        }
Example #4
0
        private async Task PublishPackagesToAzureStorageNugetFeedAsync(
            List <PackageArtifactModel> packagesToPublish,
            IMaestroApi client,
            Maestro.Client.Models.Build buildInformation,
            FeedConfig feedConfig)
        {
            PackageAssetsBasePath = PackageAssetsBasePath.TrimEnd(
                Path.DirectorySeparatorChar,
                Path.AltDirectorySeparatorChar)
                                    + Path.DirectorySeparatorChar;

            var packages       = packagesToPublish.Select(p => $"{PackageAssetsBasePath}{p.Id}.{p.Version}.nupkg");
            var blobFeedAction = CreateBlobFeedAction(feedConfig);

            var pushOptions = new PushOptions
            {
                AllowOverwrite = false,
                PassIfExistingItemIdentical = true
            };

            foreach (var package in packagesToPublish)
            {
                var assetRecord = buildInformation.Assets
                                  .Where(a => a.Name.Equals(package.Id) && a.Version.Equals(package.Version))
                                  .FirstOrDefault();

                if (assetRecord == null)
                {
                    Log.LogError($"Asset with Id {package.Id}, Version {package.Version} isn't registered on the BAR Build with ID {BARBuildId}");
                    continue;
                }

                var assetWithLocations = await client.Assets.GetAssetAsync(assetRecord.Id);

                if (assetWithLocations?.Locations.Any(al => al.Location.Equals(feedConfig.TargetFeedURL, StringComparison.OrdinalIgnoreCase)) ?? false)
                {
                    Log.LogMessage($"Asset with Id {package.Id}, Version {package.Version} already has location {feedConfig.TargetFeedURL}");
                    continue;
                }

                await client.Assets.AddAssetLocationToAssetAsync(assetRecord.Id, AddAssetLocationToAssetAssetLocationType.NugetFeed, feedConfig.TargetFeedURL);
            }

            await blobFeedAction.PushToFeedAsync(packages, pushOptions);
        }
Example #5
0
        public async Task <bool> PushAsync(IEnumerable <string> items, PushOptions options)
        {
            LocalSettings settings       = GetSettings();
            SleetLogger   log            = new SleetLogger(Log, NuGet.Common.LogLevel.Verbose);
            var           packagesToPush = items.ToList();

            // Create a new cache to be used once a lock is obtained.
            using (var fileCache = CreateFileCache())
            {
                var lockedFileSystem = GetAzureFileSystem(fileCache);

                return(await PushCommand.RunAsync(
                           settings,
                           lockedFileSystem,
                           packagesToPush,
                           force : options.AllowOverwrite,
                           skipExisting : !options.AllowOverwrite,
                           log : log));
            }
        }
        public async Task PublishAssetAsync(string file, string blobPath, PushOptions options, SemaphoreSlim clientThrottle = null)
        {
            if (!file.EndsWith(GeneralUtils.PackageSuffix, StringComparison.OrdinalIgnoreCase))
            {
                _log.LogWarning(
                    $"AzDO feed publishing not available for blobs. Blob '{file}' was not published.");
                return;
            }

            string id;
            string version;

            using (var packageReader = new PackageArchiveReader(file))
            {
                PackageIdentity packageIdentity = packageReader.GetIdentity();
                id      = packageIdentity.Id;
                version = packageIdentity.Version.ToString();
            }

            try
            {
                var config = new TargetFeedConfig(default, _targetUrl, default, default);
        public async Task PublishAssetAsync(string file, string blobPath, PushOptions options, SemaphoreSlim clientThrottle = null)
        {
            using (await SemaphoreLock.LockAsync(clientThrottle))
            {
                blobPath = blobPath.Replace("\\", "/");
                var blobClient = CreateBlobClient(blobPath);
                if (!options.AllowOverwrite && await blobClient.ExistsAsync())
                {
                    if (options.PassIfExistingItemIdentical)
                    {
                        if (!await blobClient.IsFileIdenticalToBlobAsync(file))
                        {
                            _log.LogError($"Asset '{file}' already exists with different contents at '{blobPath}'");
                        }

                        return;
                    }

                    _log.LogError($"Asset '{file}' already exists at '{blobPath}'");
                    return;
                }

                _log.LogMessage($"Uploading '{file}' to '{blobPath}'");

                try
                {
                    BlobUploadOptions blobUploadOptions = new()
                    {
                        HttpHeaders = AzureStorageUtils.GetBlobHeadersByExtension(file)
                    };
                    await blobClient.UploadAsync(file, blobUploadOptions);
                }
                catch (Exception e)
                {
                    _log.LogError($"Unexpected exception publishing file {file} to {blobPath}: {e.Message}");
                }
            }
        }
Example #8
0
        public async Task <bool> PushItemsToFeedAsync(
            IEnumerable <string> items,
            PushOptions options)
        {
            Log.LogMessage(MessageImportance.Low, $"START pushing items to feed");

            if (!items.Any())
            {
                Log.LogMessage("No items to push found in the items list.");
                return(true);
            }

            try
            {
                return(await PushAsync(items, options));
            }
            catch (Exception e)
            {
                Log.LogErrorFromException(e);
            }

            return(!Log.HasLoggedErrors);
        }
Example #9
0
 public async Task PublishToFlatContainerAsync(IEnumerable <ITaskItem> taskItems,
                                               int maxClients,
                                               int uploadTimeoutInMinutes,
                                               PushOptions pushOptions)
 {
     if (taskItems.Any())
     {
         using (var clientThrottle = new SemaphoreSlim(maxClients, maxClients))
         {
             Log.LogMessage(MessageImportance.High, $"Uploading {taskItems.Count()} items:");
             await System.Threading.Tasks.Task.WhenAll(taskItems.Select(
                                                           item =>
             {
                 Log.LogMessage(MessageImportance.High, $"Async uploading {item.ItemSpec}");
                 return(UploadAssetAsync(
                            item,
                            clientThrottle,
                            uploadTimeoutInMinutes,
                            pushOptions));
             }
                                                           ));
         }
     }
 }
Example #10
0
        private async Task <bool> PushAsync(
            IEnumerable <string> items,
            PushOptions options)
        {
            LocalSettings   settings   = GetSettings();
            AzureFileSystem fileSystem = GetAzureFileSystem();
            SleetLogger     log        = new SleetLogger(Log);

            var packagesToPush = items.ToList();

            if (!options.AllowOverwrite && options.PassIfExistingItemIdentical)
            {
                var context = new SleetContext
                {
                    LocalSettings = settings,
                    Log           = log,
                    Source        = fileSystem,
                    Token         = CancellationToken
                };
                context.SourceSettings = await FeedSettingsUtility.GetSettingsOrDefault(
                    context.Source,
                    context.Log,
                    context.Token);

                var flatContainer = new FlatContainer(context);

                var packageIndex = new PackageIndex(context);

                // Check packages sequentially: Task.WhenAll caused IO exceptions in Sleet.
                for (int i = packagesToPush.Count - 1; i >= 0; i--)
                {
                    string item = packagesToPush[i];

                    bool?identical = await IsPackageIdenticalOnFeedAsync(
                        item,
                        packageIndex,
                        context.Source,
                        flatContainer,
                        log);

                    if (identical == null)
                    {
                        continue;
                    }

                    packagesToPush.RemoveAt(i);

                    if (identical == true)
                    {
                        Log.LogMessage(
                            MessageImportance.Normal,
                            "Package exists on the feed, and is verified to be identical. " +
                            $"Skipping upload: '{item}'");
                    }
                    else
                    {
                        Log.LogError(
                            "Package exists on the feed, but contents are different. " +
                            $"Upload failed: '{item}'");
                    }
                }

                if (!packagesToPush.Any())
                {
                    Log.LogMessage("After skipping idempotent uploads, no items need pushing.");
                    return(true);
                }
            }

            return(await PushCommand.RunAsync(
                       settings,
                       fileSystem,
                       packagesToPush,
                       options.AllowOverwrite,
                       skipExisting : false,
                       log : log));
        }
Example #11
0
        public async Task UploadAssetAsync(
            ITaskItem item,
            SemaphoreSlim clientThrottle,
            int uploadTimeout,
            PushOptions options)
        {
            string relativeBlobPath = item.GetMetadata("RelativeBlobPath");

            if (string.IsNullOrEmpty(relativeBlobPath))
            {
                string fileName     = Path.GetFileName(item.ItemSpec);
                string recursiveDir = item.GetMetadata("RecursiveDir");
                relativeBlobPath = $"{recursiveDir}{fileName}";
            }

            string contentType = item.GetMetadata("ContentType");

            relativeBlobPath = $"{feed.RelativePath}{relativeBlobPath}".Replace("\\", "/");

            if (relativeBlobPath.Contains("//"))
            {
                Log.LogError(
                    $"Item '{item.ItemSpec}' RelativeBlobPath contains virtual directory " +
                    $"without name (double forward slash): '{relativeBlobPath}'");
                return;
            }

            Log.LogMessage($"Uploading {relativeBlobPath}");

            await clientThrottle.WaitAsync();

            try
            {
                UploadClient uploadClient = new UploadClient(Log);

                if (!options.AllowOverwrite && await feed.CheckIfBlobExistsAsync(relativeBlobPath))
                {
                    if (options.PassIfExistingItemIdentical)
                    {
                        if (!await uploadClient.FileEqualsExistingBlobAsync(
                                feed.AccountName,
                                feed.AccountKey,
                                feed.ContainerName,
                                item.ItemSpec,
                                relativeBlobPath,
                                uploadTimeout))
                        {
                            Log.LogError(
                                $"Item '{item}' already exists with different contents " +
                                $"at '{relativeBlobPath}'");
                        }
                    }
                    else
                    {
                        Log.LogError($"Item '{item}' already exists at '{relativeBlobPath}'");
                    }
                }
                else
                {
                    Log.LogMessage($"Uploading {item} to {relativeBlobPath}.");
                    await uploadClient.UploadBlockBlobAsync(
                        CancellationToken,
                        feed.AccountName,
                        feed.AccountKey,
                        feed.ContainerName,
                        item.ItemSpec,
                        relativeBlobPath,
                        contentType,
                        uploadTimeout);
                }
            }
            catch (Exception exc)
            {
                Log.LogError($"Unable to upload to {relativeBlobPath} due to {exc}.");
                throw;
            }
            finally
            {
                clientThrottle.Release();
            }
        }
        public async Task <bool> ExecuteAsync()
        {
            try
            {
                Log.LogMessage(MessageImportance.High, "Performing push feeds.");

                if (string.IsNullOrWhiteSpace(ExpectedFeedUrl) || string.IsNullOrWhiteSpace(AccountKey))
                {
                    Log.LogError($"{nameof(ExpectedFeedUrl)} / {nameof(AccountKey)} is not set properly.");
                }
                else if (string.IsNullOrWhiteSpace(AssetManifestPath) || !File.Exists(AssetManifestPath))
                {
                    Log.LogError($"Problem reading asset manifest path from {AssetManifestPath}");
                }
                else if (MaxClients <= 0)
                {
                    Log.LogError($"{nameof(MaxClients)} should be greater than zero.");
                }
                else if (UploadTimeoutInMinutes <= 0)
                {
                    Log.LogError($"{nameof(UploadTimeoutInMinutes)} should be greater than zero.");
                }

                var buildModel = BuildManifestUtil.ManifestFileToModel(AssetManifestPath, Log);

                // Parsing the manifest may fail for several reasons
                if (Log.HasLoggedErrors)
                {
                    return(false);
                }

                // Fetch Maestro record of the build. We're going to use it to get the BAR ID
                // of the assets being published so we can add a new location for them.
                IMaestroApi client = ApiFactory.GetAuthenticated(MaestroApiEndpoint, BuildAssetRegistryToken);
                Maestro.Client.Models.Build buildInformation = await client.Builds.GetBuildAsync(BARBuildId);

                var blobFeedAction = new BlobFeedAction(ExpectedFeedUrl, AccountKey, Log);
                var pushOptions    = new PushOptions
                {
                    AllowOverwrite = Overwrite,
                    PassIfExistingItemIdentical = PassIfExistingItemIdentical
                };

                if (buildModel.Artifacts.Packages.Any())
                {
                    if (!Directory.Exists(PackageAssetsBasePath))
                    {
                        Log.LogError($"Invalid {nameof(PackageAssetsBasePath)} was supplied: {PackageAssetsBasePath}");
                        return(false);
                    }

                    PackageAssetsBasePath = PackageAssetsBasePath.TrimEnd(Path.DirectorySeparatorChar,
                                                                          Path.AltDirectorySeparatorChar) + Path.DirectorySeparatorChar;

                    var packages = buildModel.Artifacts.Packages.Select(p => $"{PackageAssetsBasePath}{p.Id}.{p.Version}.nupkg");

                    await blobFeedAction.PushToFeedAsync(packages, pushOptions);

                    foreach (var package in buildModel.Artifacts.Packages)
                    {
                        var assetRecord = buildInformation.Assets
                                          .Where(a => a.Name.Equals(package.Id) && a.Version.Equals(package.Version))
                                          .Single();

                        if (assetRecord == null)
                        {
                            Log.LogError($"Asset with Id {package.Id}, Version {package.Version} isn't registered on the BAR Build with ID {BARBuildId}");
                            continue;
                        }

                        await client.Assets.AddAssetLocationToAssetAsync(assetRecord.Id.Value, ExpectedFeedUrl, "NugetFeed");
                    }
                }

                if (buildModel.Artifacts.Blobs.Any())
                {
                    if (!Directory.Exists(BlobAssetsBasePath))
                    {
                        Log.LogError($"Invalid {nameof(BlobAssetsBasePath)} was supplied: {BlobAssetsBasePath}");
                        return(false);
                    }

                    BlobAssetsBasePath = BlobAssetsBasePath.TrimEnd(Path.DirectorySeparatorChar,
                                                                    Path.AltDirectorySeparatorChar) + Path.DirectorySeparatorChar;

                    var blobs = buildModel.Artifacts.Blobs
                                .Select(blob =>
                    {
                        var fileName = Path.GetFileName(blob.Id);
                        return(new MSBuild.TaskItem($"{BlobAssetsBasePath}{fileName}", new Dictionary <string, string>
                        {
                            { "RelativeBlobPath", $"{BuildManifestUtil.AssetsVirtualDir}{blob.Id}" }
                        }));
                    })
                                .ToArray();

                    await blobFeedAction.PublishToFlatContainerAsync(blobs, MaxClients, UploadTimeoutInMinutes, pushOptions);

                    foreach (var package in buildModel.Artifacts.Blobs)
                    {
                        var assetRecord = buildInformation.Assets
                                          .Where(a => a.Name.Equals(package.Id))
                                          .SingleOrDefault();

                        if (assetRecord == null)
                        {
                            Log.LogError($"Asset with Id {package.Id} isn't registered on the BAR Build with ID {BARBuildId}");
                            continue;
                        }

                        await client.Assets.AddAssetLocationToAssetAsync(assetRecord.Id.Value, ExpectedFeedUrl, "NugetFeed");
                    }
                }
            }
            catch (Exception e)
            {
                Log.LogErrorFromException(e, true);
            }

            return(!Log.HasLoggedErrors);
        }
Example #13
0
        private async Task <bool> ExecuteAsync()
        {
            try
            {
                Log.LogMessage(MessageImportance.High, "Performing feed push...");

                if (ItemsToPush == null)
                {
                    Log.LogError($"No items to push. Please check ItemGroup ItemsToPush.");
                }
                else if (string.IsNullOrWhiteSpace(ExpectedFeedUrl) || string.IsNullOrWhiteSpace(AccountKey))
                {
                    Log.LogError($"{nameof(ExpectedFeedUrl)} / {nameof(AccountKey)} is not set properly.");
                }
                else if (string.IsNullOrWhiteSpace(AssetManifestPath))
                {
                    Log.LogError($"{nameof(AssetManifestPath)} is not set properly.");
                }
                else if (MaxClients <= 0)
                {
                    Log.LogError($"{nameof(MaxClients)} should be greater than zero.");
                }
                else if (UploadTimeoutInMinutes <= 0)
                {
                    Log.LogError($"{nameof(UploadTimeoutInMinutes)} should be greater than zero.");
                }

                if (Log.HasLoggedErrors)
                {
                    return(false);
                }

                BlobFeedAction blobFeedAction = new BlobFeedAction(ExpectedFeedUrl, AccountKey, Log);
                var            pushOptions    = new PushOptions
                {
                    AllowOverwrite = Overwrite,
                    PassIfExistingItemIdentical = PassIfExistingItemIdentical
                };

                IEnumerable <BlobArtifactModel>    blobArtifacts    = Enumerable.Empty <BlobArtifactModel>();
                IEnumerable <PackageArtifactModel> packageArtifacts = Enumerable.Empty <PackageArtifactModel>();

                if (!SkipCreateContainer)
                {
                    await blobFeedAction.CreateContainerAsync(BuildEngine, PublishFlatContainer);
                }

                if (PublishFlatContainer)
                {
                    await blobFeedAction.PublishToFlatContainerAsync(ItemsToPush,
                                                                     MaxClients,
                                                                     pushOptions);

                    blobArtifacts = ConcatBlobArtifacts(blobArtifacts, ItemsToPush);
                }
                else
                {
                    ITaskItem[] symbolItems = ItemsToPush
                                              .Where(i => i.ItemSpec.Contains("symbols.nupkg"))
                                              .Select(i =>
                    {
                        string fileName = Path.GetFileName(i.ItemSpec);
                        i.SetMetadata("RelativeBlobPath", $"{BuildManifestUtil.AssetsVirtualDir}symbols/{fileName}");
                        return(i);
                    })
                                              .ToArray();

                    ITaskItem[] packageItems = ItemsToPush
                                               .Where(i => !symbolItems.Contains(i))
                                               .ToArray();

                    var packagePaths = packageItems.Select(i => i.ItemSpec);

                    if (!blobFeedAction.PushToFeedAsync(packagePaths, pushOptions).Result)
                    {
                        return(!Log.HasLoggedErrors);
                    }

                    await blobFeedAction.PublishToFlatContainerAsync(symbolItems, MaxClients, pushOptions);

                    if (Log.HasLoggedErrors)
                    {
                        return(!Log.HasLoggedErrors);
                    }

                    packageArtifacts = ConcatPackageArtifacts(packageArtifacts, packageItems);
                    blobArtifacts    = ConcatBlobArtifacts(blobArtifacts, symbolItems);
                }

                if (!BuildManifestUtil.ManifestBuildDataHasLocationInformation(ManifestBuildData))
                {
                    string[] locationAttribute = new string[] { $"Location={ExpectedFeedUrl}" };
                    ManifestBuildData = ManifestBuildData == null ? locationAttribute : ManifestBuildData.Concat(locationAttribute).ToArray();
                }

                BuildManifestUtil.CreateBuildManifest(Log,
                                                      blobArtifacts,
                                                      packageArtifacts,
                                                      AssetManifestPath,
                                                      ManifestRepoUri,
                                                      ManifestBuildId,
                                                      ManifestBranch,
                                                      ManifestCommit,
                                                      ManifestBuildData,
                                                      IsStableBuild,
                                                      PublishingInfraVersion.Legacy,
                                                      IsReleaseOnlyPackageVersion);
            }
            catch (Exception e)
            {
                Log.LogErrorFromException(e, true);
            }

            return(!Log.HasLoggedErrors);
        }
Example #14
0
        public async Task UploadAssetAsync(
            ITaskItem item,
            PushOptions options,
            SemaphoreSlim clientThrottle = null)
        {
            string relativeBlobPath = item.GetMetadata("RelativeBlobPath");

            if (string.IsNullOrEmpty(relativeBlobPath))
            {
                string fileName     = Path.GetFileName(item.ItemSpec);
                string recursiveDir = item.GetMetadata("RecursiveDir");
                relativeBlobPath = $"{recursiveDir}{fileName}";
            }

            if (!string.IsNullOrEmpty(relativeBlobPath))
            {
                relativeBlobPath = $"{RelativePath}{relativeBlobPath}".Replace("\\", "/");

                if (relativeBlobPath.StartsWith("//"))
                {
                    Log.LogError(
                        $"Item '{item.ItemSpec}' RelativeBlobPath contains virtual directory " +
                        $"without name (double forward slash): '{relativeBlobPath}'");
                    return;
                }

                Log.LogMessage($"Uploading {relativeBlobPath}");

                if (clientThrottle != null)
                {
                    await clientThrottle.WaitAsync();
                }

                try
                {
                    AzureStorageUtils blobUtils = new AzureStorageUtils(AccountName, AccountKey, ContainerName);

                    if (!options.AllowOverwrite && await blobUtils.CheckIfBlobExistsAsync(relativeBlobPath))
                    {
                        if (options.PassIfExistingItemIdentical)
                        {
                            if (!await blobUtils.IsFileIdenticalToBlobAsync(item.ItemSpec, relativeBlobPath))
                            {
                                Log.LogError(
                                    $"Item '{item}' already exists with different contents " +
                                    $"at '{relativeBlobPath}'");
                            }
                        }
                        else
                        {
                            Log.LogError($"Item '{item}' already exists at '{relativeBlobPath}'");
                        }
                    }
                    else
                    {
                        using (FileStream stream =
                                   new FileStream(item.ItemSpec, FileMode.Open, FileAccess.Read, FileShare.Read))
                        {
                            Log.LogMessage($"Uploading {item} to {relativeBlobPath}.");
                            await blobUtils.UploadBlockBlobAsync(item.ItemSpec, relativeBlobPath, stream);
                        }
                    }
                }
                catch (Exception exc)
                {
                    Log.LogError(
                        $"Unable to upload to {relativeBlobPath} in Azure Storage account {AccountName}/{ContainerName} due to {exc}.");
                    throw;
                }
                finally
                {
                    if (clientThrottle != null)
                    {
                        clientThrottle.Release();
                    }
                }
            }
            else
            {
                Log.LogError($"Relative blob path is empty.");
            }
        }
Example #15
0
        private async Task <bool> PushAsync(
            IEnumerable <string> items,
            PushOptions options)
        {
            LocalSettings settings       = GetSettings();
            SleetLogger   log            = new SleetLogger(Log, NuGet.Common.LogLevel.Verbose);
            var           packagesToPush = items.ToList();

            // Create a separate LocalCache to use for read only operations on the feed.
            // Files added to the cache before the lock could be modified by the process
            // currently holding the lock. Sleet assumes that files in the cache
            // are valid and identical to the ones on the feed since operations are
            // normally performed inside the lock.
            using (var preLockCache = CreateFileCache())
            {
                var preLockFileSystem = GetAzureFileSystem(preLockCache);

                if (!options.AllowOverwrite && options.PassIfExistingItemIdentical)
                {
                    var context = new SleetContext
                    {
                        LocalSettings = settings,
                        Log           = log,
                        Source        = preLockFileSystem,
                        Token         = CancellationToken
                    };
                    context.SourceSettings = await FeedSettingsUtility.GetSettingsOrDefault(
                        context.Source,
                        context.Log,
                        context.Token);

                    var flatContainer = new FlatContainer(context);

                    var packageIndex = new PackageIndex(context);

                    // Check packages sequentially: Task.WhenAll caused IO exceptions in Sleet.
                    for (int i = packagesToPush.Count - 1; i >= 0; i--)
                    {
                        string item = packagesToPush[i];

                        bool?identical = await IsPackageIdenticalOnFeedAsync(
                            item,
                            packageIndex,
                            context.Source,
                            flatContainer,
                            log);

                        if (identical == null)
                        {
                            continue;
                        }

                        packagesToPush.RemoveAt(i);

                        if (identical == true)
                        {
                            Log.LogMessage(
                                MessageImportance.Normal,
                                "Package exists on the feed, and is verified to be identical. " +
                                $"Skipping upload: '{item}'");
                        }
                        else
                        {
                            Log.LogError(
                                "Package exists on the feed, but contents are different. " +
                                $"Upload failed: '{item}'");
                        }
                    }

                    if (!packagesToPush.Any())
                    {
                        Log.LogMessage("After skipping idempotent uploads, no items need pushing.");
                        return(true);
                    }
                }
            }

            // Create a new cache to be used once a lock is obtained.
            using (var fileCache = CreateFileCache())
            {
                var lockedFileSystem = GetAzureFileSystem(fileCache);

                return(await PushCommand.RunAsync(
                           settings,
                           lockedFileSystem,
                           packagesToPush,
                           options.AllowOverwrite,
                           skipExisting : false,
                           log : log));
            }
        }
Example #16
0
        public async Task <bool> ExecuteAsync()
        {
            try
            {
                Log.LogMessage(MessageImportance.High, "Performing push feeds.");

                if (string.IsNullOrWhiteSpace(ExpectedFeedUrl) || string.IsNullOrWhiteSpace(AccountKey))
                {
                    Log.LogError($"{nameof(ExpectedFeedUrl)} / {nameof(AccountKey)} is not set properly.");
                }
                else if (string.IsNullOrWhiteSpace(AssetManifestPath) || !File.Exists(AssetManifestPath))
                {
                    Log.LogError($"Problem reading asset manifest path from {AssetManifestPath}");
                }
                else if (MaxClients <= 0)
                {
                    Log.LogError($"{nameof(MaxClients)} should be greater than zero.");
                }
                else if (UploadTimeoutInMinutes <= 0)
                {
                    Log.LogError($"{nameof(UploadTimeoutInMinutes)} should be greater than zero.");
                }

                var buildModel = BuildManifestUtil.ManifestFileToModel(AssetManifestPath, Log);

                // Parsing the manifest may fail for several reasons
                if (Log.HasLoggedErrors)
                {
                    return(false);
                }

                var blobFeedAction = new BlobFeedAction(ExpectedFeedUrl, AccountKey, Log);
                var pushOptions    = new PushOptions
                {
                    AllowOverwrite = Overwrite,
                    PassIfExistingItemIdentical = PassIfExistingItemIdentical
                };

                if (buildModel.Artifacts.Packages.Any())
                {
                    if (!Directory.Exists(PackageAssetsBasePath))
                    {
                        Log.LogError($"Invalid {nameof(PackageAssetsBasePath)} was supplied: {PackageAssetsBasePath}");
                        return(false);
                    }

                    PackageAssetsBasePath = PackageAssetsBasePath.TrimEnd(Path.DirectorySeparatorChar,
                                                                          Path.AltDirectorySeparatorChar) + Path.DirectorySeparatorChar;

                    var packages = buildModel.Artifacts.Packages.Select(p => $"{PackageAssetsBasePath}{p.Id}.{p.Version}.nupkg");

                    await blobFeedAction.PushToFeedAsync(packages, pushOptions);
                }

                if (buildModel.Artifacts.Blobs.Any())
                {
                    if (!Directory.Exists(BlobAssetsBasePath))
                    {
                        Log.LogError($"Invalid {nameof(BlobAssetsBasePath)} was supplied: {BlobAssetsBasePath}");
                        return(false);
                    }

                    BlobAssetsBasePath = BlobAssetsBasePath.TrimEnd(Path.DirectorySeparatorChar,
                                                                    Path.AltDirectorySeparatorChar) + Path.DirectorySeparatorChar;

                    var blobs = buildModel.Artifacts.Blobs
                                .Select(blob =>
                    {
                        var fileName = Path.GetFileName(blob.Id);
                        return(new MSBuild.TaskItem($"{BlobAssetsBasePath}{fileName}", new Dictionary <string, string>
                        {
                            { "RelativeBlobPath", $"{BuildManifestUtil.AssetsVirtualDir}{blob.Id}" }
                        }));
                    })
                                .ToArray();

                    await blobFeedAction.PublishToFlatContainerAsync(blobs, MaxClients, UploadTimeoutInMinutes, pushOptions);
                }
            }
            catch (Exception e)
            {
                Log.LogErrorFromException(e, true);
            }

            return(!Log.HasLoggedErrors);
        }