Ejemplo n.º 1
0
        public static async Task Test0Async()
        {
            //string nuspecs = @"c:\data\nuget\nuspecs";
            string nuspecs = @"c:\data\nuget\nuspecs";

            //Storage storage = new FileStorage("http://*****:*****@"c:\data\site\full");
            //Storage storage = new FileStorage("http://*****:*****@"c:\data\site\dotnetrdf");
            Storage storage = new FileStorage("http://*****:*****@"c:\data\site\ordered");

            AppendOnlyCatalogWriter writer = new AppendOnlyCatalogWriter(storage, 15);

            const int BatchSize = 10;
            int i = 0;

            int commitCount = 0;

            DirectoryInfo directoryInfo = new DirectoryInfo(nuspecs);
            //foreach (FileInfo fileInfo in directoryInfo.EnumerateFiles("*.xml"))
            //foreach (FileInfo fileInfo in directoryInfo.EnumerateFiles("dotnetrdf.*.xml"))
            foreach (FileInfo fileInfo in directoryInfo.EnumerateFiles("entityframework.*.xml"))
            {
                writer.Add(new NuspecPackageCatalogItem(fileInfo.FullName));

                if (++i % BatchSize == 0)
                {
                    await writer.Commit(DateTime.UtcNow, null, CancellationToken.None);

                    Console.WriteLine("commit number {0}", commitCount++);
                }
            }

            await writer.Commit(DateTime.UtcNow, null, CancellationToken.None);

            Console.WriteLine("commit number {0}", commitCount++);
        }
        public static async Task<Uri> AddToCatalog(CatalogItem catalogItem, string connectionString, string container, string catalogBaseAddress, CancellationToken cancellationToken)
        {
            StorageWriteLock writeLock = new StorageWriteLock(connectionString, container);

            await writeLock.AquireAsync();

            Uri rootUri = null;

            Exception exception = null;
            try
            {
                Storage storage = CreateStorage(connectionString, container, catalogBaseAddress);

                AppendOnlyCatalogWriter writer = new AppendOnlyCatalogWriter(storage);
                writer.Add(catalogItem);
                await writer.Commit(null, cancellationToken);

                rootUri = writer.RootUri;
            }
            catch (Exception e)
            {
                exception = e;
            }

            await writeLock.ReleaseAsync();

            if (exception != null)
            {
                throw exception;
            }

            return rootUri;
        }
Ejemplo n.º 3
0
        public async Task Load(string path, CancellationToken cancellationToken)
        {
            var directoryInfo = new DirectoryInfo(path);

            foreach (var fileInfo in directoryInfo.EnumerateFiles("*.nuspec"))
            {
                AddNuspec(fileInfo);
            }

            //  Catalog

            var factory = new MemoryStorageFactory(new Uri(_baseAddress, Catalog), _store);
            var storage = factory.Create();

            var catalog = new AppendOnlyCatalogWriter(storage);

            foreach (var registration in Data.Values)
            {
                foreach (var package in registration.Values)
                {
                    var metadata = new NupkgMetadata
                    {
                        Nuspec = package
                    };
                    catalog.Add(new PackageCatalogItem(metadata));
                }
            }

            await catalog.Commit(null, cancellationToken);

            Uri catalogIndex = new Uri(storage.BaseAddress, "index.json");

            Func<StorageHttpMessageHandler> handlerFunc = () => { return new StorageHttpMessageHandler(storage); };

            await CreateRegistrationBlobs(catalogIndex, handlerFunc, cancellationToken);
            await CreateFlatContainer(catalogIndex, handlerFunc, cancellationToken);
            await CreateLuceneIndex(catalogIndex, handlerFunc, cancellationToken);
            await CreateIndex(cancellationToken);
        }
Ejemplo n.º 4
0
        static async Task<DateTime> DownloadMetadata2Catalog(HttpClient client, SortedList<DateTime, IList<Tuple<Uri, FeedDetails>>> packages, Storage storage, DateTime lastCreated, DateTime lastEdited, bool? createdPackages, CancellationToken cancellationToken)
        {
            AppendOnlyCatalogWriter writer = new AppendOnlyCatalogWriter(storage, 550);

            DateTime lastDate = createdPackages.HasValue ? (createdPackages.Value ? lastCreated : lastEdited) : DateTime.MinValue;

            if (packages == null || packages.Count == 0)
            {
                return lastDate;
            }

            foreach (KeyValuePair<DateTime, IList<Tuple<Uri, FeedDetails>>> entry in packages)
            {
                foreach (Tuple<Uri, FeedDetails> packageItem in entry.Value)
                {
                    Uri uri = packageItem.Item1;
                    FeedDetails details = packageItem.Item2;

                    HttpResponseMessage response = await client.GetAsync(uri,cancellationToken);

                    if (response.IsSuccessStatusCode)
                    {
                        using (Stream stream = await response.Content.ReadAsStreamAsync())
                        {
                            CatalogItem item = Utils.CreateCatalogItem(stream, entry.Key, null, uri.ToString(), details.CreatedDate, details.LastEditedDate, details.PublishedDate, details.LicenseNames, details.LicenseReportUrl);

                            if (item != null)
                            {
                                writer.Add(item);

                                Trace.TraceInformation("Add: {0}", uri);
                            }
                            else
                            {
                                Trace.TraceWarning("Unable to extract metadata from: {0}", uri);
                            }
                        }
                    }
                    else
                    {
                        if (response.StatusCode == System.Net.HttpStatusCode.NotFound)
                        {
                            //  the feed is out of sync with the actual package storage - if we don't have the package there is nothing to be done we might as well move onto the next package
                            Trace.TraceWarning(string.Format("Unable to download: {0} http status: {1}", uri, response.StatusCode));
                        }
                        else
                        {
                            //  this should trigger a restart - of this program - and not more the cursor forward
                            Trace.TraceError(string.Format("Unable to download: {0} http status: {1}", uri, response.StatusCode));
                            throw new Exception(string.Format("Unable to download: {0} http status: {1}", uri, response.StatusCode));
                        }
                    }
                }

                lastDate = entry.Key;
            }

            if (createdPackages.HasValue)
            {
                lastCreated = createdPackages.Value ? lastDate : lastCreated;
                lastEdited = !createdPackages.Value ? lastDate : lastEdited;
            }

            IGraph commitMetadata = PackageCatalog.CreateCommitMetadata(writer.RootUri, lastCreated, lastEdited);
            
            await writer.Commit(commitMetadata, cancellationToken);

            Trace.TraceInformation("COMMIT");

            return lastDate;
        }
Ejemplo n.º 5
0
        public static async Task Test1Async()
        {
            string nuspecs = @"c:\data\nuget\nuspecs";

            Storage storage = new FileStorage("http://*****:*****@"c:\data\site\full");

            AppendOnlyCatalogWriter writer = new AppendOnlyCatalogWriter(storage, 20);

            int total = 0;

            //int[] commitSize = { 50, 40, 25, 50, 10, 30, 40, 5, 400, 30, 10, 20, 40, 50, 90, 70, 50, 50, 50, 50, 60, 70 };
            int[] commitSize = { 
                20, 20, 20, 20, 20, 
                20, 20, 20, 20, 20, 
                //200, 200, 200, 200, 200, 
                //200, 200, 200, 200, 200, 
                //200, 200, 200, 200, 200, 
                //200, 200, 200, 200, 200,
                //200, 200, 200, 200, 200
            };
            int i = 0;

            int commitCount = 0;

            DirectoryInfo directoryInfo = new DirectoryInfo(nuspecs);
            foreach (FileInfo fileInfo in directoryInfo.EnumerateFiles("dotnetrdf.*.xml"))
            {
                if (commitCount == commitSize.Length)
                {
                    break;
                }

                writer.Add(new NuspecPackageCatalogItem(fileInfo.FullName));
                total++;

                if (++i == commitSize[commitCount])
                {
                    await writer.Commit(DateTime.UtcNow, null, CancellationToken.None);

                    Console.WriteLine("commit number {0}", commitCount);

                    commitCount++;
                    i = 0;
                }
            }

            if (i > 0)
            {
                await writer.Commit(DateTime.UtcNow, null, CancellationToken.None); 
            }

            Console.WriteLine("total: {0}", total);
        }
Ejemplo n.º 6
0
        public static async Task Test3Async()
        {
            System.Net.ServicePointManager.DefaultConnectionLimit = 1024;
            IDictionary<string, string> packageHashLookup = LoadPackageHashLookup();
            HashSet<string> packageExceptionLookup = LoadPackageExceptionLookup();

            string nupkgs = @"c:\data\nuget\gallery\";

            Storage storage = new FileStorage("http://*****:*****@"c:\data\site\ordered");

            //StorageCredentials credentials = new StorageCredentials("", "");
            //CloudStorageAccount account = new CloudStorageAccount(credentials, true);
            //string storageContainer = "test1";
            //string storagePath = "";
            //string storageBaseAddress = "http://nugetjohtaylo.blob.core.windows.net/test1";
            //StorageFactory storageFactory = new AzureStorageFactory(account, storageContainer, storagePath, new Uri(storageBaseAddress)) { Verbose = true };
            //Storage storage = storageFactory.Create();

            AppendOnlyCatalogWriter writer = new AppendOnlyCatalogWriter(storage, 550);

            const int BatchSize = 64;
 
            int commitCount = 0;

            IDictionary<string, DateTime> packageCreated = LoadPackageCreatedLookup();

            DateTime lastCreated = (await PackageCatalog.ReadCommitMetadata(writer, CancellationToken.None)).Item1 ?? DateTime.MinValue;

            ParallelOptions options = new ParallelOptions();
            options.MaxDegreeOfParallelism = 8;

            // filter by lastCreated here
            Queue<KeyValuePair<string, DateTime>> packageCreatedQueue = new Queue<KeyValuePair<string, DateTime>>(packageCreated.Where(p => p.Value > lastCreated && !packageExceptionLookup.Contains(p.Key)).OrderBy(p => p.Value));

            int completed = 0;
            Stopwatch runtime = new Stopwatch();
            runtime.Start();

            Task commitTask = null;
            var context = writer.Context;
            Uri rootUri = writer.RootUri;

            while (packageCreatedQueue.Count > 0)
            {
                List<KeyValuePair<string, DateTime>> batch = new List<KeyValuePair<string, DateTime>>();

                ConcurrentBag<CatalogItem> batchItems = new ConcurrentBag<CatalogItem>();

                while (batch.Count < BatchSize && packageCreatedQueue.Count > 0)
                {
                    completed++;
                    var packagePair = packageCreatedQueue.Dequeue();
                    lastCreated = packagePair.Value;
                    batch.Add(packagePair);
                }

                var commitTime = DateTime.UtcNow;

                Parallel.ForEach(batch, options, entry =>
                {
                    FileInfo fileInfo = new FileInfo(nupkgs + entry.Key);

                    if (fileInfo.Exists)
                    {
                        using (Stream stream = new FileStream(fileInfo.FullName, FileMode.Open))
                        {
                            string packageHash = null;
                            packageHashLookup.TryGetValue(fileInfo.Name, out packageHash);

                            CatalogItem item = Utils.CreateCatalogItem(stream, entry.Value, packageHash, fileInfo.FullName);
                            batchItems.Add(item);
                        }
                    }
                });

                if (commitTask != null)
                {
                    commitTask.Wait();
                }

                foreach (var item in batchItems)
                {
                    writer.Add(item);
                }

                commitTask = Task.Run(async () => await writer.Commit(commitTime, PackageCatalog.CreateCommitMetadata(writer.RootUri, lastCreated, null), CancellationToken.None));

                // stats
                double perPackage = runtime.Elapsed.TotalSeconds / (double)completed;
                DateTime finish = DateTime.Now.AddSeconds(perPackage * packageCreatedQueue.Count);

                Console.WriteLine("commit number {0} Completed: {1} Remaining: {2} Estimated Finish: {3}",
                    commitCount++,
                    completed,
                    packageCreatedQueue.Count,
                    finish.ToString("O"));
            }

            // wait for the final commit
            if (commitTask != null)
            {
                commitTask.Wait();
            }

            Console.WriteLine("Finished in: " + runtime.Elapsed);
        }
        protected override void RunCore(CancellationToken cancellationToken)
        {
            Config.Catalog.LocalFolder.Create();

            int total = _nupkgs.Count;

            Log("Processing " + total + " nupkgs");

            ParallelOptions options = new ParallelOptions();
            options.MaxDegreeOfParallelism = 8;

            Task commitTask = null;

            using (var writer = new AppendOnlyCatalogWriter(Config.Catalog.Storage))
            {
                while (_nupkgs.Count > 0)
                {
                    Queue<PackageCatalogItem> currentBatch = new Queue<PackageCatalogItem>(_batchSize);

                    // create the batch
                    while (currentBatch.Count < _batchSize && _nupkgs.Count > 0)
                    {
                        string file = _nupkgs.Dequeue();

                        if (file.EndsWith(".nupkg", StringComparison.OrdinalIgnoreCase))
                        {
                            currentBatch.Enqueue(new NupkgCatalogItem(file));
                        }
                        else
                        {
                            currentBatch.Enqueue(new NuspecPackageCatalogItem(file));
                        }
                    }

                    // process the nupkgs and nuspec files in parallel
                    Parallel.ForEach(currentBatch, options, nupkg =>
                    {
                        nupkg.Load();
                    });

                    // wait for the previous commit to finish before adding more
                    if (commitTask != null)
                    {
                        commitTask.Wait();
                    }

                    // add everything from the queue
                    foreach (PackageCatalogItem item in currentBatch)
                    {
                        writer.Add(item);
                    }

                    // commit
                    commitTask = Task.Run(async () => await writer.Commit(DateTime.UtcNow, null, cancellationToken));

                    ProgressUpdate(total - _nupkgs.Count, total);
                }

                // wait for the final commit
                if (commitTask != null)
                {
                    commitTask.Wait();
                }
            }
        }
        static async Task<Uri> AddToCatalog(Stream nupkgStream, CancellationToken cancellationToken)
        {
            string storagePrimary = _configurationService.Get("Storage.Primary");
            CloudStorageAccount account = CloudStorageAccount.Parse(storagePrimary);

            Storage storage = new AzureStorage(account, "catalog");

            AppendOnlyCatalogWriter writer = new AppendOnlyCatalogWriter(storage);
            writer.Add(Utils.CreateCatalogItem(nupkgStream, null, null, ""));
            await writer.Commit(null, cancellationToken);

            return writer.RootUri;
        }
Ejemplo n.º 9
0
        static async Task MoreTestCatalog()
        {
            string baseAddress = "http://*****:*****@"c:\data\site\cursor";

            Storage storage = new FileStorage(baseAddress, path);

            AppendOnlyCatalogWriter writer = new AppendOnlyCatalogWriter(storage, 550);

            writer.Add(new TestCatalogItem(8));
            await writer.Commit(new DateTime(2014, 1, 11, 0, 0, 0, DateTimeKind.Utc), null, CancellationToken.None);

            writer.Add(new TestCatalogItem(9));
            await writer.Commit(new DateTime(2014, 1, 13, 0, 0, 0, DateTimeKind.Utc), null, CancellationToken.None);

            writer.Add(new TestCatalogItem(10));
            await writer.Commit(new DateTime(2014, 1, 14, 0, 0, 0, DateTimeKind.Utc), null, CancellationToken.None);

            writer.Add(new TestCatalogItem(11));
            await writer.Commit(new DateTime(2014, 1, 15, 0, 0, 0, DateTimeKind.Utc), null, CancellationToken.None);

            writer.Add(new TestCatalogItem(12));
            await writer.Commit(new DateTime(2014, 1, 17, 0, 0, 0, DateTimeKind.Utc), null, CancellationToken.None);

            writer.Add(new TestCatalogItem(13));
            await writer.Commit(new DateTime(2014, 1, 18, 0, 0, 0, DateTimeKind.Utc), null, CancellationToken.None);

            writer.Add(new TestCatalogItem(14));
            await writer.Commit(new DateTime(2014, 1, 20, 0, 0, 0, DateTimeKind.Utc), null, CancellationToken.None);

            Console.WriteLine("test catalog created");
        }
Ejemplo n.º 10
0
        static async Task MakeTestCatalog()
        {
            string baseAddress = "http://*****:*****@"c:\data\site\cursor";

            DirectoryInfo folder = new DirectoryInfo(path);
            if (folder.Exists)
            {
                Console.WriteLine("test catalog already created");
                return;
            }

            Storage storage = new FileStorage(baseAddress, path);

            AppendOnlyCatalogWriter writer = new AppendOnlyCatalogWriter(storage, 550);

            writer.Add(new TestCatalogItem(1));
            await writer.Commit(new DateTime(2014, 1, 1, 0, 0, 0, DateTimeKind.Utc), null, CancellationToken.None);

            writer.Add(new TestCatalogItem(2));
            await writer.Commit(new DateTime(2014, 1, 3, 0, 0, 0, DateTimeKind.Utc),null, CancellationToken.None);

            writer.Add(new TestCatalogItem(3));
            await writer.Commit(new DateTime(2014, 1, 4, 0, 0, 0, DateTimeKind.Utc),null, CancellationToken.None);

            writer.Add(new TestCatalogItem(4));
            await writer.Commit(new DateTime(2014, 1, 5, 0, 0, 0, DateTimeKind.Utc),null, CancellationToken.None); 

            writer.Add(new TestCatalogItem(5));
            await writer.Commit(new DateTime(2014, 1, 7, 0, 0, 0, DateTimeKind.Utc), null, CancellationToken.None);

            writer.Add(new TestCatalogItem(6));
            await writer.Commit(new DateTime(2014, 1, 8, 0, 0, 0, DateTimeKind.Utc), null, CancellationToken.None); ;

            writer.Add(new TestCatalogItem(7));
            await writer.Commit(new DateTime(2014, 1, 10, 0, 0, 0, DateTimeKind.Utc), null, CancellationToken.None);

            Console.WriteLine("test catalog created");
        }
        private static async Task<DateTime> Deletes2Catalog(SortedList<DateTime, IList<PackageIdentity>> packages, Storage storage, DateTime lastCreated, DateTime lastEdited, DateTime lastDeleted, CancellationToken cancellationToken)
        {
            var writer = new AppendOnlyCatalogWriter(storage, maxPageSize: 550);

            if (packages == null || packages.Count == 0)
            {
                return lastDeleted;
            }

            foreach (var entry in packages)
            {
                foreach (var packageIdentity in entry.Value)
                {
                    var catalogItem = new DeleteCatalogItem(packageIdentity.Id, packageIdentity.Version, entry.Key);
                    writer.Add(catalogItem);

                    Trace.TraceInformation("Delete: {0} {1}", packageIdentity.Id, packageIdentity.Version);
                }

                lastDeleted = entry.Key;
            }
            
            var commitMetadata = PackageCatalog.CreateCommitMetadata(writer.RootUri, new CommitMetadata(lastCreated, lastEdited, lastDeleted));

            await writer.Commit(commitMetadata, cancellationToken);

            Trace.TraceInformation("COMMIT");

            return lastDeleted;
        }
Ejemplo n.º 12
0
        public static async Task<DateTime> DownloadMetadata2Catalog(HttpClient client, SortedList<DateTime, IList<PackageDetails>> packages, Storage storage, DateTime lastCreated, DateTime lastEdited, DateTime lastDeleted, bool? createdPackages, CancellationToken cancellationToken, ILogger logger)
        {
            var writer = new AppendOnlyCatalogWriter(storage, maxPageSize: 550);

            var lastDate = DetermineLastDate(lastCreated, lastEdited, createdPackages);

            if (packages == null || packages.Count == 0)
            {
                return lastDate;
            }

            foreach (var entry in packages)
            {
                foreach (var packageItem in entry.Value)
                {
                    // When downloading the package binary, add a query string parameter
                    // that corresponds to the operation's timestamp.
                    // This query string will ensure the package is not cached
                    // (e.g. on the CDN) and returns the "latest and greatest" package metadata.
                    var packageUri = Utilities.GetNugetCacheBustingUri(packageItem.ContentUri, entry.Key.ToString("O"));
                    var response = await client.GetAsync(packageUri, cancellationToken);

                    if (response.IsSuccessStatusCode)
                    {
                        using (var stream = await response.Content.ReadAsStreamAsync())
                        {
                            CatalogItem item = Utils.CreateCatalogItem(
                                packageItem.ContentUri.ToString(),
                                stream,
                                packageItem.CreatedDate,
                                packageItem.LastEditedDate,
                                packageItem.PublishedDate);

                            if (item != null)
                            {
                                writer.Add(item);

                                logger?.LogInformation("Add metadata from: {PackageDetailsContentUri}", packageItem.ContentUri);
                            }
                            else
                            {
                                logger?.LogWarning("Unable to extract metadata from: {PackageDetailsContentUri}", packageItem.ContentUri);
                            }
                        }
                    }
                    else
                    {
                        if (response.StatusCode == System.Net.HttpStatusCode.NotFound)
                        {
                            //  the feed is out of sync with the actual package storage - if we don't have the package there is nothing to be done we might as well move onto the next package
                            logger?.LogWarning("Unable to download: {PackageDetailsContentUri}. Http status: {HttpStatusCode}", packageItem.ContentUri, response.StatusCode);
                        }
                        else
                        {
                            //  this should trigger a restart - of this program - and not move the cursor forward
                            logger?.LogError("Unable to download: {PackageDetailsContentUri}. Http status: {HttpStatusCode}", packageItem.ContentUri, response.StatusCode);
                            throw new Exception(
                                $"Unable to download: {packageItem.ContentUri} http status: {response.StatusCode}");
                        }
                    }
                }

                lastDate = entry.Key;
            }

            if (createdPackages.HasValue)
            {
                lastCreated = createdPackages.Value ? lastDate : lastCreated;
                lastEdited = !createdPackages.Value ? lastDate : lastEdited;
            }

            var commitMetadata = PackageCatalog.CreateCommitMetadata(writer.RootUri, new CommitMetadata(lastCreated, lastEdited, lastDeleted));

            await writer.Commit(commitMetadata, cancellationToken);

            logger?.LogInformation("COMMIT metadata to catalog.");

            return lastDate;
        }
        private async Task Commit(AppendOnlyCatalogWriter writer, CantonCatalogItem[] batchItems)
        {
            var orderedBatch = batchItems.ToList();
            orderedBatch.Sort(CantonCatalogItem.Compare);

            int lastHighestCommit = 0;
            DateTime? latestPublished = null;

            // add the items to the writer
            foreach (var orderedItem in orderedBatch)
            {
                lastHighestCommit = orderedItem.CantonCommitId + 1;
                writer.Add(orderedItem);
            }

            Task cursorTask = null;

            // only save the cursor if we did something
            if (lastHighestCommit > 0)
            {
                // find the most recent package
                latestPublished = batchItems.Select(c => c.Published).OrderByDescending(d => d).FirstOrDefault();

                // update the cursor
                JObject obj = new JObject();
                // add one here since we are already added the current number
                obj.Add("cantonCommitId", lastHighestCommit);
                Log("Cursor cantonCommitId: " + lastHighestCommit);

                Cursor.Position = DateTime.UtcNow;
                Cursor.Metadata = obj;
                cursorTask = Cursor.Save();
            }

            if (writer.Count > 0)
            {
                // perform the commit
                Stopwatch timer = new Stopwatch();
                timer.Start();

                IGraph commitData = PackageCatalog.CreateCommitMetadata(writer.RootUri, latestPublished, latestPublished);

                // commit
                await writer.Commit(DateTime.UtcNow, commitData, CancellationToken.None);

                timer.Stop();
                Console.WriteLine("Commit duration: " + timer.Elapsed);
            }

            if (cursorTask != null)
            {
                await cursorTask;
            }
        }
        public static async Task BuildCatalogAsync(string path, Storage storage, IEnumerable<string> ids)
        {
            AppendOnlyCatalogWriter writer = new AppendOnlyCatalogWriter(storage, 600);

            const int BatchSize = 200;
            int i = 0;

            int commitCount = 0;

            IEnumerable<string> files = GetFileList(path, ids);

            Console.WriteLine("initial build files = {0}", files.Count());

            foreach (string fullName in files)
            {
                writer.Add(new NuspecPackageCatalogItem(fullName));

                if (++i % BatchSize == 0)
                {
                    await writer.Commit(DateTime.UtcNow, null, CancellationToken.None);

                    Console.WriteLine("commit number {0}", commitCount++);
                }
            }

            await writer.Commit(DateTime.UtcNow, null, CancellationToken.None);

            Console.WriteLine("commit number {0}", commitCount++);
        }