public GalleryExportBatcher(int batchSize, CatalogWriter writer)
 {
     _batchSize = batchSize;
     _writer = writer;
     _currentBatch = new List<GalleryExportPackage>();
     Total = 0;
 }
 public CatalogBatcher(int batchSize, CatalogWriter writer)
 {
     _batchSize = batchSize;
     _writer = writer;
     _currentBatch = new List<CatalogItem>();
     Total = 0;
 }
        public static async Task Test0Async()
        {
            Storage storage = new FileStorage
            {
                Path = @"c:\data\site\test",
                Container = "test",
                BaseAddress = "http://localhost:8000/"
            };

            //  first save the delete into the catalog

            CatalogContext context = new CatalogContext();
            using (CatalogWriter writer = new CatalogWriter(storage, context, 1000, true))
            {
                //writer.Add(new DeletePackageCatalogItem("Test.Metadata.Service", "1.0.0"));
                writer.Add(new DeletePackageCatalogItem("Test.Metadata.Service", "2.0.0"));
                //writer.Add(new DeletePackageCatalogItem("Test.Metadata.Service", "3.0.0"));
                await writer.Commit(DateTime.Now);
            }

            //  second perform that delete on the various feeds - in this case the default resolver feed
            
            ResolverDeleteCollector collector = new ResolverDeleteCollector(storage);
            await collector.Run(new Uri("http://localhost:8000/test/catalog/index.json"), DateTime.MinValue);
        }
        public static async Task Test0Async()
        {
            string nuspecs = @"c:\data\nuget\versions";

            Storage storage = new FileStorage("http://*****:*****@"c:\data\site\full");

            CatalogContext context = new CatalogContext();

            CatalogWriter writer = new CatalogWriter(storage, context, 10);

            const int BatchSize = 1;
            int i = 0;

            int commitCount = 0;

            DirectoryInfo directoryInfo = new DirectoryInfo(nuspecs);
            foreach (FileInfo fileInfo in directoryInfo.EnumerateFiles("*.xml"))
            {
                writer.Add(new NuspecPackageCatalogItem(fileInfo.FullName));

                if (++i % BatchSize == 0)
                {
                    await writer.Commit(DateTime.UtcNow);

                    Console.WriteLine("commit number {0}", commitCount++);
                }
            }

            await writer.Commit(DateTime.UtcNow);

            Console.WriteLine("commit number {0}", commitCount++);
        }
        public CatalogUpdater(CatalogWriter writer, ChecksumRecords checksums, CollectorHttpClient client)
        {
            DatabaseChecksumBatchSize = DefaultDatabaseChecksumBatchSize;
            CatalogAddBatchSize = DefaultCatalogAddBatchSize;

            _writer = writer;
            _checksums = checksums;
            _client = client;
        }
        public static async Task Test0Async()
        {
            //Storage storage = new FileStorage
            //{
            //    Path = @"c:\data\site\test",
            //    Container = "test",
            //    BaseAddress = "http://localhost:8000/"
            //};

            Storage storage = new AzureStorage(
                CloudStorageAccount.Parse("AccountName=nuget3;AccountKey=;DefaultEndpointsProtocol=https"), "test");

            CatalogContext context = new CatalogContext();

            CatalogWriter writer = new CatalogWriter(storage, context, 4, false);

            string[] first = { "john", "paul", "ringo", "george" };
            foreach (string item in first)
            {
                writer.Add(new TestCatalogItem(item));
            }
            await writer.Commit(new DateTime(2010, 12, 25, 12, 0, 0));

            string[] second = { "jimmy", "robert" };
            foreach (string item in second)
            {
                writer.Add(new TestCatalogItem(item));
            }
            await writer.Commit(new DateTime(2011, 12, 25, 12, 0, 0));

            string[] third = { "john-paul", "john" };
            foreach (string item in third)
            {
                writer.Add(new TestCatalogItem(item));
            }
            await writer.Commit(new DateTime(2012, 12, 25, 12, 0, 0));

            //  collection...

            Uri index = storage.ResolveUri("index.json");

            TestItemCollector collector = new TestItemCollector();

            Console.WriteLine("----------------");

            await collector.Run(index, new DateTime(2012, 10, 31, 12, 0, 0));

            Console.WriteLine("----------------");

            await collector.Run(index, new DateTime(2011, 10, 31, 12, 0, 0));

            Console.WriteLine("----------------");

            CollectorCursor cursor = await collector.Run(index, new CollectorCursor(DateTime.MinValue));
        }
        public static async Task Test1Async()
        {
            string nuspecs = @"c:\data\nuget\nuspecs";

            Storage storage = new FileStorage("http://*****:*****@"c:\data\site\full");

            CatalogContext context = new CatalogContext();

            CatalogWriter writer = new CatalogWriter(storage, context, 200);

            int total = 0;

            //int[] commitSize = { 50, 40, 25, 50, 10, 30, 40, 5, 400, 30, 10, 20, 40, 50, 90, 70, 50, 50, 50, 50, 60, 70 };
            int[] commitSize = { 
                200, 200, 200, 200, 200, 
                //200, 200, 200, 200, 200, 
                //200, 200, 200, 200, 200, 
                //200, 200, 200, 200, 200, 
                //200, 200, 200, 200, 200,
                //200, 200, 200, 200, 200
            };
            int i = 0;

            int commitCount = 0;

            DirectoryInfo directoryInfo = new DirectoryInfo(nuspecs);
            foreach (FileInfo fileInfo in directoryInfo.EnumerateFiles("*.xml"))
            {
                if (commitCount == commitSize.Length)
                {
                    break;
                }

                writer.Add(new NuspecPackageCatalogItem(fileInfo.FullName));
                total++;

                if (++i == commitSize[commitCount])
                {
                    await writer.Commit(DateTime.UtcNow);

                    Console.WriteLine("commit number {0}", commitCount);

                    commitCount++;
                    i = 0;
                }
            }

            Console.WriteLine("total: {0}", total);
        }
        public static void Test0()
        {
            const int SqlChunkSize = 8000;
            string sqlConnectionString = "";

            const int CatalogBatchSize = 1000;
            const int CatalogMaxPageSize = 1000;
            Storage storage = new FileStorage
            {
                Path = @"c:\data\site\export2",
                Container = "export2",
                BaseAddress = "http://localhost:8000/"
            };

            CatalogWriter writer = new CatalogWriter(storage, new CatalogContext(), CatalogMaxPageSize);

            GalleryExportBatcher batcher = new GalleryExportBatcher(CatalogBatchSize, writer);

            int lastHighestPackageKey = 0;

            int count = 0;

            while (true)
            {
                Tuple<int, int> range = GalleryExport.GetNextRange(sqlConnectionString, lastHighestPackageKey, SqlChunkSize);

                if (range.Item1 == 0 && range.Item2 == 0)
                {
                    break;
                }

                if (count++ == 3)
                {
                    break;
                }

                Console.WriteLine("{0} {1}", range.Item1, range.Item2);

                GalleryExport.FetchRange(sqlConnectionString, range, batcher);

                lastHighestPackageKey = range.Item2;
            }

            batcher.Complete();

            Console.WriteLine(batcher.Total);
        }
        public static async Task Test0Async()
        {
            string nuspecs = @"c:\data\nuspecs";

            Storage storage = new FileStorage
            {
                Path = @"c:\data\site\pub",
                Container = "pub",
                BaseAddress = "http://localhost:8000/"
            };

            //Storage storage = new AzureStorage
            //{
            //    AccountName = "nuget3",
            //    AccountKey = "",
            //    Container = "pub",
            //    BaseAddress = "http://nuget3.blob.core.windows.net"
            //};

            CatalogContext context = new CatalogContext();

            CatalogWriter writer = new CatalogWriter(storage, context, 1000);

            const int BatchSize = 1000;
            int i = 0;

            int commitCount = 0;

            DirectoryInfo directoryInfo = new DirectoryInfo(nuspecs);
            foreach (FileInfo fileInfo in directoryInfo.EnumerateFiles("*.xml"))
            {
                writer.Add(new NuspecPackageCatalogItem(fileInfo));

                if (++i % BatchSize == 0)
                {
                    await writer.Commit(DateTime.Now);

                    Console.WriteLine("commit number {0}", commitCount++);
                }
            }

            await writer.Commit(DateTime.Now);

            Console.WriteLine("commit number {0}", commitCount++);
        }
        public static async Task Test1Async()
        {
            Storage storage = new FileStorage
            {
                Path = @"c:\data\site\full",
                Container = "full",
                BaseAddress = "http://localhost:8000/"
            };

            //  first save the delete into the catalog

            CatalogContext context = new CatalogContext();
            using (CatalogWriter writer = new CatalogWriter(storage, context, 1000, true))
            {
                writer.Add(new DeleteRegistrationCatalogItem("abc"));
                await writer.Commit(DateTime.Now);
            }

            //  second perform that delete on the various feeds - in this case the default resolver feed

            ResolverDeleteCollector collector = new ResolverDeleteCollector(storage);
            await collector.Run(new Uri("http://localhost:8000/full/catalog/index.json"), DateTime.MinValue);
        }
        public static async Task CreateStatisticsCatalogAsync(Storage storage, string connectionString)
        {
            const int BatchSize = 100;
            int i = 0;

            using (CatalogWriter writer = new CatalogWriter(storage, new CatalogContext(), 500))
            {
                int lastKey = 0;
                int iterations = 0;

                while (true)
                {
                    iterations++;

                    DateTime minDownloadTimeStamp;
                    DateTime maxDownloadTimeStamp;

                    JArray batch = GetNextBatch(connectionString, ref lastKey, out minDownloadTimeStamp, out maxDownloadTimeStamp);

                    if (batch == null)
                    {
                        break;
                    }

                    writer.Add(new StatisticsCatalogItem(batch, lastKey.ToString(), minDownloadTimeStamp, maxDownloadTimeStamp));

                    if (++i % BatchSize == 0)
                    {
                        await writer.Commit();
                    }
                }

                await writer.Commit();
            }
        }
        protected override void RunCore()
        {
            Config.Catalog.LocalFolder.Create();

            int total = _nupkgs.Count;

            Log("Processing " + total + " nupkgs");

            ParallelOptions options = new ParallelOptions();
            options.MaxDegreeOfParallelism = 8;

            Task commitTask = null;

            using (var writer = new CatalogWriter(Config.Catalog.Storage, new CatalogContext()))
            {
                while (_nupkgs.Count > 0)
                {
                    Queue<PackageCatalogItem> currentBatch = new Queue<PackageCatalogItem>(_batchSize);

                    // create the batch
                    while (currentBatch.Count < _batchSize && _nupkgs.Count > 0)
                    {
                        string file = _nupkgs.Dequeue();

                        if (file.EndsWith(".nupkg", StringComparison.OrdinalIgnoreCase))
                        {
                            currentBatch.Enqueue(new NupkgCatalogItem(file));
                        }
                        else
                        {
                            currentBatch.Enqueue(new NuspecPackageCatalogItem(file));
                        }
                    }

                    // process the nupkgs and nuspec files in parallel
                    Parallel.ForEach(currentBatch, options, nupkg =>
                    {
                        nupkg.Load();
                    });

                    // wait for the previous commit to finish before adding more
                    if (commitTask != null)
                    {
                        commitTask.Wait();
                    }

                    // add everything from the queue
                    foreach (PackageCatalogItem item in currentBatch)
                    {
                        writer.Add(item);
                    }

                    // commit
                    commitTask = Task.Run(async () => await writer.Commit(DateTime.UtcNow));

                    ProgressUpdate(total - _nupkgs.Count, total);
                }

                // wait for the final commit
                if (commitTask != null)
                {
                    commitTask.Wait();
                }
            }
        }
        public static async Task Test1Async()
        {
            Storage storage = new FileStorage("http://*****:*****@"c:\data\site\test");

            //Storage storage = new AzureStorage
            //{
            //    AccountName = "",
            //    AccountKey = "",
            //    Container = "test",
            //    BaseAddress = "http://nuget3.blob.core.windows.net"
            //};

            CatalogContext context = new CatalogContext();

            CatalogWriter writer = new CatalogWriter(storage, context, 4, false);

            string[] first = { "john", "paul", "ringo", "george" };
            foreach (string item in first)
            {
                writer.Add(new TestCatalogItem(item));
            }
            await writer.Commit(new DateTime(2010, 12, 25, 12, 0, 0));

            string baseAddress = storage.BaseAddress + "/";
            Uri index = new Uri(baseAddress + "catalog/index.json");
            TestItemCollector collector = new TestItemCollector();
            CollectorCursor cursor = await collector.Run(index, DateTime.MinValue);

            string[] second = { "jimmy", "robert", "john-paul", "john" };
            foreach (string item in second)
            {
                writer.Add(new TestCatalogItem(item));
            }
            await writer.Commit(new DateTime(2011, 12, 25, 12, 0, 0));

            cursor = await collector.Run(index, cursor);
        }
        public void Rebuild(RebuildArgs args)
        {
            const int batchSize = 2000;

            if (Directory.Exists(args.CatalogFolder))
            {
                Console.WriteLine("Catalog folder exists. Deleting!");
                Directory.Delete(args.CatalogFolder, recursive: true);
            }

            // Load storage
            Storage storage = new FileStorage(args.BaseAddress, args.CatalogFolder);
            using (var writer = new CatalogWriter(storage, new CatalogContext()))
            {
                if (!String.IsNullOrEmpty(args.DatabaseConnection))
                {
                    var batcher = new GalleryExportBatcher(batchSize, writer);
                    int lastHighest = 0;
                    while (true)
                    {
                        var range = GalleryExport.GetNextRange(
                            args.DatabaseConnection,
                            lastHighest,
                            batchSize).Result;
                        if (range.Item1 == 0 && range.Item2 == 0)
                        {
                            break;
                        }
                        Console.WriteLine("Writing packages with Keys {0}-{1} to catalog...", range.Item1, range.Item2);
                        GalleryExport.WriteRange(
                            args.DatabaseConnection,
                            range,
                            batcher).Wait();
                        lastHighest = range.Item2;
                    }
                    batcher.Complete().Wait();
                }
                else if (!String.IsNullOrEmpty(args.NuPkgFolder))
                {
                    Stopwatch timer = new Stopwatch();
                    timer.Start();

                    // files are sorted by GetFiles
                    Queue<string> files = new Queue<string>(Directory.GetFiles(args.NuPkgFolder, "*.nu*", SearchOption.TopDirectoryOnly)
                        .Where(s => s.EndsWith(".nupkg", StringComparison.OrdinalIgnoreCase) || s.EndsWith(".nuspec", StringComparison.OrdinalIgnoreCase)));

                    int total = files.Count;

                    ParallelOptions options = new ParallelOptions();
                    options.MaxDegreeOfParallelism = 8;

                    Task commitTask = null;

                    while (files.Count > 0)
                    {
                        Queue<PackageCatalogItem> currentBatch = new Queue<PackageCatalogItem>(batchSize);

                        // create the batch
                        while (currentBatch.Count < batchSize && files.Count > 0)
                        {
                            string file = files.Dequeue();

                            if (file.EndsWith(".nupkg", StringComparison.OrdinalIgnoreCase))
                            {
                                currentBatch.Enqueue(new NupkgCatalogItem(file));
                            }
                            else
                            {
                                currentBatch.Enqueue(new NuspecPackageCatalogItem(file));
                            }
                        }

                        // process the nupkgs and nuspec files in parallel
                        Parallel.ForEach(currentBatch, options, nupkg =>
                        {
                            nupkg.Load();
                        });

                        // wait for the previous commit to finish before adding more
                        if (commitTask != null)
                        {
                            commitTask.Wait();
                        }

                        // add everything from the queue
                        foreach(PackageCatalogItem item in currentBatch)
                        {
                            writer.Add(item);
                        }

                        // commit
                        commitTask = Task.Run(async () => await writer.Commit(DateTime.UtcNow));
                        Console.WriteLine("committing {0}/{1}", total - files.Count, total);
                    }

                    // wait for the final commit
                    if (commitTask != null)
                    {
                        commitTask.Wait();
                    }

                    timer.Stop();

                    Console.ForegroundColor = ConsoleColor.Green;
                    Console.WriteLine("Committed {0} catalog items in {1}", total, timer.Elapsed);
                    Console.ResetColor();
                }
            }
        }
        public static async Task Test4Async()
        {
            Storage storage = new FileStorage
            {
                Path = @"c:\data\site\test",
                Container = "test",
                BaseAddress = "http://localhost:8000/"
            };

            CatalogContext context = new CatalogContext();

            CatalogWriter writer = new CatalogWriter(storage, context, 4);

            string[] names1 = { "a", "b", "c", "d", "e" };
            string[] names2 = { "f", "g", "h" };
            string[] names3 = { "i", "j", "k" };

            DateTime timeStamp = DateTime.UtcNow;

            foreach (string name in names1)
            {
                writer.Add(new TestCatalogItem(name));
            }
            await writer.Commit(timeStamp);

            Console.WriteLine("commit #1 timeStamp {0}", await CatalogWriter.GetLastCommitTimeStamp(storage));
            Console.WriteLine("commit #1 count {0}", await CatalogWriter.GetCount(storage));

            timeStamp = timeStamp.AddHours(1);

            foreach (string name in names2)
            {
                writer.Add(new TestCatalogItem(name));
            }
            await writer.Commit(timeStamp);

            Console.WriteLine("commit #2 timeStamp {0}", await CatalogWriter.GetLastCommitTimeStamp(storage));
            Console.WriteLine("commit #2 count {0}", await CatalogWriter.GetCount(storage));

            timeStamp = timeStamp.AddHours(1);

            foreach (string name in names3)
            {
                writer.Add(new TestCatalogItem(name));
            }
            await writer.Commit(timeStamp);

            Console.WriteLine("commit #3 timeStamp {0}", await CatalogWriter.GetLastCommitTimeStamp(storage));
            Console.WriteLine("commit #3 count {0}", await CatalogWriter.GetCount(storage));
        }
        public static async Task Test3Async()
        {
            Storage storage = new FileStorage
            {
                Path = @"c:\data\site\test",
                Container = "test",
                BaseAddress = "http://localhost:8000/"
            };

            CatalogContext context = new CatalogContext();

            CatalogWriter writer = new CatalogWriter(storage, context, 1000);

            string[] names1 = { "a", "b", "c", "d", "e" };
            string[] names2 = { "f", "g", "h" };
            string[] names3 = { "i", "j", "k" };

            foreach (string name in names1)
            {
                writer.Add(new TestCatalogItem(name));
            }
            await writer.Commit(new Dictionary<string, string> { { "prop1", "value1.1" }, { "prop2", "value2.1" } });

            Console.WriteLine("commit user data #1");

            foreach (KeyValuePair<string, string> items in await CatalogWriter.GetCommitUserData(storage))
            {
                Console.WriteLine("{0} {1}", items.Key, items.Value);
            }

            foreach (string name in names2)
            {
                writer.Add(new TestCatalogItem(name));
            }
            await writer.Commit(new Dictionary<string, string> { { "prop1", "value1.2" }, { "prop2", "value2.2" } });

            Console.WriteLine("commit user data #2");

            foreach (KeyValuePair<string, string> items in await CatalogWriter.GetCommitUserData(storage))
            {
                Console.WriteLine("{0} {1}", items.Key, items.Value);
            }

            foreach (string name in names3)
            {
                writer.Add(new TestCatalogItem(name));
            }
            await writer.Commit(new Dictionary<string, string> { { "prop1", "value1.3" }, { "prop2", "value2.3" } });

            Console.WriteLine("commit user data #3");

            foreach (KeyValuePair<string, string> items in await CatalogWriter.GetCommitUserData(storage))
            {
                Console.WriteLine("{0} {1}", items.Key, items.Value);
            }
        }