public static async Task ProcessGraphs(
            string id,
            IDictionary<string, IGraph> sortedGraphs,
            StorageFactory storageFactory,
            Uri contentBaseAddress,
            int partitionSize,
            int packageCountThreshold,
             CancellationToken cancellationToken)
        {
            int versionAlreadyExistsCount = 0;
            existingVersionsWithID = new List<string>();
            
            try
            {
                Storage storage = storageFactory.Create(id.ToLowerInvariant());

                Uri resourceUri = storage.ResolveUri("index.json");
                string json = await storage.LoadString(resourceUri, cancellationToken);

                int count = Utils.CountItems(json);

                //Determine if there are any versions that are existing already
                CollectorHttpClient httpClient = new CollectorHttpClient();
                foreach (var graph in sortedGraphs)
                {
                    JObject jsonContent = await httpClient.GetJObjectAsync(new Uri(graph.Key), cancellationToken);
                    string existingId = jsonContent["@id"].ToString();
                    string existingVersionWithId = existingId.Substring(existingId.LastIndexOf("/") + 1);

                    string existingVersion = jsonContent["version"].ToString() + ".json";
                    
                    //Determine if the version is actually available
                    //In Registration blobs, the format is /packageID/packageVersion.json
                    //So to check the existence of version we need to know only the version.json
                    if (storage.Exists(existingVersion))
                    {
                        //When we compare later in AddExistingItems, we need the "packageId.packageversion.json" for comparison so store it with Id
                        existingVersionsWithID.Add(existingVersionWithId);
                        versionAlreadyExistsCount++;
                    }
                }

                int total = count + sortedGraphs.Count - versionAlreadyExistsCount;

                if (total < packageCountThreshold)
                {
                    await SaveSmallRegistration(storage, storageFactory.BaseAddress, sortedGraphs, contentBaseAddress, partitionSize, cancellationToken);
                }
                else
                {
                    await SaveLargeRegistration(storage, storageFactory.BaseAddress, sortedGraphs, json, contentBaseAddress, partitionSize, cancellationToken);
                }
            }
            catch (Exception e)
            {
                throw new Exception(string.Format("Process id = {0}", id), e);
            }
        }
 public RegistrationPersistence(StorageFactory storageFactory, RegistrationKey registrationKey, int partitionSize, int packageCountThreshold, Uri contentBaseAddress)
 {
     _storage = new RecordingStorage(storageFactory.Create(registrationKey.ToString()));
     _registrationUri = _storage.ResolveUri("index.json");
     _packageCountThreshold = packageCountThreshold;
     _partitionSize = partitionSize;
     _registrationBaseAddress = storageFactory.BaseAddress;
     _contentBaseAddress = contentBaseAddress;
 }
        public async Task Loop(string source, StorageFactory storageFactory, string contentBaseAddress, bool unlistShouldDelete, bool verbose, int interval, CancellationToken cancellationToken)
        {
            CommitCollector collector = new RegistrationCollector(new Uri(source), storageFactory, CommandHelpers.GetHttpMessageHandlerFactory(verbose))
            {
                ContentBaseAddress = contentBaseAddress == null ? null : new Uri(contentBaseAddress),
                UnlistShouldDelete = unlistShouldDelete
            };

            Storage storage = storageFactory.Create();
            ReadWriteCursor front = new DurableCursor(storage.ResolveUri("cursor.json"), storage, MemoryCursor.Min.Value);
            ReadCursor back = MemoryCursor.Max;

            while (true)
            {
                bool run = false;
                do
                {
                    run = await collector.Run(front, back, cancellationToken);
                }
                while (run);

                Thread.Sleep(interval * 1000);
            }
        }
        static async Task ProcessPackages(string gallery, StorageFactory storageFactory, string id, string version, bool verbose, CancellationToken cancellationToken)
        {
            int timeout = 300;

            Func<HttpMessageHandler> handlerFunc = CommandHelpers.GetHttpMessageHandlerFactory(verbose);

            HttpMessageHandler handler = (handlerFunc != null) ? handlerFunc() : new WebRequestHandler { AllowPipelining = true };

            using (HttpClient client = new HttpClient(handler))
            {
                client.Timeout = TimeSpan.FromSeconds(timeout);

                //  if teh version is specified a single package is processed otherwise all the packages corresponding to that id are processed

                Uri uri = (version == null) ? MakePackageUri(gallery, id) : MakePackageUri(gallery, id, version);

                SortedList<DateTime, IList<Tuple<Uri, FeedDetails>>> packages = await GetPackages(client, uri, "Created");

                Trace.TraceInformation("downloading {0} packages", packages.Select(t => t.Value.Count).Sum());

                Storage storage = storageFactory.Create();

                //  the idea here is to leave the lastCreated and lastEdited values exactly as they were

                const string LastCreated = "nuget:lastCreated";
                const string LastEdited = "nuget:lastEdited";

                DateTime lastCreated = await GetCatalogProperty(storage, LastCreated, cancellationToken) ?? DateTime.MinValue.ToUniversalTime();
                DateTime lastEdited = await GetCatalogProperty(storage, LastEdited, cancellationToken) ?? DateTime.MinValue.ToUniversalTime();

                DateTime d = await DownloadMetadata2Catalog(client, packages, storage, lastCreated, lastEdited, null, cancellationToken);
            }
        }
        static async Task Loop(string gallery, StorageFactory storageFactory, bool verbose, int interval, DateTime? startDate, CancellationToken cancellationToken)
        {
            Storage storage = storageFactory.Create();

            const string LastCreated = "nuget:lastCreated";
            const string LastEdited = "nuget:lastEdited";

            int top = 20;
            int timeout = 300;

            while (true)
            {
                Func<HttpMessageHandler> handlerFunc = CommandHelpers.GetHttpMessageHandlerFactory(verbose);

                HttpMessageHandler handler = (handlerFunc != null) ? handlerFunc() : new WebRequestHandler { AllowPipelining = true };

                using (HttpClient client = new HttpClient(handler))
                {
                    client.Timeout = TimeSpan.FromSeconds(timeout);

                    //  fetch and add all newly CREATED packages - in order
                    DateTime lastCreated = await GetCatalogProperty(storage, LastCreated, cancellationToken) ?? (startDate ?? DateTime.MinValue.ToUniversalTime());
                    DateTime lastEdited = await GetCatalogProperty(storage, LastEdited, cancellationToken) ?? lastCreated;

                    SortedList<DateTime, IList<Tuple<Uri, FeedDetails>>> createdPackages;
                    DateTime previousLastCreated = DateTime.MinValue;
                    do
                    {
                        Trace.TraceInformation("CATALOG LastCreated: {0}", lastCreated.ToString("O"));

                        createdPackages = await GetCreatedPackages(client, gallery, lastCreated, top);
                        Trace.TraceInformation("FEED CreatedPackages: {0}", createdPackages.Count);

                        lastCreated = await DownloadMetadata2Catalog(client, createdPackages, storage, lastCreated, lastEdited, true, cancellationToken);
                        if (previousLastCreated == lastCreated)
                        {
                            break;
                        }
                        previousLastCreated = lastCreated;
                    }
                    while (createdPackages.Count > 0);

                    //  THEN fetch and add all EDITED packages - in order

                    SortedList<DateTime, IList<Tuple<Uri, FeedDetails>>> editedPackages;
                    DateTime previousLastEdited = DateTime.MinValue;
                    do
                    {
                        Trace.TraceInformation("CATALOG LastEdited: {0}", lastEdited.ToString("O"));

                        editedPackages = await GetEditedPackages(client, gallery, lastEdited, top);

                        Trace.TraceInformation("FEED EditedPackages: {0}", editedPackages.Count);

                        lastEdited = await DownloadMetadata2Catalog(client, editedPackages, storage, lastCreated, lastEdited, false, cancellationToken);
                        if (previousLastEdited == lastEdited)
                        {
                            break;
                        }
                        previousLastEdited = lastEdited;
                    }
                    while (editedPackages.Count > 0);
                }

                Thread.Sleep(interval * 1000);
            }
        }
        private async Task ProcessPackages(string gallery, StorageFactory storageFactory, string id, string version, bool verbose, CancellationToken cancellationToken)
        {
            var timeout = TimeSpan.FromSeconds(300);
            
            using (var client = CreateHttpClient(verbose))
            {
                client.Timeout = timeout;

                //  if the version is specified a single package is processed otherwise all the packages corresponding to that id are processed

                var uri = (version == null) ? MakePackageUri(gallery, id) : MakePackageUri(gallery, id, version);

                var packages = await GetPackages(client, uri, "Created");

                Trace.TraceInformation("downloading {0} packages", packages.Select(t => t.Value.Count).Sum());

                var storage = storageFactory.Create();

                //  the idea here is to leave the lastCreated, lastEdited and lastDeleted values exactly as they were
                var lastCreated = await GetCatalogProperty(storage, "nuget:lastCreated", cancellationToken) ?? DateTime.MinValue.ToUniversalTime();
                var lastEdited = await GetCatalogProperty(storage, "nuget:lastEdited", cancellationToken) ?? DateTime.MinValue.ToUniversalTime();
                var lastDeleted = await GetCatalogProperty(storage, "nuget:lastDeleted", cancellationToken) ?? DateTime.MinValue.ToUniversalTime();

                await DownloadMetadata2Catalog(client, packages, storage, lastCreated, lastEdited, lastDeleted, null, cancellationToken);
            }
        }
        private async Task Loop(string gallery, StorageFactory catalogStorageFactory, StorageFactory auditingStorageFactory, bool verbose, int interval, DateTime? startDate, CancellationToken cancellationToken)
        {
            var catalogStorage = catalogStorageFactory.Create();
            var auditingStorage = auditingStorageFactory.Create();

            var top = 20;
            var timeout = TimeSpan.FromSeconds(300);

            while (true)
            {
                await ProcessFeed(gallery, catalogStorage, auditingStorage, startDate, timeout, top, verbose, cancellationToken);

                Thread.Sleep(interval * 1000);
            }
        }