protected override void Init(IDictionary<string, string> arguments, CancellationToken cancellationToken) { _gallery = arguments.GetOrThrow<string>(Arguments.Gallery); _verbose = arguments.GetOrDefault(Arguments.Verbose, false); _id = arguments.GetOrThrow<string>(Arguments.Id); _version = arguments.GetOrDefault<string>(Arguments.Version); _storageFactory = CommandHelpers.CreateStorageFactory(arguments, _verbose); }
public static async Task ProcessGraphs( string id, IDictionary<string, IGraph> sortedGraphs, StorageFactory storageFactory, Uri contentBaseAddress, int partitionSize, int packageCountThreshold, CancellationToken cancellationToken) { int versionAlreadyExistsCount = 0; existingVersionsWithID = new List<string>(); try { Storage storage = storageFactory.Create(id.ToLowerInvariant()); Uri resourceUri = storage.ResolveUri("index.json"); string json = await storage.LoadString(resourceUri, cancellationToken); int count = Utils.CountItems(json); //Determine if there are any versions that are existing already CollectorHttpClient httpClient = new CollectorHttpClient(); foreach (var graph in sortedGraphs) { JObject jsonContent = await httpClient.GetJObjectAsync(new Uri(graph.Key), cancellationToken); string existingId = jsonContent["@id"].ToString(); string existingVersionWithId = existingId.Substring(existingId.LastIndexOf("/") + 1); string existingVersion = jsonContent["version"].ToString() + ".json"; //Determine if the version is actually available //In Registration blobs, the format is /packageID/packageVersion.json //So to check the existence of version we need to know only the version.json if (storage.Exists(existingVersion)) { //When we compare later in AddExistingItems, we need the "packageId.packageversion.json" for comparison so store it with Id existingVersionsWithID.Add(existingVersionWithId); versionAlreadyExistsCount++; } } int total = count + sortedGraphs.Count - versionAlreadyExistsCount; if (total < packageCountThreshold) { await SaveSmallRegistration(storage, storageFactory.BaseAddress, sortedGraphs, contentBaseAddress, partitionSize, cancellationToken); } else { await SaveLargeRegistration(storage, storageFactory.BaseAddress, sortedGraphs, json, contentBaseAddress, partitionSize, cancellationToken); } } catch (Exception e) { throw new Exception(string.Format("Process id = {0}", id), e); } }
public DnxMaker(StorageFactory storageFactory) { if (storageFactory == null) { throw new ArgumentNullException(nameof(storageFactory)); } _storageFactory = storageFactory; }
public AggregateStorageFactory(StorageFactory primaryStorageFactory, StorageFactory[] secondaryStorageFactories, AggregateStorage.WriteSecondaryStorageContentInterceptor writeSecondaryStorageContentInterceptor) { _primaryStorageFactory = primaryStorageFactory; _secondaryStorageFactories = secondaryStorageFactories; _writeSecondaryStorageContentInterceptor = writeSecondaryStorageContentInterceptor; BaseAddress = _primaryStorageFactory.BaseAddress; }
public RegistrationPersistence(StorageFactory storageFactory, RegistrationKey registrationKey, int partitionSize, int packageCountThreshold, Uri contentBaseAddress) { _storage = new RecordingStorage(storageFactory.Create(registrationKey.ToString())); _registrationUri = _storage.ResolveUri("index.json"); _packageCountThreshold = packageCountThreshold; _partitionSize = partitionSize; _registrationBaseAddress = storageFactory.BaseAddress; _contentBaseAddress = contentBaseAddress; }
public RegistrationCatalogCollector(Uri index, StorageFactory storageFactory, Func<HttpMessageHandler> handlerFunc = null) : base(index, new Uri[] { Schema.DataTypes.PackageDetails }, handlerFunc) { _storageFactory = storageFactory; ContentBaseAddress = new Uri("http://tempuri.org"); PartitionSize = 64; PackageCountThreshold = 128; }
public static async Task Process(RegistrationKey registrationKey, IDictionary<string, IGraph> newItems, StorageFactory storageFactory, Uri contentBaseAddress, int partitionSize, int packageCountThreshold, CancellationToken cancellationToken) { Trace.TraceInformation("RegistrationMaker.Process: registrationKey = {0} newItems: {1}", registrationKey, newItems.Count); IRegistrationPersistence registration = new RegistrationPersistence(storageFactory, registrationKey, partitionSize, packageCountThreshold, contentBaseAddress); IDictionary<RegistrationEntryKey, RegistrationCatalogEntry> existing = await registration.Load(cancellationToken); Trace.TraceInformation("RegistrationMaker.Process: existing = {0}", existing.Count); IDictionary<RegistrationEntryKey, RegistrationCatalogEntry> delta = PromoteRegistrationKey(newItems); Trace.TraceInformation("RegistrationMaker.Process: delta = {0}", delta.Count); IDictionary<RegistrationEntryKey, RegistrationCatalogEntry> resulting = Apply(existing, delta); Trace.TraceInformation("RegistrationMaker.Process: resulting = {0}", resulting.Count); await registration.Save(resulting, cancellationToken); }
public async Task Loop(string source, StorageFactory storageFactory, string contentBaseAddress, bool unlistShouldDelete, bool verbose, int interval, CancellationToken cancellationToken) { CommitCollector collector = new RegistrationCollector(new Uri(source), storageFactory, CommandHelpers.GetHttpMessageHandlerFactory(verbose)) { ContentBaseAddress = contentBaseAddress == null ? null : new Uri(contentBaseAddress), UnlistShouldDelete = unlistShouldDelete }; Storage storage = storageFactory.Create(); ReadWriteCursor front = new DurableCursor(storage.ResolveUri("cursor.json"), storage, MemoryCursor.Min.Value); ReadCursor back = MemoryCursor.Max; while (true) { bool run = false; do { run = await collector.Run(front, back, cancellationToken); } while (run); Thread.Sleep(interval * 1000); } }
public PartitionedRegJob(Config config, Storage storage, StorageFactory factory, CollectorHttpClient client) : base(config, storage, "partitionedreg") { _factory = factory; _client = client; }
static async Task Loop(string gallery, StorageFactory storageFactory, bool verbose, int interval, DateTime? startDate, CancellationToken cancellationToken) { Storage storage = storageFactory.Create(); const string LastCreated = "nuget:lastCreated"; const string LastEdited = "nuget:lastEdited"; int top = 20; int timeout = 300; while (true) { Func<HttpMessageHandler> handlerFunc = CommandHelpers.GetHttpMessageHandlerFactory(verbose); HttpMessageHandler handler = (handlerFunc != null) ? handlerFunc() : new WebRequestHandler { AllowPipelining = true }; using (HttpClient client = new HttpClient(handler)) { client.Timeout = TimeSpan.FromSeconds(timeout); // fetch and add all newly CREATED packages - in order DateTime lastCreated = await GetCatalogProperty(storage, LastCreated, cancellationToken) ?? (startDate ?? DateTime.MinValue.ToUniversalTime()); DateTime lastEdited = await GetCatalogProperty(storage, LastEdited, cancellationToken) ?? lastCreated; SortedList<DateTime, IList<Tuple<Uri, FeedDetails>>> createdPackages; DateTime previousLastCreated = DateTime.MinValue; do { Trace.TraceInformation("CATALOG LastCreated: {0}", lastCreated.ToString("O")); createdPackages = await GetCreatedPackages(client, gallery, lastCreated, top); Trace.TraceInformation("FEED CreatedPackages: {0}", createdPackages.Count); lastCreated = await DownloadMetadata2Catalog(client, createdPackages, storage, lastCreated, lastEdited, true, cancellationToken); if (previousLastCreated == lastCreated) { break; } previousLastCreated = lastCreated; } while (createdPackages.Count > 0); // THEN fetch and add all EDITED packages - in order SortedList<DateTime, IList<Tuple<Uri, FeedDetails>>> editedPackages; DateTime previousLastEdited = DateTime.MinValue; do { Trace.TraceInformation("CATALOG LastEdited: {0}", lastEdited.ToString("O")); editedPackages = await GetEditedPackages(client, gallery, lastEdited, top); Trace.TraceInformation("FEED EditedPackages: {0}", editedPackages.Count); lastEdited = await DownloadMetadata2Catalog(client, editedPackages, storage, lastCreated, lastEdited, false, cancellationToken); if (previousLastEdited == lastEdited) { break; } previousLastEdited = lastEdited; } while (editedPackages.Count > 0); } Thread.Sleep(interval * 1000); } }
static async Task ProcessPackages(string gallery, StorageFactory storageFactory, string id, string version, bool verbose, CancellationToken cancellationToken) { int timeout = 300; Func<HttpMessageHandler> handlerFunc = CommandHelpers.GetHttpMessageHandlerFactory(verbose); HttpMessageHandler handler = (handlerFunc != null) ? handlerFunc() : new WebRequestHandler { AllowPipelining = true }; using (HttpClient client = new HttpClient(handler)) { client.Timeout = TimeSpan.FromSeconds(timeout); // if teh version is specified a single package is processed otherwise all the packages corresponding to that id are processed Uri uri = (version == null) ? MakePackageUri(gallery, id) : MakePackageUri(gallery, id, version); SortedList<DateTime, IList<Tuple<Uri, FeedDetails>>> packages = await GetPackages(client, uri, "Created"); Trace.TraceInformation("downloading {0} packages", packages.Select(t => t.Value.Count).Sum()); Storage storage = storageFactory.Create(); // the idea here is to leave the lastCreated and lastEdited values exactly as they were const string LastCreated = "nuget:lastCreated"; const string LastEdited = "nuget:lastEdited"; DateTime lastCreated = await GetCatalogProperty(storage, LastCreated, cancellationToken) ?? DateTime.MinValue.ToUniversalTime(); DateTime lastEdited = await GetCatalogProperty(storage, LastEdited, cancellationToken) ?? DateTime.MinValue.ToUniversalTime(); DateTime d = await DownloadMetadata2Catalog(client, packages, storage, lastCreated, lastEdited, null, cancellationToken); } }
private async Task Loop(string gallery, StorageFactory catalogStorageFactory, StorageFactory auditingStorageFactory, bool verbose, int interval, DateTime? startDate, CancellationToken cancellationToken) { var catalogStorage = catalogStorageFactory.Create(); var auditingStorage = auditingStorageFactory.Create(); var top = 20; var timeout = TimeSpan.FromSeconds(300); while (true) { await ProcessFeed(gallery, catalogStorage, auditingStorage, startDate, timeout, top, verbose, cancellationToken); Thread.Sleep(interval * 1000); } }
public DnxCatalogCollector(Uri index, StorageFactory storageFactory, Func<HttpMessageHandler> handlerFunc = null) : base(index, handlerFunc) { _dnxMaker = new DnxMaker(storageFactory); }
private async Task ProcessPackages(string gallery, StorageFactory storageFactory, string id, string version, bool verbose, CancellationToken cancellationToken) { var timeout = TimeSpan.FromSeconds(300); using (var client = CreateHttpClient(verbose)) { client.Timeout = timeout; // if the version is specified a single package is processed otherwise all the packages corresponding to that id are processed var uri = (version == null) ? MakePackageUri(gallery, id) : MakePackageUri(gallery, id, version); var packages = await GetPackages(client, uri, "Created"); Trace.TraceInformation("downloading {0} packages", packages.Select(t => t.Value.Count).Sum()); var storage = storageFactory.Create(); // the idea here is to leave the lastCreated, lastEdited and lastDeleted values exactly as they were var lastCreated = await GetCatalogProperty(storage, "nuget:lastCreated", cancellationToken) ?? DateTime.MinValue.ToUniversalTime(); var lastEdited = await GetCatalogProperty(storage, "nuget:lastEdited", cancellationToken) ?? DateTime.MinValue.ToUniversalTime(); var lastDeleted = await GetCatalogProperty(storage, "nuget:lastDeleted", cancellationToken) ?? DateTime.MinValue.ToUniversalTime(); await DownloadMetadata2Catalog(client, packages, storage, lastCreated, lastEdited, lastDeleted, null, cancellationToken); } }
public ReindexCatalogCollector(Uri index, StorageFactory storageFactory, Func<HttpMessageHandler> handlerFunc = null) : base(index, handlerFunc) { _storageFactory = storageFactory; }
public AggregateStorageFactory(StorageFactory primaryStorageFactory, StorageFactory[] secondaryStorageFactories) : this(primaryStorageFactory, secondaryStorageFactories, null) { }
private static Task ProcessGraphsAsync(string packageId, IDictionary<string, IGraph> sortedGraphs, StorageFactory storageFactory, string contentBaseAddress) { return RegistrationMaker.Process(new RegistrationKey(packageId), sortedGraphs, storageFactory, new Uri(contentBaseAddress), 64, 128, CancellationToken.None); }
public BatchRegistrationCollector(Uri catalogUri, StorageFactory factory) : base(null, factory, null) { }