protected override void Init(IDictionary<string, string> arguments, CancellationToken cancellationToken)
{
_gallery = arguments.GetOrThrow<string>(Arguments.Gallery);
_verbose = arguments.GetOrDefault(Arguments.Verbose, false);
_id = arguments.GetOrThrow<string>(Arguments.Id);
_version = arguments.GetOrDefault<string>(Arguments.Version);
_storageFactory = CommandHelpers.CreateStorageFactory(arguments, _verbose);
}
public static async Task ProcessGraphs(
string id,
IDictionary<string, IGraph> sortedGraphs,
StorageFactory storageFactory,
Uri contentBaseAddress,
int partitionSize,
int packageCountThreshold,
CancellationToken cancellationToken)
{
int versionAlreadyExistsCount = 0;
existingVersionsWithID = new List<string>();
try
{
Storage storage = storageFactory.Create(id.ToLowerInvariant());
Uri resourceUri = storage.ResolveUri("index.json");
string json = await storage.LoadString(resourceUri, cancellationToken);
int count = Utils.CountItems(json);
//Determine if there are any versions that are existing already
CollectorHttpClient httpClient = new CollectorHttpClient();
foreach (var graph in sortedGraphs)
{
JObject jsonContent = await httpClient.GetJObjectAsync(new Uri(graph.Key), cancellationToken);
string existingId = jsonContent["@id"].ToString();
string existingVersionWithId = existingId.Substring(existingId.LastIndexOf("/") + 1);
string existingVersion = jsonContent["version"].ToString() + ".json";
//Determine if the version is actually available
//In Registration blobs, the format is /packageID/packageVersion.json
//So to check the existence of version we need to know only the version.json
if (storage.Exists(existingVersion))
{
//When we compare later in AddExistingItems, we need the "packageId.packageversion.json" for comparison so store it with Id
existingVersionsWithID.Add(existingVersionWithId);
versionAlreadyExistsCount++;
}
}
int total = count + sortedGraphs.Count - versionAlreadyExistsCount;
if (total < packageCountThreshold)
{
await SaveSmallRegistration(storage, storageFactory.BaseAddress, sortedGraphs, contentBaseAddress, partitionSize, cancellationToken);
}
else
{
await SaveLargeRegistration(storage, storageFactory.BaseAddress, sortedGraphs, json, contentBaseAddress, partitionSize, cancellationToken);
}
}
catch (Exception e)
{
throw new Exception(string.Format("Process id = {0}", id), e);
}
}
public DnxMaker(StorageFactory storageFactory)
{
if (storageFactory == null)
{
throw new ArgumentNullException(nameof(storageFactory));
}
_storageFactory = storageFactory;
}
public AggregateStorageFactory(StorageFactory primaryStorageFactory, StorageFactory[] secondaryStorageFactories,
AggregateStorage.WriteSecondaryStorageContentInterceptor writeSecondaryStorageContentInterceptor)
{
_primaryStorageFactory = primaryStorageFactory;
_secondaryStorageFactories = secondaryStorageFactories;
_writeSecondaryStorageContentInterceptor = writeSecondaryStorageContentInterceptor;
BaseAddress = _primaryStorageFactory.BaseAddress;
}
public RegistrationPersistence(StorageFactory storageFactory, RegistrationKey registrationKey, int partitionSize, int packageCountThreshold, Uri contentBaseAddress)
{
_storage = new RecordingStorage(storageFactory.Create(registrationKey.ToString()));
_registrationUri = _storage.ResolveUri("index.json");
_packageCountThreshold = packageCountThreshold;
_partitionSize = partitionSize;
_registrationBaseAddress = storageFactory.BaseAddress;
_contentBaseAddress = contentBaseAddress;
}
public RegistrationCatalogCollector(Uri index, StorageFactory storageFactory, Func<HttpMessageHandler> handlerFunc = null)
: base(index, new Uri[] { Schema.DataTypes.PackageDetails }, handlerFunc)
{
_storageFactory = storageFactory;
ContentBaseAddress = new Uri("http://tempuri.org");
PartitionSize = 64;
PackageCountThreshold = 128;
}
public static async Task Process(RegistrationKey registrationKey, IDictionary<string, IGraph> newItems, StorageFactory storageFactory, Uri contentBaseAddress, int partitionSize, int packageCountThreshold, CancellationToken cancellationToken)
{
Trace.TraceInformation("RegistrationMaker.Process: registrationKey = {0} newItems: {1}", registrationKey, newItems.Count);
IRegistrationPersistence registration = new RegistrationPersistence(storageFactory, registrationKey, partitionSize, packageCountThreshold, contentBaseAddress);
IDictionary<RegistrationEntryKey, RegistrationCatalogEntry> existing = await registration.Load(cancellationToken);
Trace.TraceInformation("RegistrationMaker.Process: existing = {0}", existing.Count);
IDictionary<RegistrationEntryKey, RegistrationCatalogEntry> delta = PromoteRegistrationKey(newItems);
Trace.TraceInformation("RegistrationMaker.Process: delta = {0}", delta.Count);
IDictionary<RegistrationEntryKey, RegistrationCatalogEntry> resulting = Apply(existing, delta);
Trace.TraceInformation("RegistrationMaker.Process: resulting = {0}", resulting.Count);
await registration.Save(resulting, cancellationToken);
}
public async Task Loop(string source, StorageFactory storageFactory, string contentBaseAddress, bool unlistShouldDelete, bool verbose, int interval, CancellationToken cancellationToken)
{
CommitCollector collector = new RegistrationCollector(new Uri(source), storageFactory, CommandHelpers.GetHttpMessageHandlerFactory(verbose))
{
ContentBaseAddress = contentBaseAddress == null ? null : new Uri(contentBaseAddress),
UnlistShouldDelete = unlistShouldDelete
};
Storage storage = storageFactory.Create();
ReadWriteCursor front = new DurableCursor(storage.ResolveUri("cursor.json"), storage, MemoryCursor.Min.Value);
ReadCursor back = MemoryCursor.Max;
while (true)
{
bool run = false;
do
{
run = await collector.Run(front, back, cancellationToken);
}
while (run);
Thread.Sleep(interval * 1000);
}
}
public PartitionedRegJob(Config config, Storage storage, StorageFactory factory, CollectorHttpClient client)
: base(config, storage, "partitionedreg")
{
_factory = factory;
_client = client;
}
static async Task Loop(string gallery, StorageFactory storageFactory, bool verbose, int interval, DateTime? startDate, CancellationToken cancellationToken)
{
Storage storage = storageFactory.Create();
const string LastCreated = "nuget:lastCreated";
const string LastEdited = "nuget:lastEdited";
int top = 20;
int timeout = 300;
while (true)
{
Func<HttpMessageHandler> handlerFunc = CommandHelpers.GetHttpMessageHandlerFactory(verbose);
HttpMessageHandler handler = (handlerFunc != null) ? handlerFunc() : new WebRequestHandler { AllowPipelining = true };
using (HttpClient client = new HttpClient(handler))
{
client.Timeout = TimeSpan.FromSeconds(timeout);
// fetch and add all newly CREATED packages - in order
DateTime lastCreated = await GetCatalogProperty(storage, LastCreated, cancellationToken) ?? (startDate ?? DateTime.MinValue.ToUniversalTime());
DateTime lastEdited = await GetCatalogProperty(storage, LastEdited, cancellationToken) ?? lastCreated;
SortedList<DateTime, IList<Tuple<Uri, FeedDetails>>> createdPackages;
DateTime previousLastCreated = DateTime.MinValue;
do
{
Trace.TraceInformation("CATALOG LastCreated: {0}", lastCreated.ToString("O"));
createdPackages = await GetCreatedPackages(client, gallery, lastCreated, top);
Trace.TraceInformation("FEED CreatedPackages: {0}", createdPackages.Count);
lastCreated = await DownloadMetadata2Catalog(client, createdPackages, storage, lastCreated, lastEdited, true, cancellationToken);
if (previousLastCreated == lastCreated)
{
break;
}
previousLastCreated = lastCreated;
}
while (createdPackages.Count > 0);
// THEN fetch and add all EDITED packages - in order
SortedList<DateTime, IList<Tuple<Uri, FeedDetails>>> editedPackages;
DateTime previousLastEdited = DateTime.MinValue;
do
{
Trace.TraceInformation("CATALOG LastEdited: {0}", lastEdited.ToString("O"));
editedPackages = await GetEditedPackages(client, gallery, lastEdited, top);
Trace.TraceInformation("FEED EditedPackages: {0}", editedPackages.Count);
lastEdited = await DownloadMetadata2Catalog(client, editedPackages, storage, lastCreated, lastEdited, false, cancellationToken);
if (previousLastEdited == lastEdited)
{
break;
}
previousLastEdited = lastEdited;
}
while (editedPackages.Count > 0);
}
Thread.Sleep(interval * 1000);
}
}
static async Task ProcessPackages(string gallery, StorageFactory storageFactory, string id, string version, bool verbose, CancellationToken cancellationToken)
{
int timeout = 300;
Func<HttpMessageHandler> handlerFunc = CommandHelpers.GetHttpMessageHandlerFactory(verbose);
HttpMessageHandler handler = (handlerFunc != null) ? handlerFunc() : new WebRequestHandler { AllowPipelining = true };
using (HttpClient client = new HttpClient(handler))
{
client.Timeout = TimeSpan.FromSeconds(timeout);
// if teh version is specified a single package is processed otherwise all the packages corresponding to that id are processed
Uri uri = (version == null) ? MakePackageUri(gallery, id) : MakePackageUri(gallery, id, version);
SortedList<DateTime, IList<Tuple<Uri, FeedDetails>>> packages = await GetPackages(client, uri, "Created");
Trace.TraceInformation("downloading {0} packages", packages.Select(t => t.Value.Count).Sum());
Storage storage = storageFactory.Create();
// the idea here is to leave the lastCreated and lastEdited values exactly as they were
const string LastCreated = "nuget:lastCreated";
const string LastEdited = "nuget:lastEdited";
DateTime lastCreated = await GetCatalogProperty(storage, LastCreated, cancellationToken) ?? DateTime.MinValue.ToUniversalTime();
DateTime lastEdited = await GetCatalogProperty(storage, LastEdited, cancellationToken) ?? DateTime.MinValue.ToUniversalTime();
DateTime d = await DownloadMetadata2Catalog(client, packages, storage, lastCreated, lastEdited, null, cancellationToken);
}
}
private async Task Loop(string gallery, StorageFactory catalogStorageFactory, StorageFactory auditingStorageFactory, bool verbose, int interval, DateTime? startDate, CancellationToken cancellationToken)
{
var catalogStorage = catalogStorageFactory.Create();
var auditingStorage = auditingStorageFactory.Create();
var top = 20;
var timeout = TimeSpan.FromSeconds(300);
while (true)
{
await ProcessFeed(gallery, catalogStorage, auditingStorage, startDate, timeout, top, verbose, cancellationToken);
Thread.Sleep(interval * 1000);
}
}
public DnxCatalogCollector(Uri index, StorageFactory storageFactory, Func<HttpMessageHandler> handlerFunc = null)
: base(index, handlerFunc)
{
_dnxMaker = new DnxMaker(storageFactory);
}
private async Task ProcessPackages(string gallery, StorageFactory storageFactory, string id, string version, bool verbose, CancellationToken cancellationToken)
{
var timeout = TimeSpan.FromSeconds(300);
using (var client = CreateHttpClient(verbose))
{
client.Timeout = timeout;
// if the version is specified a single package is processed otherwise all the packages corresponding to that id are processed
var uri = (version == null) ? MakePackageUri(gallery, id) : MakePackageUri(gallery, id, version);
var packages = await GetPackages(client, uri, "Created");
Trace.TraceInformation("downloading {0} packages", packages.Select(t => t.Value.Count).Sum());
var storage = storageFactory.Create();
// the idea here is to leave the lastCreated, lastEdited and lastDeleted values exactly as they were
var lastCreated = await GetCatalogProperty(storage, "nuget:lastCreated", cancellationToken) ?? DateTime.MinValue.ToUniversalTime();
var lastEdited = await GetCatalogProperty(storage, "nuget:lastEdited", cancellationToken) ?? DateTime.MinValue.ToUniversalTime();
var lastDeleted = await GetCatalogProperty(storage, "nuget:lastDeleted", cancellationToken) ?? DateTime.MinValue.ToUniversalTime();
await DownloadMetadata2Catalog(client, packages, storage, lastCreated, lastEdited, lastDeleted, null, cancellationToken);
}
}
public ReindexCatalogCollector(Uri index, StorageFactory storageFactory, Func<HttpMessageHandler> handlerFunc = null)
: base(index, handlerFunc)
{
_storageFactory = storageFactory;
}
public AggregateStorageFactory(StorageFactory primaryStorageFactory, StorageFactory[] secondaryStorageFactories)
: this(primaryStorageFactory, secondaryStorageFactories, null)
{
}
private static Task ProcessGraphsAsync(string packageId, IDictionary<string, IGraph> sortedGraphs, StorageFactory storageFactory, string contentBaseAddress)
{
return RegistrationMaker.Process(new RegistrationKey(packageId), sortedGraphs, storageFactory, new Uri(contentBaseAddress), 64, 128, CancellationToken.None);
}
public BatchRegistrationCollector(Uri catalogUri, StorageFactory factory)
: base(null, factory, null)
{
}
