// Save implementation
private static async Task Save(
IStorage storage,
RegistrationMakerCatalogItem.PostProcessGraph preprocessGraph,
Uri registrationBaseAddress,
IDictionary <RegistrationEntryKey, RegistrationCatalogEntry> registration,
int partitionSize,
int packageCountThreshold,
Uri contentBaseAddress,
Uri galleryBaseAddress,
bool forcePackagePathProviderForIcons,
CancellationToken cancellationToken)
{
Trace.TraceInformation("RegistrationPersistence.Save");
var items = registration.Values.Where(v => v != null).ToList();
if (items.Count == 0)
{
return;
}
if (items.Count < packageCountThreshold)
{
await SaveSmallRegistration(storage, preprocessGraph, registrationBaseAddress, items, partitionSize, contentBaseAddress, galleryBaseAddress, forcePackagePathProviderForIcons, cancellationToken);
}
else
{
await SaveLargeRegistration(storage, preprocessGraph, registrationBaseAddress, items, partitionSize, contentBaseAddress, galleryBaseAddress, forcePackagePathProviderForIcons, cancellationToken);
}
}
protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken)
{
PrintLightning();
// Hard code against Azure storage.
arguments[Arguments.StorageType] = Arguments.AzureStorageType;
// Configure the package path provider.
var useFlatContainerAsPackageContent = arguments.GetOrDefault <bool>(Arguments.ContentIsFlatContainer, false);
if (!useFlatContainerAsPackageContent)
{
RegistrationMakerCatalogItem.PackagePathProvider = new PackagesFolderPackagePathProvider();
}
else
{
var flatContainerName = arguments.GetOrThrow <string>(Arguments.FlatContainerName);
RegistrationMakerCatalogItem.PackagePathProvider = new FlatContainerPackagePathProvider(flatContainerName);
}
_command = arguments.GetOrThrow <string>(Arguments.Command);
_verbose = arguments.GetOrDefault(Arguments.Verbose, false);
_log = _verbose ? Console.Out : new StringWriter();
_contentBaseAddress = arguments.GetOrThrow <string>(Arguments.ContentBaseAddress);
_galleryBaseAddress = arguments.GetOrThrow <string>(Arguments.GalleryBaseAddress);
_storageFactories = CommandHelpers.CreateRegistrationStorageFactories(arguments, _verbose);
_shouldIncludeSemVer2ForLegacyStorageFactory = RegistrationCollector.GetShouldIncludeRegistrationPackageForLegacyStorageFactory(_storageFactories.SemVer2StorageFactory);
_postProcessGraphForLegacyStorageFactory = RegistrationCollector.GetPostProcessGraphForLegacyStorageFactory(_storageFactories.SemVer2StorageFactory);
_forceIconsFromFlatContainer = arguments.GetOrDefault <bool>(Arguments.AllIconsInFlatContainer);
_driver = arguments.GetOrDefault(Arguments.Driver, GraphDriver).ToLowerInvariant();
// We save the arguments because the "prepare" command generates "strike" commands. Some of the arguments
// used by "prepare" should be used when executing "strike".
_arguments = arguments;
if (_driver != GraphDriver && _driver != JsonDriver)
{
throw new NotSupportedException($"The lightning driver '{_driver}' is not supported.");
}
switch (_command.ToLowerInvariant())
{
case "charge":
case "prepare":
InitPrepare(arguments);
break;
case "strike":
InitStrike(arguments);
break;
default:
throw new NotSupportedException($"The lightning command '{_command}' is not supported.");
}
}
private static async Task SaveLargeRegistration(
IStorage storage,
RegistrationMakerCatalogItem.PostProcessGraph preprocessGraph,
Uri registrationBaseAddress,
IList <RegistrationCatalogEntry> items,
int partitionSize,
Uri contentBaseAddress,
Uri galleryBaseAddress,
bool forcePackagePathProviderForIcons,
CancellationToken cancellationToken)
{
Trace.TraceInformation("RegistrationPersistence.SaveLargeRegistration: registrationBaseAddress = {0} items: {1}", registrationBaseAddress, items.Count);
IList <Uri> cleanUpList = new List <Uri>();
await SaveRegistration(storage, preprocessGraph, registrationBaseAddress, items, cleanUpList, null, partitionSize, contentBaseAddress, galleryBaseAddress, forcePackagePathProviderForIcons, cancellationToken);
}
public RegistrationPersistence(
StorageFactory storageFactory,
RegistrationMakerCatalogItem.PostProcessGraph postProcessGraph,
RegistrationKey registrationKey,
int partitionSize,
int packageCountThreshold,
Uri contentBaseAddress,
Uri galleryBaseAddress,
bool forcePackagePathProviderForIcons)
{
_storage = new RecordingStorage(storageFactory.Create(registrationKey.ToString()));
_postProcessGraph = postProcessGraph;
_registrationUri = _storage.ResolveUri("index.json");
_packageCountThreshold = packageCountThreshold;
_partitionSize = partitionSize;
_registrationBaseAddress = storageFactory.BaseAddress;
_contentBaseAddress = contentBaseAddress;
_galleryBaseAddress = galleryBaseAddress;
_forcePackagePathProviderForIcons = forcePackagePathProviderForIcons;
}
public RegistrationCollector(
Uri index,
StorageFactory legacyStorageFactory,
StorageFactory semVer2StorageFactory,
Uri contentBaseAddress,
Uri galleryBaseAddress,
bool forcePackagePathProviderForIcons,
ITelemetryService telemetryService,
ILogger logger,
Func <HttpMessageHandler> handlerFunc = null,
IHttpRetryStrategy httpRetryStrategy = null,
int maxConcurrentBatches = DefaultMaxConcurrentBatches)
: base(
index,
new Uri[] { Schema.DataTypes.PackageDetails, Schema.DataTypes.PackageDelete },
telemetryService,
handlerFunc,
httpRetryStrategy)
{
_legacyStorageFactory = legacyStorageFactory ?? throw new ArgumentNullException(nameof(legacyStorageFactory));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_semVer2StorageFactory = semVer2StorageFactory;
_shouldIncludeSemVer2ForLegacyStorageFactory = GetShouldIncludeRegistrationPackageForLegacyStorageFactory(_semVer2StorageFactory);
_postProcessGraphForLegacyStorageFactory = GetPostProcessGraphForLegacyStorageFactory(_semVer2StorageFactory);
ContentBaseAddress = contentBaseAddress;
GalleryBaseAddress = galleryBaseAddress;
_forcePackagePathProviderForIcons = forcePackagePathProviderForIcons;
if (maxConcurrentBatches < 1)
{
throw new ArgumentOutOfRangeException(
nameof(maxConcurrentBatches),
maxConcurrentBatches,
string.Format(Strings.ArgumentOutOfRange, 1, int.MaxValue));
}
_maxConcurrentBatches = maxConcurrentBatches;
}
public static async Task ProcessAsync(
RegistrationKey registrationKey,
IReadOnlyDictionary <string, IGraph> newItems,
ShouldIncludeRegistrationPackage shouldInclude,
StorageFactory storageFactory,
RegistrationMakerCatalogItem.PostProcessGraph postProcessGraph,
Uri contentBaseAddress,
Uri galleryBaseAddress,
int partitionSize,
int packageCountThreshold,
bool forcePackagePathProviderForIcons,
ITelemetryService telemetryService,
CancellationToken cancellationToken)
{
Trace.TraceInformation("RegistrationMaker.Process: registrationKey = {0} newItems: {1}", registrationKey, newItems.Count);
IRegistrationPersistence registration = new RegistrationPersistence(storageFactory, postProcessGraph, registrationKey, partitionSize, packageCountThreshold, contentBaseAddress, galleryBaseAddress, forcePackagePathProviderForIcons);
IDictionary <RegistrationEntryKey, RegistrationCatalogEntry> existing = await registration.Load(cancellationToken);
Trace.TraceInformation("RegistrationMaker.Process: existing = {0}", existing.Count);
IDictionary <RegistrationEntryKey, RegistrationCatalogEntry> delta = PromoteRegistrationKey(newItems, shouldInclude);
Trace.TraceInformation("RegistrationMaker.Process: delta = {0}", delta.Count);
telemetryService.TrackMetric(TelemetryConstants.RegistrationDeltaCount, (uint)delta.Count, new Dictionary <string, string>()
{
{ TelemetryConstants.ContentBaseAddress, contentBaseAddress.AbsoluteUri },
{ TelemetryConstants.GalleryBaseAddress, galleryBaseAddress.AbsoluteUri }
});
IDictionary <RegistrationEntryKey, RegistrationCatalogEntry> resulting = Apply(existing, delta);
Trace.TraceInformation("RegistrationMaker.Process: resulting = {0}", resulting.Count);
await registration.Save(resulting, cancellationToken);
}
private static async Task SaveRegistration(
IStorage storage,
RegistrationMakerCatalogItem.PostProcessGraph postProcessGraph,
Uri registrationBaseAddress,
IList <RegistrationCatalogEntry> items,
IList <Uri> cleanUpList,
SingleGraphPersistence graphPersistence,
int partitionSize,
Uri contentBaseAddress,
Uri galleryBaseAddress,
bool forcePackagePathProviderForIcons,
CancellationToken cancellationToken)
{
Trace.TraceInformation("RegistrationPersistence.SaveRegistration: registrationBaseAddress = {0} items: {1}", registrationBaseAddress, items.Count);
using (RegistrationMakerCatalogWriter writer = new RegistrationMakerCatalogWriter(storage, partitionSize, cleanUpList, graphPersistence))
{
foreach (var item in items)
{
writer.Add(new RegistrationMakerCatalogItem(new Uri(item.ResourceUri), item.Graph, registrationBaseAddress, item.IsExistingItem, postProcessGraph, forcePackagePathProviderForIcons, contentBaseAddress, galleryBaseAddress));
}
await writer.Commit(DateTime.UtcNow, null, cancellationToken);
}
}
private static async Task SaveSmallRegistration(
IStorage storage,
RegistrationMakerCatalogItem.PostProcessGraph preprocessGraph,
Uri registrationBaseAddress,
IList <RegistrationCatalogEntry> items,
int partitionSize, Uri contentBaseAddress,
Uri galleryBaseAddress,
bool forcePackagePathProviderForIcons,
CancellationToken cancellationToken)
{
Trace.TraceInformation("RegistrationPersistence.SaveSmallRegistration");
SingleGraphPersistence graphPersistence = new SingleGraphPersistence(storage);
//await graphPersistence.Initialize();
await SaveRegistration(storage, preprocessGraph, registrationBaseAddress, items, null, graphPersistence, partitionSize, contentBaseAddress, galleryBaseAddress, forcePackagePathProviderForIcons, cancellationToken);
// now the commit has happened the graphPersistence.Graph should contain all the data
JObject frame = (new CatalogContext()).GetJsonLdContext("context.Registration.json", graphPersistence.TypeUri);
StorageContent content = new JTokenStorageContent(Utils.CreateJson(graphPersistence.Graph, frame), "application/json", "no-store");
await storage.SaveAsync(graphPersistence.ResourceUri, content, cancellationToken);
}