public static async Task Test0Async()
{
Storage storage = new FileStorage
{
Path = @"c:\data\site\test",
Container = "test",
BaseAddress = "http://localhost:8000/"
};
// first save the delete into the catalog
CatalogContext context = new CatalogContext();
using (CatalogWriter writer = new CatalogWriter(storage, context, 1000, true))
{
//writer.Add(new DeletePackageCatalogItem("Test.Metadata.Service", "1.0.0"));
writer.Add(new DeletePackageCatalogItem("Test.Metadata.Service", "2.0.0"));
//writer.Add(new DeletePackageCatalogItem("Test.Metadata.Service", "3.0.0"));
await writer.Commit(DateTime.Now);
}
// second perform that delete on the various feeds - in this case the default resolver feed
ResolverDeleteCollector collector = new ResolverDeleteCollector(storage);
await collector.Run(new Uri("http://localhost:8000/test/catalog/index.json"), DateTime.MinValue);
}
public static async Task Test0Async()
{
string nuspecs = @"c:\data\nuget\versions";
Storage storage = new FileStorage("http://*****:*****@"c:\data\site\full");
CatalogContext context = new CatalogContext();
CatalogWriter writer = new CatalogWriter(storage, context, 10);
const int BatchSize = 1;
int i = 0;
int commitCount = 0;
DirectoryInfo directoryInfo = new DirectoryInfo(nuspecs);
foreach (FileInfo fileInfo in directoryInfo.EnumerateFiles("*.xml"))
{
writer.Add(new NuspecPackageCatalogItem(fileInfo.FullName));
if (++i % BatchSize == 0)
{
await writer.Commit(DateTime.UtcNow);
Console.WriteLine("commit number {0}", commitCount++);
}
}
await writer.Commit(DateTime.UtcNow);
Console.WriteLine("commit number {0}", commitCount++);
}
public override string CreateContent(CatalogContext context)
{
string resourceUri = GetBaseAddress() + GetRelativeAddress();
JObject content = CreateContent(resourceUri, _export);
JObject frame = context.GetJsonLdContext("context.Package.json", GetItemType());
content.Add("@context", frame["@context"]);
return content.ToString();
}
public StorageContent CreateContent(CatalogContext context)
{
IGraph graph = new Graph();
INode rdfTypePredicate = graph.CreateUriNode(Schema.Predicates.Type);
INode timeStampPredicate = graph.CreateUriNode(Schema.Predicates.CatalogTimestamp);
INode commitIdPredicate = graph.CreateUriNode(Schema.Predicates.CatalogCommitId);
INode container = graph.CreateUriNode(_resourceUri);
graph.Assert(container, rdfTypePredicate, graph.CreateUriNode(GetContainerType()));
graph.Assert(container, timeStampPredicate, graph.CreateLiteralNode(_timeStamp.ToString("O"), Schema.DataTypes.DateTime));
graph.Assert(container, commitIdPredicate, graph.CreateLiteralNode(_commitId.ToString()));
if (_parent != null)
{
graph.Assert(container, graph.CreateUriNode(Schema.Predicates.Parent), graph.CreateUriNode(_parent));
}
AddCustomContent(container, graph);
INode itemPredicate = graph.CreateUriNode(Schema.Predicates.CatalogItem);
INode countPredicate = graph.CreateUriNode(Schema.Predicates.CatalogCount);
foreach (KeyValuePair<Uri, CatalogContainerItem> item in _items)
{
INode itemNode = graph.CreateUriNode(item.Key);
graph.Assert(container, itemPredicate, itemNode);
graph.Assert(itemNode, rdfTypePredicate, graph.CreateUriNode(item.Value.Type));
if (item.Value.PageContent != null)
{
graph.Merge(item.Value.PageContent);
}
graph.Assert(itemNode, timeStampPredicate, graph.CreateLiteralNode(item.Value.TimeStamp.ToString("O"), Schema.DataTypes.DateTime));
graph.Assert(itemNode, commitIdPredicate, graph.CreateLiteralNode(item.Value.CommitId.ToString()));
if (item.Value.Count != null)
{
graph.Assert(itemNode, countPredicate, graph.CreateLiteralNode(item.Value.Count.ToString(), Schema.DataTypes.Integer));
}
}
JObject frame = context.GetJsonLdContext("context.Container.json", GetContainerType());
// The below code could be used to compact data storage by using relative URIs.
//frame = (JObject)frame.DeepClone();
//frame["@context"]["@base"] = _resourceUri.ToString();
StorageContent content = new StringStorageContent(Utils.CreateJson(graph, frame), "application/json", "no-store");
return content;
}
public override string CreateContent(CatalogContext context)
{
XDocument original = GetNuspec();
XDocument nuspec = NormalizeNuspecNamespace(original, context.GetXslt("xslt.normalizeNuspecNamespace.xslt"));
IGraph graph = CreateNuspecGraph(nuspec, GetBaseAddress(), context.GetXslt("xslt.nuspec.xslt"));
JObject frame = context.GetJsonLdContext("context.Package.json", GetItemType());
string content = Utils.CreateJson(graph, frame);
return content;
}
public CatalogWriter(Storage storage, CatalogContext context, int maxPageSize = 1000, bool append = true)
{
Options.InternUris = false;
_storage = storage;
_context = context;
_append = append;
_batch = new List<CatalogItem>();
MaxPageSize = maxPageSize;
_first = true;
_open = true;
}
public string CreateContent(CatalogContext context)
{
IGraph graph = new Graph();
graph.NamespaceMap.AddNamespace("rdf", new Uri("http://www.w3.org/1999/02/22-rdf-syntax-ns#"));
graph.NamespaceMap.AddNamespace("catalog", new Uri("http://nuget.org/catalog#"));
INode rdfTypePredicate = graph.CreateUriNode("rdf:type");
INode timeStampPredicate = graph.CreateUriNode("catalog:timeStamp");
Uri dateTimeDatatype = new Uri("http://www.w3.org/2001/XMLSchema#dateTime");
INode container = graph.CreateUriNode(_resourceUri);
graph.Assert(container, rdfTypePredicate, graph.CreateUriNode(GetContainerType()));
graph.Assert(container, timeStampPredicate, graph.CreateLiteralNode(_timeStamp.ToString(), dateTimeDatatype));
if (_parent != null)
{
graph.Assert(container, graph.CreateUriNode("catalog:parent"), graph.CreateUriNode(_parent));
}
AddCustomContent(container, graph);
INode itemPredicate = graph.CreateUriNode("catalog:item");
INode countPredicate = graph.CreateUriNode("catalog:count");
foreach (KeyValuePair<Uri, Tuple<Uri, IGraph, DateTime, int?>> item in GetItems())
{
INode itemNode = graph.CreateUriNode(item.Key);
graph.Assert(container, itemPredicate, itemNode);
graph.Assert(itemNode, rdfTypePredicate, graph.CreateUriNode(item.Value.Item1));
if (item.Value.Item2 != null)
{
graph.Merge(item.Value.Item2);
}
graph.Assert(itemNode, timeStampPredicate, graph.CreateLiteralNode(item.Value.Item3.ToString(), dateTimeDatatype));
if (item.Value.Item4 != null)
{
Uri integerDatatype = new Uri("http://www.w3.org/2001/XMLSchema#integer");
graph.Assert(itemNode, countPredicate, graph.CreateLiteralNode(item.Value.Item4.ToString(), integerDatatype));
}
}
JObject frame = context.GetJsonLdContext("context.Container.json", GetContainerType());
string content = Utils.CreateJson(graph, frame);
return content;
}
public override string CreateContent(CatalogContext context)
{
string id = GetBaseAddress() + _name + ".json";
JObject obj = new JObject
{
{ "name", _name },
{ "@id", id },
{ "@type", _type },
{ "@context", new JObject { { "@vocab", "http://test.org/schema#" } } }
};
return obj.ToString();
}
public static async Task Test1Async()
{
string nuspecs = @"c:\data\nuget\nuspecs";
Storage storage = new FileStorage("http://*****:*****@"c:\data\site\full");
CatalogContext context = new CatalogContext();
CatalogWriter writer = new CatalogWriter(storage, context, 200);
int total = 0;
//int[] commitSize = { 50, 40, 25, 50, 10, 30, 40, 5, 400, 30, 10, 20, 40, 50, 90, 70, 50, 50, 50, 50, 60, 70 };
int[] commitSize = {
200, 200, 200, 200, 200,
//200, 200, 200, 200, 200,
//200, 200, 200, 200, 200,
//200, 200, 200, 200, 200,
//200, 200, 200, 200, 200,
//200, 200, 200, 200, 200
};
int i = 0;
int commitCount = 0;
DirectoryInfo directoryInfo = new DirectoryInfo(nuspecs);
foreach (FileInfo fileInfo in directoryInfo.EnumerateFiles("*.xml"))
{
if (commitCount == commitSize.Length)
{
break;
}
writer.Add(new NuspecPackageCatalogItem(fileInfo.FullName));
total++;
if (++i == commitSize[commitCount])
{
await writer.Commit(DateTime.UtcNow);
Console.WriteLine("commit number {0}", commitCount);
commitCount++;
i = 0;
}
}
Console.WriteLine("total: {0}", total);
}
public override IGraph CreatePageContent(CatalogContext context)
{
Uri resourceUri = new Uri(GetBaseAddress() + GetRelativeAddress());
Graph graph = new Graph();
INode subject = graph.CreateUriNode(resourceUri);
INode galleryKeyPredicate = graph.CreateUriNode(new Uri("http://nuget.org/gallery#key"));
string key = _export.Package["Key"].ToString();
Uri integerDatatype = new Uri("http://www.w3.org/2001/XMLSchema#integer");
graph.Assert(subject, galleryKeyPredicate, graph.CreateLiteralNode(key, integerDatatype));
return graph;
}
public static async Task Test0Async()
{
string nuspecs = @"c:\data\nuspecs";
Storage storage = new FileStorage
{
Path = @"c:\data\site\pub",
Container = "pub",
BaseAddress = "http://localhost:8000/"
};
//Storage storage = new AzureStorage
//{
// AccountName = "nuget3",
// AccountKey = "",
// Container = "pub",
// BaseAddress = "http://nuget3.blob.core.windows.net"
//};
CatalogContext context = new CatalogContext();
CatalogWriter writer = new CatalogWriter(storage, context, 1000);
const int BatchSize = 1000;
int i = 0;
int commitCount = 0;
DirectoryInfo directoryInfo = new DirectoryInfo(nuspecs);
foreach (FileInfo fileInfo in directoryInfo.EnumerateFiles("*.xml"))
{
writer.Add(new NuspecPackageCatalogItem(fileInfo));
if (++i % BatchSize == 0)
{
await writer.Commit(DateTime.Now);
Console.WriteLine("commit number {0}", commitCount++);
}
}
await writer.Commit(DateTime.Now);
Console.WriteLine("commit number {0}", commitCount++);
}
public override string CreateContent(CatalogContext context)
{
IGraph graph = new Graph();
graph.NamespaceMap.AddNamespace("rdf", new Uri("http://www.w3.org/1999/02/22-rdf-syntax-ns#"));
graph.NamespaceMap.AddNamespace("nuget", new Uri("http://nuget.org/schema#"));
INode subject = graph.CreateUriNode(new Uri(GetBaseAddress() + GetItemIdentity()));
graph.Assert(subject, graph.CreateUriNode("rdf:type"), graph.CreateUriNode("nuget:DeleteRegistration"));
graph.Assert(subject, graph.CreateUriNode("nuget:id"), graph.CreateLiteralNode(_id));
JObject frame = context.GetJsonLdContext("context.DeletePackage.json", GetItemType());
string content = Utils.CreateJson(graph, frame);
return content;
}
public static async Task Test1Async()
{
Storage storage = new FileStorage
{
Path = @"c:\data\site\full",
Container = "full",
BaseAddress = "http://localhost:8000/"
};
// first save the delete into the catalog
CatalogContext context = new CatalogContext();
using (CatalogWriter writer = new CatalogWriter(storage, context, 1000, true))
{
writer.Add(new DeleteRegistrationCatalogItem("abc"));
await writer.Commit(DateTime.Now);
}
// second perform that delete on the various feeds - in this case the default resolver feed
ResolverDeleteCollector collector = new ResolverDeleteCollector(storage);
await collector.Run(new Uri("http://localhost:8000/full/catalog/index.json"), DateTime.MinValue);
}
public abstract StorageContent CreateContent(CatalogContext context);
public abstract string CreateContent(CatalogContext context);
public virtual IGraph CreatePageContent(CatalogContext context)
{
return null;
}
public static async Task Test3Async()
{
Storage storage = new FileStorage
{
Path = @"c:\data\site\test",
Container = "test",
BaseAddress = "http://localhost:8000/"
};
CatalogContext context = new CatalogContext();
CatalogWriter writer = new CatalogWriter(storage, context, 1000);
string[] names1 = { "a", "b", "c", "d", "e" };
string[] names2 = { "f", "g", "h" };
string[] names3 = { "i", "j", "k" };
foreach (string name in names1)
{
writer.Add(new TestCatalogItem(name));
}
await writer.Commit(new Dictionary<string, string> { { "prop1", "value1.1" }, { "prop2", "value2.1" } });
Console.WriteLine("commit user data #1");
foreach (KeyValuePair<string, string> items in await CatalogWriter.GetCommitUserData(storage))
{
Console.WriteLine("{0} {1}", items.Key, items.Value);
}
foreach (string name in names2)
{
writer.Add(new TestCatalogItem(name));
}
await writer.Commit(new Dictionary<string, string> { { "prop1", "value1.2" }, { "prop2", "value2.2" } });
Console.WriteLine("commit user data #2");
foreach (KeyValuePair<string, string> items in await CatalogWriter.GetCommitUserData(storage))
{
Console.WriteLine("{0} {1}", items.Key, items.Value);
}
foreach (string name in names3)
{
writer.Add(new TestCatalogItem(name));
}
await writer.Commit(new Dictionary<string, string> { { "prop1", "value1.3" }, { "prop2", "value2.3" } });
Console.WriteLine("commit user data #3");
foreach (KeyValuePair<string, string> items in await CatalogWriter.GetCommitUserData(storage))
{
Console.WriteLine("{0} {1}", items.Key, items.Value);
}
}
public static async Task Test4Async()
{
Storage storage = new FileStorage
{
Path = @"c:\data\site\test",
Container = "test",
BaseAddress = "http://localhost:8000/"
};
CatalogContext context = new CatalogContext();
CatalogWriter writer = new CatalogWriter(storage, context, 4);
string[] names1 = { "a", "b", "c", "d", "e" };
string[] names2 = { "f", "g", "h" };
string[] names3 = { "i", "j", "k" };
DateTime timeStamp = DateTime.UtcNow;
foreach (string name in names1)
{
writer.Add(new TestCatalogItem(name));
}
await writer.Commit(timeStamp);
Console.WriteLine("commit #1 timeStamp {0}", await CatalogWriter.GetLastCommitTimeStamp(storage));
Console.WriteLine("commit #1 count {0}", await CatalogWriter.GetCount(storage));
timeStamp = timeStamp.AddHours(1);
foreach (string name in names2)
{
writer.Add(new TestCatalogItem(name));
}
await writer.Commit(timeStamp);
Console.WriteLine("commit #2 timeStamp {0}", await CatalogWriter.GetLastCommitTimeStamp(storage));
Console.WriteLine("commit #2 count {0}", await CatalogWriter.GetCount(storage));
timeStamp = timeStamp.AddHours(1);
foreach (string name in names3)
{
writer.Add(new TestCatalogItem(name));
}
await writer.Commit(timeStamp);
Console.WriteLine("commit #3 timeStamp {0}", await CatalogWriter.GetLastCommitTimeStamp(storage));
Console.WriteLine("commit #3 count {0}", await CatalogWriter.GetCount(storage));
}
