public static async Task Test0Async()
{
Storage storage = new FileStorage
{
Path = @"c:\data\site\test",
Container = "test",
BaseAddress = "http://localhost:8000/"
};
// first save the delete into the catalog
CatalogContext context = new CatalogContext();
using (CatalogWriter writer = new CatalogWriter(storage, context, 1000, true))
{
//writer.Add(new DeletePackageCatalogItem("Test.Metadata.Service", "1.0.0"));
writer.Add(new DeletePackageCatalogItem("Test.Metadata.Service", "2.0.0"));
//writer.Add(new DeletePackageCatalogItem("Test.Metadata.Service", "3.0.0"));
await writer.Commit(DateTime.Now);
}
// second perform that delete on the various feeds - in this case the default resolver feed
ResolverDeleteCollector collector = new ResolverDeleteCollector(storage);
await collector.Run(new Uri("http://localhost:8000/test/catalog/index.json"), DateTime.MinValue);
}
public static async Task Test0Async()
{
//string nuspecs = @"c:\data\nuget\nuspecs";
string nuspecs = @"c:\data\nuget\nuspecs";
//Storage storage = new FileStorage("http://*****:*****@"c:\data\site\full");
//Storage storage = new FileStorage("http://*****:*****@"c:\data\site\dotnetrdf");
Storage storage = new FileStorage("http://*****:*****@"c:\data\site\ordered");
AppendOnlyCatalogWriter writer = new AppendOnlyCatalogWriter(storage, 15);
const int BatchSize = 10;
int i = 0;
int commitCount = 0;
DirectoryInfo directoryInfo = new DirectoryInfo(nuspecs);
//foreach (FileInfo fileInfo in directoryInfo.EnumerateFiles("*.xml"))
//foreach (FileInfo fileInfo in directoryInfo.EnumerateFiles("dotnetrdf.*.xml"))
foreach (FileInfo fileInfo in directoryInfo.EnumerateFiles("entityframework.*.xml"))
{
writer.Add(new NuspecPackageCatalogItem(fileInfo.FullName));
if (++i % BatchSize == 0)
{
await writer.Commit(DateTime.UtcNow, null, CancellationToken.None);
Console.WriteLine("commit number {0}", commitCount++);
}
}
await writer.Commit(DateTime.UtcNow, null, CancellationToken.None);
Console.WriteLine("commit number {0}", commitCount++);
}
public static async Task Test0Async()
{
string nuspecs = @"c:\data\nuget\versions";
Storage storage = new FileStorage("http://*****:*****@"c:\data\site\full");
CatalogContext context = new CatalogContext();
CatalogWriter writer = new CatalogWriter(storage, context, 10);
const int BatchSize = 1;
int i = 0;
int commitCount = 0;
DirectoryInfo directoryInfo = new DirectoryInfo(nuspecs);
foreach (FileInfo fileInfo in directoryInfo.EnumerateFiles("*.xml"))
{
writer.Add(new NuspecPackageCatalogItem(fileInfo.FullName));
if (++i % BatchSize == 0)
{
await writer.Commit(DateTime.UtcNow);
Console.WriteLine("commit number {0}", commitCount++);
}
}
await writer.Commit(DateTime.UtcNow);
Console.WriteLine("commit number {0}", commitCount++);
}
public static async Task Test1Async()
{
string nuspecs = @"c:\data\nuget\nuspecs";
Storage storage = new FileStorage("http://*****:*****@"c:\data\site\full");
AppendOnlyCatalogWriter writer = new AppendOnlyCatalogWriter(storage, 20);
int total = 0;
//int[] commitSize = { 50, 40, 25, 50, 10, 30, 40, 5, 400, 30, 10, 20, 40, 50, 90, 70, 50, 50, 50, 50, 60, 70 };
int[] commitSize = {
20, 20, 20, 20, 20,
20, 20, 20, 20, 20,
//200, 200, 200, 200, 200,
//200, 200, 200, 200, 200,
//200, 200, 200, 200, 200,
//200, 200, 200, 200, 200,
//200, 200, 200, 200, 200
};
int i = 0;
int commitCount = 0;
DirectoryInfo directoryInfo = new DirectoryInfo(nuspecs);
foreach (FileInfo fileInfo in directoryInfo.EnumerateFiles("dotnetrdf.*.xml"))
{
if (commitCount == commitSize.Length)
{
break;
}
writer.Add(new NuspecPackageCatalogItem(fileInfo.FullName));
total++;
if (++i == commitSize[commitCount])
{
await writer.Commit(DateTime.UtcNow, null, CancellationToken.None);
Console.WriteLine("commit number {0}", commitCount);
commitCount++;
i = 0;
}
}
if (i > 0)
{
await writer.Commit(DateTime.UtcNow, null, CancellationToken.None);
}
Console.WriteLine("total: {0}", total);
}
public static void Test0()
{
Storage storage = new FileStorage("http://*****:*****@"c:\data\site\demo");
SqlConnectionStringBuilder connStrBldr = new SqlConnectionStringBuilder();
connStrBldr.IntegratedSecurity = true;
connStrBldr.InitialCatalog = "TestSourceWarehouse";
connStrBldr.DataSource = @"(LocalDB)\v11.0";
string connectionString = connStrBldr.ToString();
WarehouseHelper.CreateStatisticsCatalogAsync(storage, connectionString, CancellationToken.None).Wait();
}
public static void Test0()
{
const int SqlChunkSize = 8000;
string sqlConnectionString = "";
const int CatalogBatchSize = 1000;
const int CatalogMaxPageSize = 1000;
Storage storage = new FileStorage
{
Path = @"c:\data\site\export2",
Container = "export2",
BaseAddress = "http://localhost:8000/"
};
CatalogWriter writer = new CatalogWriter(storage, new CatalogContext(), CatalogMaxPageSize);
GalleryExportBatcher batcher = new GalleryExportBatcher(CatalogBatchSize, writer);
int lastHighestPackageKey = 0;
int count = 0;
while (true)
{
Tuple<int, int> range = GalleryExport.GetNextRange(sqlConnectionString, lastHighestPackageKey, SqlChunkSize);
if (range.Item1 == 0 && range.Item2 == 0)
{
break;
}
if (count++ == 3)
{
break;
}
Console.WriteLine("{0} {1}", range.Item1, range.Item2);
GalleryExport.FetchRange(sqlConnectionString, range, batcher);
lastHighestPackageKey = range.Item2;
}
batcher.Complete();
Console.WriteLine(batcher.Total);
}
public static async Task Test1Async()
{
string nuspecs = @"c:\data\nuspecs";
Storage storage = new FileStorage
{
Path = @"c:\data\site\full",
Container = "full",
BaseAddress = "http://localhost:8000/"
};
CatalogContext context = new CatalogContext();
CatalogWriter writer = new CatalogWriter(storage, context, 150);
int total = 0;
int[] commitSize = { 50, 40, 25, 50, 10, 30, 40, 5, 400, 30, 10 };
int i = 0;
int commitCount = 0;
DirectoryInfo directoryInfo = new DirectoryInfo(nuspecs);
foreach (FileInfo fileInfo in directoryInfo.EnumerateFiles("*.xml"))
{
if (commitCount == commitSize.Length)
{
break;
}
writer.Add(new NuspecPackageCatalogItem(fileInfo));
total++;
if (++i == commitSize[commitCount])
{
await writer.Commit(DateTime.Now);
Console.WriteLine("commit number {0}", commitCount);
commitCount++;
i = 0;
}
}
Console.WriteLine("total: {0}", total);
}
public static async Task Test0Async()
{
string nuspecs = @"c:\data\nuspecs";
Storage storage = new FileStorage
{
Path = @"c:\data\site\pub",
Container = "pub",
BaseAddress = "http://localhost:8000/"
};
//Storage storage = new AzureStorage
//{
// AccountName = "nuget3",
// AccountKey = "",
// Container = "pub",
// BaseAddress = "http://nuget3.blob.core.windows.net"
//};
CatalogContext context = new CatalogContext();
CatalogWriter writer = new CatalogWriter(storage, context, 1000);
const int BatchSize = 1000;
int i = 0;
int commitCount = 0;
DirectoryInfo directoryInfo = new DirectoryInfo(nuspecs);
foreach (FileInfo fileInfo in directoryInfo.EnumerateFiles("*.xml"))
{
writer.Add(new NuspecPackageCatalogItem(fileInfo));
if (++i % BatchSize == 0)
{
await writer.Commit(DateTime.Now);
Console.WriteLine("commit number {0}", commitCount++);
}
}
await writer.Commit(DateTime.Now);
Console.WriteLine("commit number {0}", commitCount++);
}
public static async Task Test0Async()
{
Storage storage = new FileStorage("http://*****:*****@"c:\data\site\resolver");
FileSystemEmulatorHandler handler = new VerboseHandler
{
BaseAddress = new Uri("http://*****:*****@"c:\data\site",
InnerHandler = new HttpClientHandler()
};
ResolverCollector collector = new ResolverCollector(storage, 200);
//await collector.Run(new Uri("http://localhost:8000/full/index.json"), DateTime.MinValue);
//await collector.Run(new Uri("http://partitions.blob.core.windows.net/partition0/index.json"), DateTime.MinValue);
await collector.Run(new Uri("http://localhost:8000/partition/partition0/index.json"), DateTime.MinValue, handler);
Console.WriteLine("http requests: {0} batch count: {1}", collector.RequestCount, collector.BatchCount);
}
public static async Task Test0Async()
{
//Storage storage = new AzureStorage
//{
// AccountName = "nuget3",
// AccountKey = "",
// Container = "feed",
// BaseAddress = "http://nuget3.blob.core.windows.net"
//};
Storage storage = new FileStorage
{
Path = @"c:\data\site\export2",
Container = "export2",
BaseAddress = "http://localhost:8000/"
};
ResolverCollector collector = new ResolverCollector(storage, 200);
await collector.Run(new Uri("http://localhost:8000/export/catalog/index.json"), DateTime.MinValue);
Console.WriteLine("http requests: {0} batch count: {1}", collector.RequestCount, collector.BatchCount);
}
public static async Task Test1Async()
{
Storage storage = new FileStorage
{
Path = @"c:\data\site\full",
Container = "full",
BaseAddress = "http://localhost:8000/"
};
// first save the delete into the catalog
CatalogContext context = new CatalogContext();
using (CatalogWriter writer = new CatalogWriter(storage, context, 1000, true))
{
writer.Add(new DeleteRegistrationCatalogItem("abc"));
await writer.Commit(DateTime.Now);
}
// second perform that delete on the various feeds - in this case the default resolver feed
ResolverDeleteCollector collector = new ResolverDeleteCollector(storage);
await collector.Run(new Uri("http://localhost:8000/full/catalog/index.json"), DateTime.MinValue);
}
public static async Task Test1Async()
{
Storage storage = new FileStorage("http://*****:*****@"c:\data\site\test");
//Storage storage = new AzureStorage
//{
// AccountName = "",
// AccountKey = "",
// Container = "test",
// BaseAddress = "http://nuget3.blob.core.windows.net"
//};
CatalogContext context = new CatalogContext();
CatalogWriter writer = new CatalogWriter(storage, context, 4, false);
string[] first = { "john", "paul", "ringo", "george" };
foreach (string item in first)
{
writer.Add(new TestCatalogItem(item));
}
await writer.Commit(new DateTime(2010, 12, 25, 12, 0, 0));
string baseAddress = storage.BaseAddress + "/";
Uri index = new Uri(baseAddress + "catalog/index.json");
TestItemCollector collector = new TestItemCollector();
CollectorCursor cursor = await collector.Run(index, DateTime.MinValue);
string[] second = { "jimmy", "robert", "john-paul", "john" };
foreach (string item in second)
{
writer.Add(new TestCatalogItem(item));
}
await writer.Commit(new DateTime(2011, 12, 25, 12, 0, 0));
cursor = await collector.Run(index, cursor);
}
public static async Task Test5Async()
{
Storage storage = new FileStorage("http://*****:*****@"c:\data\site\resolver");
TimeSpan prev = TimeSpan.MinValue;
Uri longest = null;
for (int i = 0; i < 100; i++)
{
Uri indexUri = new Uri(string.Format("http://localhost:8000/partition/partition{0}/index.json", i));
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
ResolverCollector collector = new ResolverCollector(storage, 200);
await collector.Run(indexUri, DateTime.MinValue);
Console.WriteLine("http requests: {0} batch count: {1}", collector.RequestCount, collector.BatchCount);
stopwatch.Stop();
TimeSpan current = stopwatch.Elapsed;
if (longest == null || current > prev)
{
longest = indexUri;
prev = current;
}
Console.WriteLine("{0} {1} seconds", indexUri, current.TotalSeconds);
}
Console.WriteLine("the winner is {0}", longest);
}
public static async Task Test3Async()
{
Func<HttpMessageHandler> handlerFunc = () =>
{
return new FileSystemEmulatorHandler
{
BaseAddress = new Uri("http://*****:*****@"c:\data\site",
InnerHandler = new HttpClientHandler()
};
};
Storage storage = new FileStorage("http://*****:*****@"c:\data\site\nuspec");
CommitCollector collector = new NuspecCollector(new Uri("http://localhost:8000/full/index.json"), storage, handlerFunc);
await collector.Run(CancellationToken.None);
Console.WriteLine("http requests: {0}", collector.RequestCount);
}
public static async Task Test3Async()
{
System.Net.ServicePointManager.DefaultConnectionLimit = 1024;
IDictionary<string, string> packageHashLookup = LoadPackageHashLookup();
HashSet<string> packageExceptionLookup = LoadPackageExceptionLookup();
string nupkgs = @"c:\data\nuget\gallery\";
Storage storage = new FileStorage("http://*****:*****@"c:\data\site\ordered");
//StorageCredentials credentials = new StorageCredentials("", "");
//CloudStorageAccount account = new CloudStorageAccount(credentials, true);
//string storageContainer = "test1";
//string storagePath = "";
//string storageBaseAddress = "http://nugetjohtaylo.blob.core.windows.net/test1";
//StorageFactory storageFactory = new AzureStorageFactory(account, storageContainer, storagePath, new Uri(storageBaseAddress)) { Verbose = true };
//Storage storage = storageFactory.Create();
AppendOnlyCatalogWriter writer = new AppendOnlyCatalogWriter(storage, 550);
const int BatchSize = 64;
int commitCount = 0;
IDictionary<string, DateTime> packageCreated = LoadPackageCreatedLookup();
DateTime lastCreated = (await PackageCatalog.ReadCommitMetadata(writer, CancellationToken.None)).Item1 ?? DateTime.MinValue;
ParallelOptions options = new ParallelOptions();
options.MaxDegreeOfParallelism = 8;
// filter by lastCreated here
Queue<KeyValuePair<string, DateTime>> packageCreatedQueue = new Queue<KeyValuePair<string, DateTime>>(packageCreated.Where(p => p.Value > lastCreated && !packageExceptionLookup.Contains(p.Key)).OrderBy(p => p.Value));
int completed = 0;
Stopwatch runtime = new Stopwatch();
runtime.Start();
Task commitTask = null;
var context = writer.Context;
Uri rootUri = writer.RootUri;
while (packageCreatedQueue.Count > 0)
{
List<KeyValuePair<string, DateTime>> batch = new List<KeyValuePair<string, DateTime>>();
ConcurrentBag<CatalogItem> batchItems = new ConcurrentBag<CatalogItem>();
while (batch.Count < BatchSize && packageCreatedQueue.Count > 0)
{
completed++;
var packagePair = packageCreatedQueue.Dequeue();
lastCreated = packagePair.Value;
batch.Add(packagePair);
}
var commitTime = DateTime.UtcNow;
Parallel.ForEach(batch, options, entry =>
{
FileInfo fileInfo = new FileInfo(nupkgs + entry.Key);
if (fileInfo.Exists)
{
using (Stream stream = new FileStream(fileInfo.FullName, FileMode.Open))
{
string packageHash = null;
packageHashLookup.TryGetValue(fileInfo.Name, out packageHash);
CatalogItem item = Utils.CreateCatalogItem(stream, entry.Value, packageHash, fileInfo.FullName);
batchItems.Add(item);
}
}
});
if (commitTask != null)
{
commitTask.Wait();
}
foreach (var item in batchItems)
{
writer.Add(item);
}
commitTask = Task.Run(async () => await writer.Commit(commitTime, PackageCatalog.CreateCommitMetadata(writer.RootUri, lastCreated, null), CancellationToken.None));
// stats
double perPackage = runtime.Elapsed.TotalSeconds / (double)completed;
DateTime finish = DateTime.Now.AddSeconds(perPackage * packageCreatedQueue.Count);
Console.WriteLine("commit number {0} Completed: {1} Remaining: {2} Estimated Finish: {3}",
commitCount++,
completed,
packageCreatedQueue.Count,
finish.ToString("O"));
}
// wait for the final commit
if (commitTask != null)
{
commitTask.Wait();
}
Console.WriteLine("Finished in: " + runtime.Elapsed);
}
static async Task MoreTestCatalog()
{
string baseAddress = "http://*****:*****@"c:\data\site\cursor";
Storage storage = new FileStorage(baseAddress, path);
AppendOnlyCatalogWriter writer = new AppendOnlyCatalogWriter(storage, 550);
writer.Add(new TestCatalogItem(8));
await writer.Commit(new DateTime(2014, 1, 11, 0, 0, 0, DateTimeKind.Utc), null, CancellationToken.None);
writer.Add(new TestCatalogItem(9));
await writer.Commit(new DateTime(2014, 1, 13, 0, 0, 0, DateTimeKind.Utc), null, CancellationToken.None);
writer.Add(new TestCatalogItem(10));
await writer.Commit(new DateTime(2014, 1, 14, 0, 0, 0, DateTimeKind.Utc), null, CancellationToken.None);
writer.Add(new TestCatalogItem(11));
await writer.Commit(new DateTime(2014, 1, 15, 0, 0, 0, DateTimeKind.Utc), null, CancellationToken.None);
writer.Add(new TestCatalogItem(12));
await writer.Commit(new DateTime(2014, 1, 17, 0, 0, 0, DateTimeKind.Utc), null, CancellationToken.None);
writer.Add(new TestCatalogItem(13));
await writer.Commit(new DateTime(2014, 1, 18, 0, 0, 0, DateTimeKind.Utc), null, CancellationToken.None);
writer.Add(new TestCatalogItem(14));
await writer.Commit(new DateTime(2014, 1, 20, 0, 0, 0, DateTimeKind.Utc), null, CancellationToken.None);
Console.WriteLine("test catalog created");
}
public static void Test0()
{
Uri indexUri = new Uri("http://nugetprod0.blob.core.windows.net/ng-catalogs/0/index.json");
IDictionary<string, IList<JObject>> packages = IndexingHelpers.GetPackages(indexUri, true).Result;
PrintMetrics(packages);
int partitionCount = 100;
int count = packages.Count;
int chunkSize = count / (partitionCount - 1);
IDictionary<string, IList<JObject>>[] partitions = new Dictionary<string, IList<JObject>>[partitionCount];
IDictionary<string, IList<JObject>> batch = new Dictionary<string, IList<JObject>>();
int batchNumber = 0;
foreach (KeyValuePair<string, IList<JObject>> package in packages)
{
batch.Add(package);
if (batch.Count == chunkSize)
{
partitions[batchNumber++] = new Dictionary<string, IList<JObject>>(batch);
batch.Clear();
}
}
if (batch.Count > 0)
{
partitions[batchNumber++] = new Dictionary<string, IList<JObject>>(batch);
}
//CloudStorageAccount account = new CloudStorageAccount(new StorageCredentials("partitions", ""), false);
IList<Task> createTasks = new List<Task>();
int partitionNumber = 0;
foreach (IDictionary<string, IList<JObject>> partition in partitions)
{
string name = string.Format("partition{0}", partitionNumber++);
Storage storage = new FileStorage("http://*****:*****@"c:\data\site\partition\" + name);
//Storage storage = new AzureStorage(account, name);
createTasks.Add(IndexingHelpers.CreateNewCatalog(storage, partition));
}
Task.WaitAll(createTasks.ToArray());
}
public void Rebuild(RebuildArgs args)
{
const int batchSize = 2000;
if (Directory.Exists(args.CatalogFolder))
{
Console.WriteLine("Catalog folder exists. Deleting!");
Directory.Delete(args.CatalogFolder, recursive: true);
}
// Load storage
Storage storage = new FileStorage(args.BaseAddress, args.CatalogFolder);
using (var writer = new CatalogWriter(storage, new CatalogContext()))
{
if (!String.IsNullOrEmpty(args.DatabaseConnection))
{
var batcher = new GalleryExportBatcher(batchSize, writer);
int lastHighest = 0;
while (true)
{
var range = GalleryExport.GetNextRange(
args.DatabaseConnection,
lastHighest,
batchSize).Result;
if (range.Item1 == 0 && range.Item2 == 0)
{
break;
}
Console.WriteLine("Writing packages with Keys {0}-{1} to catalog...", range.Item1, range.Item2);
GalleryExport.WriteRange(
args.DatabaseConnection,
range,
batcher).Wait();
lastHighest = range.Item2;
}
batcher.Complete().Wait();
}
else if (!String.IsNullOrEmpty(args.NuPkgFolder))
{
Stopwatch timer = new Stopwatch();
timer.Start();
// files are sorted by GetFiles
Queue<string> files = new Queue<string>(Directory.GetFiles(args.NuPkgFolder, "*.nu*", SearchOption.TopDirectoryOnly)
.Where(s => s.EndsWith(".nupkg", StringComparison.OrdinalIgnoreCase) || s.EndsWith(".nuspec", StringComparison.OrdinalIgnoreCase)));
int total = files.Count;
ParallelOptions options = new ParallelOptions();
options.MaxDegreeOfParallelism = 8;
Task commitTask = null;
while (files.Count > 0)
{
Queue<PackageCatalogItem> currentBatch = new Queue<PackageCatalogItem>(batchSize);
// create the batch
while (currentBatch.Count < batchSize && files.Count > 0)
{
string file = files.Dequeue();
if (file.EndsWith(".nupkg", StringComparison.OrdinalIgnoreCase))
{
currentBatch.Enqueue(new NupkgCatalogItem(file));
}
else
{
currentBatch.Enqueue(new NuspecPackageCatalogItem(file));
}
}
// process the nupkgs and nuspec files in parallel
Parallel.ForEach(currentBatch, options, nupkg =>
{
nupkg.Load();
});
// wait for the previous commit to finish before adding more
if (commitTask != null)
{
commitTask.Wait();
}
// add everything from the queue
foreach(PackageCatalogItem item in currentBatch)
{
writer.Add(item);
}
// commit
commitTask = Task.Run(async () => await writer.Commit(DateTime.UtcNow));
Console.WriteLine("committing {0}/{1}", total - files.Count, total);
}
// wait for the final commit
if (commitTask != null)
{
commitTask.Wait();
}
timer.Stop();
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("Committed {0} catalog items in {1}", total, timer.Elapsed);
Console.ResetColor();
}
}
}
static async Task Test2Async()
{
await MakeTestCatalog();
Func<HttpMessageHandler> handlerFunc = () =>
{
return new FileSystemEmulatorHandler
{
BaseAddress = new Uri("http://*****:*****@"c:\data\site",
InnerHandler = new HttpClientHandler()
};
};
TestCollector collectorA = new TestCollector("A", new Uri("http://*****:*****@"c:\data\site\cursor";
Storage storage = new FileStorage(baseAddress, path);
DurableCursor cursorA = new DurableCursor(new Uri("http://localhost:8000/cursor/cursorA.json"), storage, MemoryCursor.MinValue);
DurableCursor cursorB = new DurableCursor(new Uri("http://localhost:8000/cursor/cursorB.json"), storage, MemoryCursor.MinValue);
Console.WriteLine("check catalog...");
bool run = false;
do
{
run = false;
run |= await collectorA.Run(cursorA, MemoryCursor.CreateMax(), CancellationToken.None);
run |= await collectorB.Run(cursorB, cursorA, CancellationToken.None);
}
while (run);
Console.WriteLine("ADDING MORE CATALOG");
await MoreTestCatalog();
do
{
run = false;
run |= await collectorA.Run(cursorA, MemoryCursor.CreateMax(), CancellationToken.None);
run |= await collectorB.Run(cursorB, cursorA, CancellationToken.None);
}
while (run);
Console.WriteLine("ALL DONE");
}
public static async Task Test4Async()
{
Storage storage = new FileStorage
{
Path = @"c:\data\site\test",
Container = "test",
BaseAddress = "http://localhost:8000/"
};
CatalogContext context = new CatalogContext();
CatalogWriter writer = new CatalogWriter(storage, context, 4);
string[] names1 = { "a", "b", "c", "d", "e" };
string[] names2 = { "f", "g", "h" };
string[] names3 = { "i", "j", "k" };
DateTime timeStamp = DateTime.UtcNow;
foreach (string name in names1)
{
writer.Add(new TestCatalogItem(name));
}
await writer.Commit(timeStamp);
Console.WriteLine("commit #1 timeStamp {0}", await CatalogWriter.GetLastCommitTimeStamp(storage));
Console.WriteLine("commit #1 count {0}", await CatalogWriter.GetCount(storage));
timeStamp = timeStamp.AddHours(1);
foreach (string name in names2)
{
writer.Add(new TestCatalogItem(name));
}
await writer.Commit(timeStamp);
Console.WriteLine("commit #2 timeStamp {0}", await CatalogWriter.GetLastCommitTimeStamp(storage));
Console.WriteLine("commit #2 count {0}", await CatalogWriter.GetCount(storage));
timeStamp = timeStamp.AddHours(1);
foreach (string name in names3)
{
writer.Add(new TestCatalogItem(name));
}
await writer.Commit(timeStamp);
Console.WriteLine("commit #3 timeStamp {0}", await CatalogWriter.GetLastCommitTimeStamp(storage));
Console.WriteLine("commit #3 count {0}", await CatalogWriter.GetCount(storage));
}
public static async Task Test3Async()
{
Storage storage = new FileStorage
{
Path = @"c:\data\site\test",
Container = "test",
BaseAddress = "http://localhost:8000/"
};
CatalogContext context = new CatalogContext();
CatalogWriter writer = new CatalogWriter(storage, context, 1000);
string[] names1 = { "a", "b", "c", "d", "e" };
string[] names2 = { "f", "g", "h" };
string[] names3 = { "i", "j", "k" };
foreach (string name in names1)
{
writer.Add(new TestCatalogItem(name));
}
await writer.Commit(new Dictionary<string, string> { { "prop1", "value1.1" }, { "prop2", "value2.1" } });
Console.WriteLine("commit user data #1");
foreach (KeyValuePair<string, string> items in await CatalogWriter.GetCommitUserData(storage))
{
Console.WriteLine("{0} {1}", items.Key, items.Value);
}
foreach (string name in names2)
{
writer.Add(new TestCatalogItem(name));
}
await writer.Commit(new Dictionary<string, string> { { "prop1", "value1.2" }, { "prop2", "value2.2" } });
Console.WriteLine("commit user data #2");
foreach (KeyValuePair<string, string> items in await CatalogWriter.GetCommitUserData(storage))
{
Console.WriteLine("{0} {1}", items.Key, items.Value);
}
foreach (string name in names3)
{
writer.Add(new TestCatalogItem(name));
}
await writer.Commit(new Dictionary<string, string> { { "prop1", "value1.3" }, { "prop2", "value2.3" } });
Console.WriteLine("commit user data #3");
foreach (KeyValuePair<string, string> items in await CatalogWriter.GetCommitUserData(storage))
{
Console.WriteLine("{0} {1}", items.Key, items.Value);
}
}
static async Task Test1Async()
{
await MakeTestCatalog();
Func<HttpMessageHandler> handlerFunc = () =>
{
return new FileSystemEmulatorHandler
{
BaseAddress = new Uri("http://*****:*****@"c:\data\site",
InnerHandler = new HttpClientHandler()
};
};
TestCollector collector = new TestCollector("Test1", new Uri("http://*****:*****@"c:\data\site\cursor";
Storage storage = new FileStorage(baseAddress, path);
DurableCursor front = new DurableCursor(new Uri("http://localhost:8000/cursor/front.json"), storage, MemoryCursor.MinValue);
//DurableCursor back = new DurableCursor(new Uri("http://localhost:8000/cursor/back.json"), storage);
MemoryCursor back = MemoryCursor.CreateMax();
bool didWork = await collector.Run(front, back, CancellationToken.None);
if (!didWork)
{
Console.WriteLine("executed but no work was done");
}
}
static async Task MakeTestCatalog()
{
string baseAddress = "http://*****:*****@"c:\data\site\cursor";
DirectoryInfo folder = new DirectoryInfo(path);
if (folder.Exists)
{
Console.WriteLine("test catalog already created");
return;
}
Storage storage = new FileStorage(baseAddress, path);
AppendOnlyCatalogWriter writer = new AppendOnlyCatalogWriter(storage, 550);
writer.Add(new TestCatalogItem(1));
await writer.Commit(new DateTime(2014, 1, 1, 0, 0, 0, DateTimeKind.Utc), null, CancellationToken.None);
writer.Add(new TestCatalogItem(2));
await writer.Commit(new DateTime(2014, 1, 3, 0, 0, 0, DateTimeKind.Utc),null, CancellationToken.None);
writer.Add(new TestCatalogItem(3));
await writer.Commit(new DateTime(2014, 1, 4, 0, 0, 0, DateTimeKind.Utc),null, CancellationToken.None);
writer.Add(new TestCatalogItem(4));
await writer.Commit(new DateTime(2014, 1, 5, 0, 0, 0, DateTimeKind.Utc),null, CancellationToken.None);
writer.Add(new TestCatalogItem(5));
await writer.Commit(new DateTime(2014, 1, 7, 0, 0, 0, DateTimeKind.Utc), null, CancellationToken.None);
writer.Add(new TestCatalogItem(6));
await writer.Commit(new DateTime(2014, 1, 8, 0, 0, 0, DateTimeKind.Utc), null, CancellationToken.None); ;
writer.Add(new TestCatalogItem(7));
await writer.Commit(new DateTime(2014, 1, 10, 0, 0, 0, DateTimeKind.Utc), null, CancellationToken.None);
Console.WriteLine("test catalog created");
}