public override async Task RunCore()
{
if (!_complete)
{
_complete = true;
var queueClient = Account.CreateCloudQueueClient();
await CreateQueue(queueClient, CantonConstants.GalleryPagesQueue);
await CreateQueue(queueClient, CantonConstants.CatalogCommitQueue);
await CreateQueue(queueClient, CantonConstants.CatalogPageQueue);
var blobClient = Account.CreateCloudBlobClient();
await CreateContainer(blobClient, Config.GetProperty("CatalogContainer"));
await CreateContainer(blobClient, Config.GetProperty("RegistrationContainer"));
await CreateContainer(blobClient, Config.GetProperty("GalleryPageContainer"));
await CreateContainer(blobClient, Config.GetProperty("tmp"));
var tableClient = Account.CreateCloudTableClient();
var cursorTable = tableClient.GetTableReference(CantonConstants.CursorTable);
await cursorTable.CreateIfNotExistsAsync();
string localTmp = Config.GetProperty("localtmp");
DirectoryInfo tmpDir = new DirectoryInfo(localTmp);
if (!tmpDir.Exists)
{
tmpDir.Create();
}
}
}
/// <summary>
/// Canton jobs that can run many instances.
/// </summary>
static void Main(string[] args)
{
if (args.Length != 2)
{
Console.WriteLine(".exe <config path> <thread count>");
Environment.Exit(1);
}
Console.CancelKeyPress += Console_CancelKeyPress;
CantonUtilities.Init();
Config config = new Config(args[0]);
int threadCount = 1;
Int32.TryParse(args[1], out threadCount);
CloudStorageAccount account = CloudStorageAccount.Parse(config.GetProperty("StorageConnectionString"));
Queue <Func <CantonJob> > jobs = new Queue <Func <CantonJob> >();
// process gallery pages and nupkgs into catalog pages
jobs.Enqueue(() => new CatalogPageJob(config, new AzureStorage(account, config.GetProperty("tmp")), CantonConstants.GalleryPagesQueue));
Stopwatch timer = new Stopwatch();
// avoid flooding storage
TimeSpan minWait = TimeSpan.FromMinutes(2);
while (_run)
{
timer.Restart();
CantonUtilities.RunManyJobs(jobs, threadCount);
TimeSpan waitTime = minWait.Subtract(timer.Elapsed);
Console.WriteLine("Completed jobs in: " + timer.Elapsed);
if (waitTime.TotalMilliseconds > 0 && _run)
{
Console.WriteLine("Sleeping: " + waitTime.TotalSeconds + "s");
Thread.Sleep(waitTime);
}
}
}
/// <summary>
/// Canton jobs that can run many instances.
/// </summary>
static void Main(string[] args)
{
if (args.Length != 2)
{
Console.WriteLine(".exe <config path> <thread count>");
Environment.Exit(1);
}
Console.CancelKeyPress += Console_CancelKeyPress;
CantonUtilities.Init();
Config config = new Config(args[0]);
int threadCount = 1;
Int32.TryParse(args[1], out threadCount);
CloudStorageAccount account = CloudStorageAccount.Parse(config.GetProperty("StorageConnectionString"));
Queue<Func<CantonJob>> jobs = new Queue<Func<CantonJob>>();
// process gallery pages and nupkgs into catalog pages
jobs.Enqueue(() => new CatalogPageJob(config, new AzureStorage(account, config.GetProperty("tmp")), CantonConstants.GalleryPagesQueue));
Stopwatch timer = new Stopwatch();
// avoid flooding storage
TimeSpan minWait = TimeSpan.FromMinutes(2);
while (_run)
{
timer.Restart();
CantonUtilities.RunManyJobs(jobs, threadCount);
TimeSpan waitTime = minWait.Subtract(timer.Elapsed);
Console.WriteLine("Completed jobs in: " + timer.Elapsed);
if (waitTime.TotalMilliseconds > 0 && _run)
{
Console.WriteLine("Sleeping: " + waitTime.TotalSeconds + "s");
Thread.Sleep(waitTime);
}
}
}
public CatalogPageJob(Config config, Storage storage, string queueName)
: base(config, storage, queueName)
{
_packagesStorageAccount = CloudStorageAccount.Parse(config.GetProperty("PackagesStorageConnectionString"));
_queueTasks = new ConcurrentQueue <Task>();
_workQueueStatus = new ConcurrentDictionary <int, bool>();
_tmpDir = new DirectoryInfo(Config.GetProperty("localtmp"));
if (!_tmpDir.Exists)
{
_tmpDir.Create();
}
}
public CatalogPageJob(Config config, Storage storage, string queueName)
: base(config, storage, queueName)
{
_packagesStorageAccount = CloudStorageAccount.Parse(config.GetProperty("PackagesStorageConnectionString"));
_queueTasks = new ConcurrentQueue<Task>();
_workQueueStatus = new ConcurrentDictionary<int, bool>();
_tmpDir = new DirectoryInfo(Config.GetProperty("localtmp"));
if (!_tmpDir.Exists)
{
_tmpDir.Create();
}
}
private async Task <FileInfo> GetNupkg(string id, string version)
{
// TODO: Use the MD5 hash on the prod packgae to see if we have the current one
//NuGetVersion nugetVersion = new NuGetVersion(version);
var tmpBlobClient = Account.CreateCloudBlobClient();
string packageName = String.Format(CultureInfo.InvariantCulture, "{0}.{1}.nupkg", id, version).ToLowerInvariant();
FileInfo file = new FileInfo(Path.Combine(_tmpDir.FullName, Guid.NewGuid().ToString() + ".nupkg"));
var tmpContainer = tmpBlobClient.GetContainerReference(Config.GetProperty("tmp"));
string tmpFile = String.Format(CultureInfo.InvariantCulture, "packages/{0}", packageName);
var tmpBlob = tmpContainer.GetBlockBlobReference(tmpFile);
if (await tmpBlob.ExistsAsync())
{
Log("Downloading from tmp: " + packageName);
await tmpBlob.DownloadToFileAsync(file.FullName, FileMode.CreateNew);
}
else
{
var blobClient = _packagesStorageAccount.CreateCloudBlobClient();
var prodPackages = blobClient.GetContainerReference("packages");
var prodBlob = prodPackages.GetBlockBlobReference(packageName);
if (prodBlob.Exists())
{
Log("Downloading from prod: " + packageName);
await prodBlob.DownloadToFileAsync(file.FullName, FileMode.CreateNew);
// store this in tmp also
await tmpBlob.UploadFromFileAsync(file.FullName, FileMode.Open);
}
else
{
throw new FileNotFoundException(prodBlob.Uri.AbsoluteUri);
}
}
return(file);
}
/// <summary>
/// Canton jobs that can only run as single instances.
/// </summary>
static void Main(string[] args)
{
if (args.Length != 1)
{
Console.WriteLine(".exe <config path>");
Environment.Exit(1);
}
Console.CancelKeyPress += Console_CancelKeyPress;
CantonUtilities.Init();
Config config = new Config(args[0]);
CloudStorageAccount account = CloudStorageAccount.Parse(config.GetProperty("StorageConnectionString"));
CloudStorageAccount outputAccount = CloudStorageAccount.Parse(config.GetProperty("OutputStorageConnectionString"));
Uri baseAddress = new Uri(config.GetProperty("BaseAddress"));
Queue<CantonJob> jobs = new Queue<CantonJob>();
// set up the storage account
jobs.Enqueue(new InitStorageJob(config));
// read the gallery to find new packages
// jobs.Enqueue(new QueueNewPackagesFromGallery(config, new AzureStorage(account, config.GetProperty("GalleryPageContainer"))));
// this does the work of Many, just so this can all be in one .exe
jobs.Enqueue(new CatalogPageJob(config, new AzureStorage(account, config.GetProperty("tmp")), CantonConstants.GalleryPagesQueue));
// commit pages to the catalog
jobs.Enqueue(new CatalogPageCommitJob(config, new AzureStorage(outputAccount, config.GetProperty("CatalogContainer"),
string.Empty, new Uri(baseAddress.AbsoluteUri + config.GetProperty("CatalogContainer") + "/"))));
// registrations
Uri regBase = new Uri(baseAddress, config.GetProperty("RegistrationContainer") + "/");
TransHttpClient httpClient = new TransHttpClient(outputAccount, config.GetProperty("BaseAddress"));
jobs.Enqueue(new PartitionedRegJob(config, new AzureStorage(outputAccount, config.GetProperty("RegistrationContainer"), string.Empty, regBase),
new AzureStorageFactory(outputAccount, config.GetProperty("RegistrationContainer"), null, regBase), httpClient));
Stopwatch timer = new Stopwatch();
// avoid flooding the gallery
TimeSpan minWait = TimeSpan.FromMinutes(10);
while (_run)
{
timer.Restart();
CantonUtilities.RunJobs(jobs);
TimeSpan waitTime = minWait.Subtract(timer.Elapsed);
Console.WriteLine("Completed jobs in: " + timer.Elapsed);
// min sleep
Thread.Sleep(60 * 1000);
if (waitTime.TotalSeconds > 0 && _run)
{
Console.WriteLine("Sleeping: " + waitTime.TotalSeconds + "s");
Thread.Sleep(waitTime);
}
}
}
public override async Task RunCore()
{
int nextMasterRegId = 0;
DateTime position = Cursor.Position;
JToken nextMasterRegIdToken = null;
if (Cursor.Metadata.TryGetValue("nextMasterRegId", out nextMasterRegIdToken))
{
nextMasterRegId = nextMasterRegIdToken.ToObject <int>();
}
// Get the catalog index
Uri catalogIndexUri = new Uri(Config.GetProperty("CatalogIndex"));
Log("Reading index entries");
var indexReader = new CatalogIndexReader(catalogIndexUri);
var indexEntries = await indexReader.GetRolledUpEntries();
var context = indexReader.GetContext();
Log("Finding new or editted entries");
var changedEntries = new HashSet <string>(indexEntries.Where(e => e.CommitTimeStamp.CompareTo(position) > 0)
.Select(e => e.Id.ToLowerInvariant()), StringComparer.OrdinalIgnoreCase);
DateTime newPosition = indexEntries.Select(e => e.CommitTimeStamp).OrderByDescending(e => e).FirstOrDefault();
ConcurrentDictionary <string, ConcurrentBag <Uri> > batches = new ConcurrentDictionary <string, ConcurrentBag <Uri> >(StringComparer.OrdinalIgnoreCase);
ParallelOptions options = new ParallelOptions();
options.MaxDegreeOfParallelism = 8;
Parallel.ForEach(indexEntries, options, entry =>
{
if (changedEntries.Contains(entry.Id))
{
batches.AddOrUpdate(entry.Id, new ConcurrentBag <Uri>()
{
entry.Uri
}, (id, uris) =>
{
uris.Add(entry.Uri);
return(uris);
});
}
});
Uri contentBaseAddress = new Uri(Config.GetProperty("ContentBaseAddress"));
if (batches.Count > 0)
{
Log("Building registrations from: " + position.ToString("O"));
options.MaxDegreeOfParallelism = 4;
for (int i = 0; i < 3 && batches.Count > 0; i++)
{
if (i != 0)
{
options.MaxDegreeOfParallelism = 1;
Console.WriteLine("Single batch run.");
}
var ids = batches.Keys.OrderBy(s => s).ToArray();
Stopwatch buildTimer = new Stopwatch();
buildTimer.Start();
int startingCount = ids.Length;
Parallel.ForEach(ids, options, id =>
{
try
{
BatchRegistrationCollector regCollector = new BatchRegistrationCollector(null, _factory);
regCollector.ContentBaseAddress = contentBaseAddress;
Stopwatch timer = new Stopwatch();
timer.Start();
var uriGroup = batches[id].ToArray();
regCollector.ProcessGraphs(_client, id, uriGroup, context).Wait();
int rem = batches.Count;
timer.Stop();
string log = String.Format("Completed: {0} Duration: {1} Uris: {2} Remaining Ids: {3} Loop: {4}", id, timer.Elapsed, uriGroup.Length, rem, i);
Console.WriteLine(log);
// stats
double perPackage = buildTimer.Elapsed.TotalSeconds / (double)(startingCount - rem + 1);
DateTime finish = DateTime.Now.AddSeconds(Math.Ceiling(perPackage * rem));
Console.WriteLine("Estimated Finish: " + finish.ToString("O"));
ConcurrentBag <Uri> vals;
if (!batches.TryRemove(id, out vals))
{
Console.WriteLine("Unable to remove!");
}
}
catch (Exception ex)
{
LogError("Registration failed: " + id + " " + ex.ToString());
}
});
}
// mark this with the last commit we included
Cursor.Position = newPosition;
await Cursor.Save();
Log("Finished registrations: " + newPosition.ToString("O"));
}
}