public InitializeTask(IAppParser appParser, IAppIndexer indexer, RepositoryEmitter repository, ProgramSettings settings) : base(settings) { this.appParser = appParser; this.indexer = indexer; this.repository = repository; }
public RescueTask(IAppParser appParser, IAppIndexer indexer, RepositoryEmitter repository, ProgramSettings settings) : base(settings) { this.appParser = appParser; this.indexer = indexer; this.repository = repository; limit = settings.BatchSize / 200 * 200; // 因为Search API是200一批,找个最接近的200的倍数,以免浪费 }
public UpdateNotifier(RepositoryEmitter repository, SmtpClient smtp, ProgramSettings settings) { this.repository = repository; this.smtp = smtp; this.settings = settings; }
static Program() { souce = new MySqlConnection(ConfigurationManager.ConnectionStrings["Source"].ConnectionString); souce.Open(); destination = new MySqlConnection( ConfigurationManager.ConnectionStrings["Destination"].ConnectionString); destination.Open(); repository = new RepositoryEmitter( new AppRepository(destination), new AppUpdateRepository(destination), new AppTrackRepository(destination), new UserRepository(destination) ); }
private void RevokeApps(RepositoryEmitter repository) { Stopwatch watch = new Stopwatch(); watch.Start(); foreach (App app in revokedApps) { // 添加下架信息 AppUpdate update = new AppUpdate() { App = app.Id, Type = AppUpdateType.Revoke, OldValue = app.Brief.Version + ", " + app.Brief.PriceWithSymbol, Time = DateTime.Now }; repository.AppUpdate.Save(update); // 从在售应用中移除 repository.App.Delete(app.Id); // 添加到下架应用中 RevokedApp revoked = new RevokedApp(app); revoked.RevokeTime = DateTime.Now; repository.App.SaveRevoked(revoked); // 移除全文索引 indexer.DeleteApp(app); logger.Trace("Set app {0}-{1} to be revoked", app.Id, app.Brief.Name); } watch.Stop(); logger.Info("Revoked {0} apps using {1}ms", revokedApps.Count, watch.ElapsedMilliseconds); watch.Reset(); watch.Stop(); indexer.Flush(); watch.Stop(); logger.Info("Deleted index for {0} apps using {1}ms", revokedApps.Count, watch.ElapsedMilliseconds); }
private bool CheckUpdateForApp(App original, App updated, RepositoryEmitter repository) { if (original.Equals(updated)) { return false; } // 检查/添加更新信息 ICollection<AppUpdate> validUpdates = original.Brief.CheckForUpdate(updated.Brief); foreach (AppUpdate update in validUpdates) { repository.AppUpdate.Save(update); logger.Trace( "Added update of type {0} for app {1}-{2}", update.Type, original.Id, original.Brief.Name ); if (AppUpdate.IsValidUpdate(update.Type)) { updated.Brief.LastValidUpdate = update; } // 通知用户 notifier.ProcessUpdate(updated, update); } // 更新应用 original.UpdateFrom(updated); repository.App.Update(original); // 更新索引 indexer.UpdateApp(original); logger.Trace("Updated app {0}-{1}", original.Id, original.Brief.Name); return true; }
private int CheckUpdates(IEnumerable<App> apps, RepositoryEmitter repository) { int[] identities = apps.Select(a => a.Id).ToArray(); ICollection<App> retrievedApps = appParser.RetrieveApps(identities); if (retrievedApps == null) { return 0; } Dictionary<int, App> updated = retrievedApps.ToDictionary(a => a.Id); int count = 0; foreach (App app in apps) { if (updated.ContainsKey(app.Id)) { // 检查是否有更新 bool changed = CheckUpdateForApp(app, updated[app.Id], repository); if (changed) { count++; } } else { /* * 数据库中有,但Search API上没有,定义为下架 * 由于正在不断获取数据,此时移除Collection中的内容,会导致数据获取的不准确性,遗漏数据 * 因此先将下架的应用放入集合中,等待后续统一处理 * NOTICE: 由于这个逻辑的存在,Update任务不能和其他任务一起跑,会引起并发冲突(基本所有任务都不能和其他的并行) */ lock (revokedApps) { revokedApps.Add(app); } } } Stopwatch watch = new Stopwatch(); watch.Start(); indexer.Flush(); watch.Stop(); logger.Debug("Indexed {0} apps using {1}ms", retrievedApps.Count, watch.ElapsedMilliseconds); return count; }
private int RetrieveAndUpdate(RepositoryEmitter repository) { while (true) { ICollection<App> apps; lock (repository) { if (offset < 0) { return 0; } logger.Trace("Retrieve apps in range {0}-{1}", offset, offset + limit); Stopwatch stepWatch = new Stopwatch(); stepWatch.Start(); apps = repository.App.Retrieve(offset, limit); if (apps.Count < limit) { offset = -1; } else { offset += limit; } stepWatch.Stop(); logger.Debug("Retrieved {0} apps from database using {1}ms", apps.Count, stepWatch.ElapsedMilliseconds); } foreach (IEnumerable<App> partition in apps.Partition(200)) { CheckUpdates(partition, repository); } } }