private static async Task FullRun(HttpListenerContext context)
{
_ = TaskManager.Run(async() =>
{
await FullUpdateProcessor.FullUpdateAppsMetadata(true);
await FullUpdateProcessor.FullUpdatePackagesMetadata();
});
await WriteJsonResponse("Triggered full run", context.Response);
}
public PICSChanges(CallbackManager manager)
{
if (Settings.IsFullRun)
{
var timer = new Timer
{
Interval = TimeSpan.FromSeconds(10).TotalMilliseconds
};
timer.Elapsed += (sender, args) => FullUpdateProcessor.IsBusy();
timer.Start();
return;
}
manager.Subscribe <SteamApps.PICSChangesCallback>(OnPICSChanges);
using (var db = Database.Get())
{
BillingTypeKey = KeyNameCache.GetSubKeyID("root_billingtype");
if (LocalConfig.Current.ChangeNumber == 0)
{
PreviousChangeNumber = db.ExecuteScalar <uint>("SELECT `ChangeID` FROM `Changelists` ORDER BY `ChangeID` DESC LIMIT 1");
LocalConfig.Current.ChangeNumber = PreviousChangeNumber;
}
else
{
PreviousChangeNumber = LocalConfig.Current.ChangeNumber;
}
Log.WriteInfo(nameof(PICSChanges), $"Previous changelist was {PreviousChangeNumber}");
}
if (PreviousChangeNumber == 0)
{
Log.WriteWarn(nameof(PICSChanges), "Looks like there are no changelists in the database.");
Log.WriteWarn(nameof(PICSChanges), $"If you want to fill up your database first, restart with \"FullRun\" setting set to {(int)FullRunState.Enumerate}.");
}
}
private void OnPICSChanges(SteamApps.PICSChangesCallback callback)
{
if (PreviousChangeNumber == callback.CurrentChangeNumber)
{
return;
}
Log.WriteInfo(nameof(PICSChanges), $"Changelist {PreviousChangeNumber} -> {callback.CurrentChangeNumber} ({callback.AppChanges.Count} apps, {callback.PackageChanges.Count} packages)");
LocalConfig.Current.ChangeNumber = PreviousChangeNumber = callback.CurrentChangeNumber;
TaskManager.Run(async() => await HandleChangeNumbers(callback));
if (callback.RequiresFullAppUpdate || callback.RequiresFullPackageUpdate)
{
TaskManager.Run(async() =>
{
if (callback.RequiresFullAppUpdate)
{
IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} has forced a full app update");
await FullUpdateProcessor.FullUpdateAppsMetadata();
}
if (callback.RequiresFullPackageUpdate)
{
IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} has forced a full package update");
await FullUpdateProcessor.FullUpdatePackagesMetadata();
}
});
}
if (callback.AppChanges.Count == 0 && callback.PackageChanges.Count == 0)
{
IRC.Instance.SendAnnounce($"{Colors.RED}»{Colors.NORMAL} Changelist {Colors.BLUE}{PreviousChangeNumber}{Colors.DARKGRAY} (empty)");
return;
}
const int appsPerJob = 50;
if (callback.AppChanges.Count > appsPerJob)
{
foreach (var list in callback.AppChanges.Keys.Split(appsPerJob))
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(list, Enumerable.Empty <uint>()),
new PICSTokens.RequestedTokens
{
Apps = list.ToList()
});
}
}
else if (callback.AppChanges.Count > 0)
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(callback.AppChanges.Keys, Enumerable.Empty <uint>()),
new PICSTokens.RequestedTokens
{
Apps = callback.AppChanges.Keys.ToList()
});
}
if (callback.PackageChanges.Count > appsPerJob)
{
foreach (var list in callback.PackageChanges.Keys.Split(appsPerJob))
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), list),
new PICSTokens.RequestedTokens
{
Packages = list.ToList()
});
}
}
else if (callback.PackageChanges.Count > 0)
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), callback.PackageChanges.Keys),
new PICSTokens.RequestedTokens
{
Packages = callback.PackageChanges.Keys.ToList()
});
}
if (callback.AppChanges.Count > 0)
{
_ = TaskManager.Run(async() => await HandleApps(callback));
}
if (callback.PackageChanges.Count > 0)
{
_ = TaskManager.Run(async() => await HandlePackages(callback));
_ = TaskManager.Run(async() => await HandlePackagesChangelists(callback));
}
_ = TaskManager.Run(async() => await SendChangelistsToIRC(callback));
PrintImportants(callback);
}
private static void OnPICSProductInfo(SteamApps.PICSProductInfoCallback callback)
{
JobManager.TryRemoveJob(callback.JobID);
if (callback.MetaDataOnly)
{
TaskManager.Run(async() => await FullUpdateProcessor.HandleMetadataInfo(callback));
return;
}
var processors = new List <BaseProcessor>(
callback.Apps.Count +
callback.Packages.Count +
callback.UnknownApps.Count +
callback.UnknownPackages.Count
);
processors.AddRange(callback.Apps.Select(app => new AppProcessor(app.Key, app.Value)));
processors.AddRange(callback.Packages.Select(package => new SubProcessor(package.Key, package.Value)));
processors.AddRange(callback.UnknownApps.Select(app => new AppProcessor(app, null)));
processors.AddRange(callback.UnknownPackages.Select(package => new SubProcessor(package, null)));
foreach (var workaround in processors)
{
var processor = workaround;
Task mostRecentItem;
lock (CurrentlyProcessing)
{
CurrentlyProcessing.TryGetValue(processor.Id, out mostRecentItem);
}
var workerItem = TaskManager.Run(async() =>
{
try
{
await Semaphore.WaitAsync(TaskManager.TaskCancellationToken.Token).ConfigureAwait(false);
if (mostRecentItem?.IsCompleted == false)
{
Log.WriteDebug(processor.ToString(), $"Waiting for previous task to finish processing ({CurrentlyProcessing.Count})");
await mostRecentItem.ConfigureAwait(false);
#if DEBUG
Log.WriteDebug(processor.ToString(), "Previous task lock ended");
#endif
}
await processor.Process().ConfigureAwait(false);
}
catch (Exception e)
{
ErrorReporter.Notify(processor.ToString(), e);
}
finally
{
Semaphore.Release();
processor.Dispose();
}
return(processor);
});
lock (CurrentlyProcessing)
{
CurrentlyProcessing[processor.Id] = workerItem;
}
workerItem.ContinueWith(RemoveProcessorLock, TaskManager.TaskCancellationToken.Token);
}
}
private void OnPICSChanges(SteamApps.PICSChangesCallback callback)
{
if (PreviousChangeNumber == callback.CurrentChangeNumber)
{
return;
}
Log.WriteInfo(nameof(PICSChanges), $"Changelist {callback.LastChangeNumber} -> {callback.CurrentChangeNumber} ({callback.AppChanges.Count} apps, {callback.PackageChanges.Count} packages)");
PreviousChangeNumber = callback.CurrentChangeNumber;
TaskManager.Run(async() => await HandleChangeNumbers(callback));
if (callback.RequiresFullAppUpdate || callback.RequiresFullPackageUpdate)
{
TaskManager.Run(async() =>
{
if (callback.RequiresFullAppUpdate)
{
IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} has forced a full app update");
// When full update flag is set, presumably Steam client start hammering the servers
// and the PICS service just does not return any data for a while until it clears up
await FullUpdateProcessor.FullUpdateAppsMetadata(true);
}
if (callback.RequiresFullPackageUpdate)
{
IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} has forced a full package update");
await FullUpdateProcessor.FullUpdatePackagesMetadata();
}
IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} full update has finished");
});
}
if (callback.AppChanges.Count == 0 && callback.PackageChanges.Count == 0)
{
return;
}
const int appsPerJob = 50;
if (callback.AppChanges.Count > appsPerJob)
{
foreach (var list in callback.AppChanges.Keys.Split(appsPerJob))
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(list, Enumerable.Empty <uint>()),
new PICSTokens.RequestedTokens
{
Apps = list.ToList()
});
}
}
else if (callback.AppChanges.Count > 0)
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(callback.AppChanges.Keys, Enumerable.Empty <uint>()),
new PICSTokens.RequestedTokens
{
Apps = callback.AppChanges.Keys.ToList()
});
}
if (callback.PackageChanges.Count > appsPerJob)
{
foreach (var list in callback.PackageChanges.Keys.Split(appsPerJob))
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), list),
new PICSTokens.RequestedTokens
{
Packages = list.ToList()
});
}
}
else if (callback.PackageChanges.Count > 0)
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), callback.PackageChanges.Keys),
new PICSTokens.RequestedTokens
{
Packages = callback.PackageChanges.Keys.ToList()
});
}
if (callback.AppChanges.Count > 0)
{
_ = TaskManager.Run(async() => await HandleApps(callback));
}
if (callback.PackageChanges.Count > 0)
{
_ = TaskManager.Run(async() => await HandlePackages(callback));
_ = TaskManager.Run(async() => await HandlePackagesChangelists(callback));
}
if (PreviousChangeNumber - LastStoredChangeNumber >= 1000)
{
LastStoredChangeNumber = PreviousChangeNumber;
_ = TaskManager.Run(async() => await LocalConfig.Update("backend.changenumber", LastStoredChangeNumber.ToString()));
}
PrintImportants(callback);
}
public override async Task OnCommand(CommandArguments command)
{
var s = command.Message.Split(' ');
var count = s.Length;
if (count > 0)
{
uint id;
switch (s[0])
{
case "reload":
await Application.ReloadImportant();
await PICSTokens.Reload();
command.Notice("Reloaded important apps and pics tokens");
return;
case "fullrun":
_ = TaskManager.Run(async() =>
{
command.Reply("Started full metadata scan, this will take a while…");
await FullUpdateProcessor.FullUpdateAppsMetadata(true);
command.Reply("App full scan finished, starting packages, this will take even longer…");
await FullUpdateProcessor.FullUpdatePackagesMetadata();
command.Reply("Full metadata scan finished.");
});
return;
case "add":
if (count < 3)
{
break;
}
if (!uint.TryParse(s[2], out id))
{
break;
}
switch (s[1])
{
case "app":
if (Application.ImportantApps.Contains(id))
{
command.Reply($"App {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetAppName(id)}) is already important.");
}
else
{
Application.ImportantApps.Add(id);
await using (var db = await Database.GetConnectionAsync())
{
await db.ExecuteAsync("INSERT INTO `ImportantApps` (`AppID`) VALUES (@AppID)", new { AppID = id });
}
command.Reply($"Marked app {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetAppName(id)}) as important.");
}
return;
case "sub":
if (Application.ImportantSubs.Contains(id))
{
command.Reply($"Package {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetPackageName(id)}) is already important.");
}
else
{
Application.ImportantSubs.Add(id);
await using (var db = await Database.GetConnectionAsync())
{
await db.ExecuteAsync("INSERT INTO `ImportantSubs` (`SubID`) VALUES (@SubID)", new { SubID = id });
}
command.Reply($"Marked package {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetPackageName(id)}) as important.");
}
return;
}
break;
case "remove":
if (count < 3)
{
break;
}
if (!uint.TryParse(s[2], out id))
{
break;
}
switch (s[1])
{
case "app":
if (!Application.ImportantApps.Contains(id))
{
command.Reply($"App {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetAppName(id)}) is not important.");
}
else
{
Application.ImportantApps.Remove(id);
await using (var db = await Database.GetConnectionAsync())
{
await db.ExecuteAsync("DELETE FROM `ImportantApps` WHERE `AppID` = @AppID", new { AppID = id });
}
command.Reply($"Removed app {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetAppName(id)}) from the important list.");
}
return;
case "sub":
if (!Application.ImportantSubs.Contains(id))
{
command.Reply($"Package {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetPackageName(id)}) is not important.");
}
else
{
Application.ImportantSubs.Remove(id);
await using (var db = await Database.GetConnectionAsync())
{
await db.ExecuteAsync("DELETE FROM `ImportantSubs` WHERE `SubID` = @SubID", new { SubID = id });
}
command.Reply($"Removed package {Colors.BLUE}{id}{Colors.NORMAL} ({Steam.GetPackageName(id)}) from the important list.");
}
return;
}
break;
}
}
command.Reply($"Usage:{Colors.OLIVE} important reload {Colors.NORMAL}or{Colors.OLIVE} important <add/remove> <app/sub> <id> {Colors.NORMAL}or{Colors.OLIVE} important fullrun");
}
