static async Task Main(string[] args)
{
Log.Logger = new LoggerConfiguration()
.WriteTo.Console()
.CreateLogger();
var config = await JsonSerializer.DeserializeAsync <Configuration[]>(File.OpenRead("./config/spider/config.json"));
foreach (var configuration in config)
{
var baseMod = await UrlLib.GetModInfoAsync(configuration.SpiderConfiguration.ModCount, configuration.Version);
var allMod = await UrlLib.GetModInfoWhenInWhiteList(configuration.SpiderConfiguration.WhiteList);
allMod.AddRange(baseMod);
await(await(await allMod.ToArray().GenerateBases(configuration.Version).DownloadModAsync()).ParseModAsync(
configuration.Version, configuration.SpiderConfiguration.BlackList)).ExtractResource(configuration.Version).WriteToAsync(configuration.Version, "mod_info.json");
if (configuration.Version == "1.12.2")
{
var file = Util.SearchLangFiles(configuration.Version);
await Util.FormatLangFile(file, await Util.ReadBlackKey());
await Util.CrateEmptyLangFile(configuration.Version);
}
else
{
await Util.CrateEmptyJsonFile(configuration.Version);
}
}
}
public static ModBase[] GenerateBases(this Addon[] addons, string version)
{
var bases = new List <ModBase>();
foreach (var addon in addons)
{
var name = addon.Name;
var projectName = UrlLib.GetProjectName(addon.WebsiteUrl);
var projectId = addon.Id;
var projectUrl = addon.WebsiteUrl;
var mod = addon.GameVersionLatestFiles.First(_ => _.GameVersion == version);
var downloadUrl = UrlLib.GetDownloadUrl(mod.ProjectFileId.ToString(), mod.ProjectFileName);
var modBase = new ModBase()
{
DownloadUrl = downloadUrl, Name = name, ProjectId = projectId, ProjectName = projectName,
ProjectUrl = projectUrl
};
bases.Add(modBase);
}
return(bases.ToArray());
}
static async Task Main()
{
//Directory.SetCurrentDirectory(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) ?? throw new InvalidOperationException());
Log.Logger = new LoggerConfiguration()
.WriteTo.Console()
.Enrich.FromLogContext()
.MinimumLevel.Information()
.CreateLogger();
var c = await JsonReader.ReadConfigAsync();
foreach (var cfg in c)
{
Log.Logger.Information("Loading version configration: {0}", cfg.Version);
//初始化parser
var parser = new InfoParser(cfg.Configuration, cfg.CustomConfigurations);
if (cfg.Configuration is not null)
{
var dict = await JsonReader.ReadIntroAsync(cfg.Configuration.Version, cfg.Version);
var root = Directory.CreateDirectory(
$"{Directory.GetCurrentDirectory()}\\projects\\{cfg.Version}\\assets");
var names = root.GetDirectories().Select(_ => _.Name).ToList();
if (cfg.CustomConfigurations is not null)
{
foreach (var configuration in cfg.CustomConfigurations)
{
if (!names.Contains(configuration.ProjectName))
{
names.Add(configuration.ProjectName);
}
}
}
if (cfg.Count is not null)
{
var allMods = await UrlLib.GetModInfoAsync(cfg.Count.Value, cfg.Configuration.Version);
foreach (var modInfo in allMods)
{
if (dict.ContainsKey(modInfo.Slug) || dict.ContainsValue(modInfo.Id))
{
continue;
}
dict.Add(modInfo.Slug, modInfo.Id);
}
if (names.Count > cfg.Count)
{
var bin = allMods.Where(_ => !names.Contains(_.Slug));
var l = allMods.ToList();
foreach (var info in bin)
{
l.Remove(info);
}
allMods = l.ToArray();
}
var allNames = allMods.ToList().Select(_ => _.Slug).Distinct().ToList();
var l1 = parser.SerializeAll(allMods).Distinct().ToList();
var pending = new List <string>();
foreach (var str in names)
{
if (!allNames.Contains(str))
{
pending.Add(str);
}
}
Log.Logger.Information($"该版本[assets]文件夹下含有 {names.Count} 个mod，有 {pending.Count} 要单独处理");
var semaphore = new SemaphoreSlim(16, 16);
var tasks = l1.Select(async mod =>
{
try
{
await semaphore.WaitAsync();
await Utils.ParseModsAsync(mod, cfg);
}
catch (Exception e)
{
Log.Logger.Error(e.Message);
}
finally
{
semaphore.Release();
}
});
await Task.WhenAll(tasks);
if (!Directory.Exists(@$ "{Directory.GetCurrentDirectory()}\config\spider\{cfg.Configuration.Version}"))
{
Directory.CreateDirectory(
@$ "{Directory.GetCurrentDirectory()}\config\spider\{cfg.Configuration.Version}");
}
var inf = new List <ModIntro>();
foreach (var(key, value) in dict)
{
inf.Add(new ModIntro()
{
Name = key, Id = value
});
}
await File.WriteAllTextAsync(@$ "{Directory.GetCurrentDirectory()}\config\spider\{cfg.Version}\intro.json",
JsonSerializer.Serialize(inf, new JsonSerializerOptions()
{
WriteIndented = true
}));
var ids = pending.Where(name => dict.ContainsKey(name)).Select(name => dict[name].ToString());
var infos = await UrlLib.GetModInfosAsync(ids);
var cfgs = parser.SerializeAll(infos);
var semaphore2 = new SemaphoreSlim(16, 16);
var postTask = cfgs.Select(async _ =>
{
try
{
await semaphore2.WaitAsync();
await Utils.ParseModsAsync(_, cfg);
}
catch (Exception e)
{
Log.Logger.Error(e.Message);
}
finally
{
semaphore2.Release();
}
});
await Task.WhenAll(postTask);
}
static async Task Main(string[] args)
{
//Directory.SetCurrentDirectory(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) ?? throw new InvalidOperationException());
Log.Logger = new LoggerConfiguration()
.WriteTo.Console()
.CreateLogger();
var c = await JsonReader.ReadConfigAsync();
foreach (var cfg in c)
{
var parser = new InfoParser(cfg.Configuration, cfg.CustomConfigurations);
var root = Directory.CreateDirectory(
$"{Directory.GetCurrentDirectory()}\\projects\\{cfg.Version}\\assets");
var names = root.GetDirectories().Select(_ => _.Name).ToList();
foreach (var configuration in cfg.CustomConfigurations)
{
if (!names.Contains(configuration.ProjectName))
{
names.Add(configuration.ProjectName);
}
}
var allM = await UrlLib.GetModInfoAsync(cfg.Count, cfg.Version);
var allN = allM.ToList().Select(_ => _.ShortWebsiteUrl).ToList();
var pending = new List <string>();
foreach (var info in names)
{
if (!allN.Contains(info))
{
pending.Add(info);
}
}
Log.Logger.Information($"该版本[assets]文件夹下含有 {names.Count} 个mod，{pending.Count} 个mod需要单独处理");
var dict = await JsonReader.ReadIntroAsync(cfg.Version);
if (names.Count > cfg.Count)
{
var bin = allM.Where(_ => !names.Contains(_.ShortWebsiteUrl));
var l = allM.ToList();
foreach (var info in bin)
{
l.Remove(info);
}
allM = l.ToArray();
}
parser.Infos = allM.ToList();
var l1 = parser.SerializeAll();
var semaphore = new Semaphore(32, 40);
foreach (var l in l1)
{
try {
semaphore.WaitOne();
await Utils.ParseModsAsync(l);
}
catch (Exception e) {
Log.Logger.Error(e.Message);
}
finally {
semaphore.Release();
}
}
foreach (var name in pending)
{
if (dict.ContainsKey(name))
{
var m = await UrlLib.GetModInfoAsync(dict[name]);
var i = parser.Serialize(m);
try {
await Utils.ParseModsAsync(i);
}
catch (Exception e) {
Log.Logger.Error(e.Message);
}
Thread.Sleep(5000);
}
}
}
}
static async Task Main()
{
//Directory.SetCurrentDirectory(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) ?? throw new InvalidOperationException());
Log.Logger = new LoggerConfiguration()
.WriteTo.Console()
.Enrich.FromLogContext()
.MinimumLevel.Information()
.CreateLogger();
var c = await JsonReader.ReadConfigAsync();
foreach (var cfg in c)
{
var parser = new InfoParser(cfg.Configuration, cfg.CustomConfigurations);
if (cfg.Configuration != null)
{
var dict = await JsonReader.ReadIntroAsync(cfg.Configuration.Version, cfg.Version);
var pending = new List <string>();
var root = Directory.CreateDirectory(
$"{Directory.GetCurrentDirectory()}\\projects\\{cfg.Version}\\assets");
var names = root.GetDirectories().Select(_ => _.Name).ToList();
if (cfg.CustomConfigurations != null)
{
foreach (var configuration in cfg.CustomConfigurations)
{
if (!names.Contains(configuration.ProjectName))
{
names.Add(configuration.ProjectName);
}
}
}
if (cfg.Count != null)
{
var allM = await UrlLib.GetModInfoAsync(cfg.Count.Value, cfg.Configuration.Version);
if (names.Count > cfg.Count)
{
var bin = allM.Where(_ => !names.Contains(_.Slug));
var l = allM.ToList();
foreach (var info in bin)
{
l.Remove(info);
}
allM = l.ToArray();
}
var allN = allM.ToList().Select(_ => _.Slug).Distinct().ToList();
var l1 = parser.SerializeAll(allM).Distinct().ToList();
//var parallelOption = new ParallelOptions {
// MaxDegreeOfParallelism = 16
//};
foreach (var str in names)
{
if (!allN.Contains(str))
{
pending.Add(str);
}
}
Log.Logger.Information($"该版本[assets]文件夹下含有 {names.Count} 个mod，有 {pending.Count} 要单独处理");
var semaphore = new SemaphoreSlim(16, 16);
//Parallel.ForEach(l1, parallelOption, (async tuple => {
// try {
// semaphore.WaitOne();
// await Utils.ParseModsAsync(tuple, cfg);
// }
// catch (Exception e) {
// Log.Logger.Error(e.Message);
// }
// finally {
// semaphore.Release();
// }
//}));
var tasks = l1.Select(async _ =>
{
try
{
await semaphore.WaitAsync();
await Utils.ParseModsAsync(_, cfg);
}
catch (Exception e)
{
Log.Logger.Error(e.Message);
}
finally
{
semaphore.Release();
}
});
await Task.WhenAll(tasks);
}
foreach (var name in pending)
{
if (dict.ContainsKey(name))
{
var m = await UrlLib.GetModInfoAsync(dict[name]);
var i = parser.Serialize(m);
try
{
var task = new Task(async() =>
{
try
{
await Utils.ParseModsAsync(i, cfg);
}
catch (Exception e)
{
Log.Logger.Error(e.Message);
}
});
task.Start();
//await Utils.ParseModsAsync(i,cfg);
}
catch (Exception e)
{
Log.Logger.Error(e.Message);
}
Thread.Sleep(5000);
}
}
}
//var semaphore = new Semaphore(32, 40);
//foreach (var l in l1) {
// try {
// semaphore.WaitOne();
// await Utils.ParseModsAsync(l,cfg);
// }
// catch (Exception e) {
// Log.Logger.Error(e.Message);
// }
// finally {
// semaphore.Release();
// }
//}
}
}