protected override async Task ExecuteAsync(CancellationToken stoppingToken) { try { Logger.PrintHeader(ConsoleText); } catch { // ignored as header is not important } var manifest = await _client.DownloadManifest(); var fileslist = _parserInputFiles.Select(o => $"NostaleData{Path.DirectorySeparatorChar}{o}").ToList(); manifest.Entries = manifest.Entries.Where(s => fileslist.Contains(s.File)).ToArray(); await _client.DownloadClientAsync(manifest); foreach (var file in fileslist) { var rename = file.Contains("NScliData"); var dest = file.Contains("NStcData") ? $".{Path.DirectorySeparatorChar}output{Path.DirectorySeparatorChar}parser{Path.DirectorySeparatorChar}maps{Path.DirectorySeparatorChar}" : $".{Path.DirectorySeparatorChar}output{Path.DirectorySeparatorChar}parser{Path.DirectorySeparatorChar}"; var fileInfo = new FileInfo($".{Path.DirectorySeparatorChar}output{Path.DirectorySeparatorChar}{file}"); await _extractor.ExtractAsync(fileInfo, dest, rename); } var directoryOfFilesToBeTarred = new DirectoryInfo(".{Path.DirectorySeparatorChar}output{Path.DirectorySeparatorChar}parser"); var filesInDirectory = directoryOfFilesToBeTarred.GetFiles("*.*", SearchOption.AllDirectories); var tarArchiveName = $".{Path.DirectorySeparatorChar}output{Path.DirectorySeparatorChar}parser-input-files.tar.bz2"; if (File.Exists(tarArchiveName)) { File.Delete(tarArchiveName); } await using Stream targetStream = new BZip2OutputStream(File.Create(tarArchiveName)); using var tarArchive = TarArchive.CreateOutputTarArchive(targetStream, TarBuffer.DefaultBlockFactor); foreach (var fileToBeTarred in filesInDirectory) { var entry = TarEntry.CreateEntryFromFile(fileToBeTarred.FullName); tarArchive.WriteEntry(entry, true); } tarArchive.Dispose(); _logger.LogInformation(LogLanguage.Instance.GetMessageFromKey(LogLanguageKey.PARSER_INPUT_GENERATED)); }
public async Task UploadInputFilesAsync() { using var client = new AmazonS3Client(new BasicAWSCredentials( Environment.GetEnvironmentVariable("S3_ACCESS_KEY"), Environment.GetEnvironmentVariable("S3_SECRET_KEY")), RegionEndpoint.USWest2); var manifest = await _client.DownloadManifest(); manifest.Entries = manifest.Entries.Select(s => { s.File = s.File.Replace('\\', Path.DirectorySeparatorChar); return(s); }).ToArray(); var fileslist = _parserInputFiles.Select(o => $"NostaleData{Path.DirectorySeparatorChar}{o}").ToList(); manifest.Entries = manifest.Entries.Where(s => fileslist.Contains(s.File)).ToArray(); var request = new GetObjectRequest { BucketName = Environment.GetEnvironmentVariable("S3_BUCKET"), Key = "clientmanifest.json", }; ClientManifest previousManifest; try { { using var response = await client.GetObjectAsync(request); await using var responseStream = response.ResponseStream; using var reader = new StreamReader(responseStream); previousManifest = JsonConvert.DeserializeObject <ClientManifest>(await reader.ReadToEndAsync()); } } catch { previousManifest = new ClientManifest() { Entries = new Entry[0] }; } var previoussha1s = previousManifest.Entries.Select(s => s.Sha1); if (!manifest.Entries.Select(s => s.Sha1).All(s => previoussha1s.Contains(s))) { var archiveName = $".{Path.DirectorySeparatorChar}output{Path.DirectorySeparatorChar}parser-input-files.tar.bz2"; await _client.DownloadClientAsync(manifest); await Task.WhenAll(fileslist.Select(file => { var rename = file.Contains("NScliData"); var dest = file.Contains("NStcData") ? $".{Path.DirectorySeparatorChar}output{Path.DirectorySeparatorChar}parser{Path.DirectorySeparatorChar}map{Path.DirectorySeparatorChar}" : $".{Path.DirectorySeparatorChar}output{Path.DirectorySeparatorChar}parser{Path.DirectorySeparatorChar}"; var fileInfo = new FileInfo($".{Path.DirectorySeparatorChar}output{Path.DirectorySeparatorChar}{file}"); return(_extractor.ExtractAsync(fileInfo, dest, rename)); })); var directoryOfFilesToBeTarred = $".{Path.DirectorySeparatorChar}output{Path.DirectorySeparatorChar}parser"; var filesInDirectory = Directory.GetFiles(directoryOfFilesToBeTarred, "*.*", SearchOption.AllDirectories); if (File.Exists(archiveName)) { File.Delete(archiveName); } await Task.Delay(10000); { await using var targetStream = new BZip2OutputStream(File.Create(archiveName)); using var tarArchive = TarArchive.CreateOutputTarArchive(targetStream, TarBuffer.DefaultBlockFactor); foreach (var file in filesInDirectory) { var entry = TarEntry.CreateEntryFromFile(file); tarArchive.WriteEntry(entry, true); } } var emptyfile = JsonConvert.SerializeObject(manifest); { await using var stream = new MemoryStream(Encoding.UTF8.GetBytes(emptyfile)); var putRequest = new PutObjectRequest { BucketName = Environment.GetEnvironmentVariable("S3_BUCKET"), Key = "clientmanifest.json", ContentType = "text/json", InputStream = stream }; await client.PutObjectAsync(putRequest); } if (_discord.GetChannel(719772084968095775) is SocketTextChannel channel) { var file = new FileInfo(archiveName); if (file.Length > 8388119) { var send = await channel.SendMessageAsync($"<:altz:699420721088168036><:altz:699420721088168036><:altz:699420721088168036>Parser Too Heavy<:altz:699420721088168036><:altz:699420721088168036><:altz:699420721088168036>\n - Size : {file.Length}"); } else { var alq = string.Concat(Enumerable.Repeat("<:altq:699420721130242159>", 20)); var send = await channel.SendFileAsync(archiveName, $"{alq}\n<:altp:699420720819732651><:altp:699420720819732651><:altp:699420720819732651><:altp:699420720819732651><:altp:699420720819732651><:altp:699420720819732651>PARSER FILES GENERATED<:altp:699420720819732651><:altp:699420720819732651><:altp:699420720819732651><:altp:699420720819732651><:altp:699420720819732651><:altp:699420720819732651>\n{alq}"); } } } }