public async Task Execute(IJobExecutionContext context) { // Get the values from the JobDataMap JobDataMap dataMap = context.JobDetail.JobDataMap; string namePrefix = dataMap.GetString(ARCHIVAL_DATA_PREFIX); string id = dataMap.GetString(ARCHIVAL_DATA_ID); string unformattedUrl = dataMap.GetString(ARCHIVAL_DATA_URL); // Format the S3 path string s3Path = $"/smash/{namePrefix}/{id}/page"; try { // Loop over every Language foreach (Language language in Nintendo.SmashUltimate.Bcat.Container.LanguageOrder) { // Get the URL for this language string url = unformattedUrl.Replace("{$lang}", language.GetCode()); // Perform an HTTP request using (HttpResponseMessage response = await httpClient.GetAsync(url)) using (HttpContent content = response.Content) { // Check for a success if (!response.IsSuccessStatusCode) { // Check for 404 if (response.StatusCode == HttpStatusCode.NotFound) { // Log await DiscordBot.LoggingChannel.SendMessageAsync($"**[ArchivalJob]** {language} for {namePrefix}_{id} not found"); continue; } // Unknown error throw new Exception("Archival failure on HTTP request (" + response.StatusCode + ")"); } // Get the content byte[] rawContent = await content.ReadAsByteArrayAsync(); // Write the data to S3 S3Api.TransferFile(rawContent, s3Path, language.GetCode() + ".html"); await DiscordBot.LoggingChannel.SendMessageAsync($"**[ArchivalJob]** Downloaded {language} for {namePrefix}_{id}"); } } } catch (Exception exception) { // Notify the logging channel await DiscordUtil.HandleException(exception, $"in ``ContainerArchivalJob``"); } }
protected override async Task Run() { await DiscordBot.LoggingChannel.SendMessageAsync($"**[WebPConversionOneTimeTask]** Starting WebP image conversion"); List <Container> allContainers = new List <Container>(); // Add all Containers to the List allContainers.AddRange(ContainerCache.GetEvents()); allContainers.AddRange(ContainerCache.GetPopUpNews()); allContainers.AddRange(ContainerCache.GetPresents()); // Loop over every Container foreach (Container container in allContainers) { // Get the FileType FileType fileType = FileTypeExtensions.GetTypeFromContainer(container); // Format the destination S3 path string s3Path = $"/smash/{FileTypeExtensions.GetNamePrefixFromType(fileType)}/{container.Id}"; // Get the raw image byte[] jpgImage = (byte[])container.GetType().GetProperty("Image").GetValue(container); // Create a new MagickImage using (MagickImage image = new MagickImage(jpgImage)) { // Set the output format to WebP image.Format = MagickFormat.WebP; // Create the raw WebP byte[] webpImage = image.ToByteArray(); await DiscordBot.LoggingChannel.SendMessageAsync($"**[WebPConversionOneTimeTask]** Uploading image for {fileType.ToString()} ID {container.Id}"); // Upload to S3 S3Api.TransferFile(webpImage, s3Path, "image.webp", "image/webp"); } } }
public static Task HandleFestival(RomType romType, Dictionary <string, byte[]> data, FestivalSetting previousFestival, FestivalSetting newFestival, byte[] rawFile) { // Check if this is the same if (previousFestival.FestivalId == newFestival.FestivalId) { //return Task.FromResult(0); } // Construct the path string s3Path = $"/splatoon/festival/{romType.ToString()}/{newFestival.FestivalId}"; // Deserialize the FestivalSetting dynamically dynamic settingDynamic = ByamlLoader.GetByamlDynamic(rawFile); // Serialize the FestivalSetting to JSON string json = JsonConvert.SerializeObject(settingDynamic); // Upload to S3 S3Api.TransferFile(Encoding.UTF8.GetBytes(json), s3Path, "setting.json", "application/json"); // Load the panel texture file using (MemoryStream panelStream = new MemoryStream(data[FileType.FestivalPanelTexture.GetPath()])) { // Parse the BFRES ResFile panelRes = new ResFile(panelStream); // Load the BNTX using (MemoryStream bntxStream = new MemoryStream(panelRes.ExternalFiles[0].Data)) { // Parse the BNTX BinaryTexture bt = new BinaryTexture(bntxStream); // Decode the first texture if (PixelDecoder.TryDecode(bt.Textures[0], out Bitmap Img)) { // Open a destination MemoryStream using (MemoryStream bitmapStream = new MemoryStream()) { // Write the bitmap as a PNG to the MemoryStream Img.Save(bitmapStream, ImageFormat.Png); // Get the data byte[] imageData = bitmapStream.ToArray(); // Upload to S3 S3Api.TransferFile(imageData, s3Path, "panel.png"); // Write to local File.WriteAllBytes(string.Format(FileCache.FESTIVAL_PANEL_PATH, romType.ToString(), newFestival.FestivalId), imageData); } } } } lock (WebFileHandler.Lock) { // Get the WebConfig JelonzoBotWebConfig webConfig = ((JelonzoBotConfiguration)Configuration.LoadedConfiguration).WebConfig; // Connect to the remote server if needed WebFileHandler.Connect(webConfig); // Format the container list path string manifestPath = webConfig.LatestFestivalManifestPath; // Check if the file exists LatestFestivalManifest manifest; if (WebFileHandler.Exists(manifestPath)) { // Deserialize the manifest manifest = WebFileHandler.ReadAllText <LatestFestivalManifest>(manifestPath); } else { // Create a new manifest manifest = new LatestFestivalManifest(); manifest.NorthAmerica = FileCache.GetLatestFestivalSettingForRomType(RomType.NorthAmerica).FestivalId; manifest.Europe = FileCache.GetLatestFestivalSettingForRomType(RomType.Europe).FestivalId; manifest.Japan = FileCache.GetLatestFestivalSettingForRomType(RomType.Japan).FestivalId; } // Update the manifest switch (romType) { case RomType.NorthAmerica: manifest.NorthAmerica = newFestival.FestivalId; break; case RomType.Europe: manifest.Europe = newFestival.FestivalId; break; case RomType.Japan: manifest.Japan = newFestival.FestivalId; break; default: throw new Exception("Invalid RomType"); } // Upload the manifest WebFileHandler.WriteSerializedJson(manifestPath, manifest); // Get the FestivalSetting JSON path string path = webConfig.FestivalSettingPath; switch (romType) { case RomType.NorthAmerica: path = string.Format(path, "na"); break; case RomType.Europe: path = string.Format(path, "eu"); break; case RomType.Japan: path = string.Format(path, "jp"); break; default: throw new Exception("Invalid RomType"); } // Upload to the server WebFileHandler.WriteAllText(path, json); // Disconnect WebFileHandler.Disconnect(); } return(Task.FromResult(0)); }
public async Task Execute(IJobExecutionContext context) { try { // Log that we're about to begin uploading data await DiscordBot.LoggingChannel.SendMessageAsync("**[RomDataUploadJob]** Beginning ROM data upload"); // Create a list of paths whose cache needs to be cleared List <string> clearPaths = new List <string>(); // Create the MSBT S3 path format string string msbtS3BasePath = "/splatoon/blitz_rom/Message/CommonMsg_{0}"; // Loop over every language foreach (Language language in BlitzUtil.SupportedLanguages) { // Log the language await DiscordBot.LoggingChannel.SendMessageAsync($"**[RomDataUploadJob]** Uploading {language.ToString()}'s MSBTs"); // Get the MsbtHolder MsbtHolder msbtHolder = BlitzLocalizer.MsbtHolders[language]; // Loop over every MSBT foreach (KeyValuePair <string, Dictionary <string, string> > pair in msbtHolder.Msbts) { // Construct the path string msbtPath = string.Format(msbtS3BasePath, language.GetSeadCode()); // Construct the file name string fileName = $"{pair.Key}.json"; // Serialize to JSON string json = JsonConvert.SerializeObject(pair.Value); // Upload to S3 S3Api.TransferFile(Encoding.UTF8.GetBytes(json), msbtPath, fileName, "application/json"); // Add to the cache list clearPaths.Add($"{msbtPath}/{fileName}"); } } // Log MSBT upload await DiscordBot.LoggingChannel.SendMessageAsync("**[RomDataUploadJob]** Uploading Mush"); // Create the Mush S3 path string string mushS3Path = "/splatoon/blitz_rom/Mush"; // Get every file in Mush foreach (string path in RomResourceLoader.GetFilesInDirectory("/Mush")) { // Get the BYAML dynamic byaml = ByamlLoader.GetByamlDynamic(path); // Construct the file name string fileName = $"{Path.GetFileNameWithoutExtension(path)}.json"; // Serialize to JSON string json = JsonConvert.SerializeObject(byaml); // Upload to S3 S3Api.TransferFile(Encoding.UTF8.GetBytes(json), mushS3Path, fileName, "application/json"); // Add to the paths to clear clearPaths.Add($"{mushS3Path}/{fileName}"); } // Log CDN cache purge starting await DiscordBot.LoggingChannel.SendMessageAsync($"**[RomDataUploadJob]** Requesting CDN cache purge ({clearPaths.Count} of {clearPaths.Count} files left)"); IEnumerable <string> pathsEnumerable = (IEnumerable <string>)clearPaths; while (pathsEnumerable.Any()) { // Tell DigitalOcean to clear the cache await DoApi.SendRequest(new DoCdnCachePurgeRequest(Configuration.LoadedConfiguration.DoConfig.EndpointId, pathsEnumerable.Take(15).ToList())); // Advance to the next set pathsEnumerable = pathsEnumerable.Skip(15); // Log files left await DiscordBot.LoggingChannel.SendMessageAsync($"**[RomDataUploadJob]** Requesting CDN cache purge ({pathsEnumerable.Count()} of {clearPaths.Count} files left)"); } // Write the app version await DiscordBot.LoggingChannel.SendMessageAsync("**[RomDataUploadJob]** Saving new ROM version to configuration"); (Configuration.LoadedConfiguration as JelonzoBotConfiguration).RomConfig.LastRomVersion = (int)context.JobDetail.JobDataMap["version"]; Configuration.LoadedConfiguration.Write(); // Log that it's complete await DiscordBot.LoggingChannel.SendMessageAsync("**[RomDataUploadJob]** ROM data upload complete"); } catch (Exception e) { await DiscordUtil.HandleException(e, "in ``RomDataUploadJob``"); } }
public static void HandleContainer(Container container) { // Get the FileType FileType fileType = FileTypeExtensions.GetTypeFromContainer(container); // Format the destination S3 path string s3Path = $"/smash/{FileTypeExtensions.GetNamePrefixFromType(fileType)}/{container.Id}"; // Convert the Container into a JSON string byte[] json = Encoding.UTF8.GetBytes(WebFileHandler.ToJson(container)); // Write the data to S3 S3Api.TransferFile(json, s3Path, "data.json", "application/json"); // Check if this has an image if (fileType == FileType.Event || fileType == FileType.PopUpNews || fileType == FileType.Present) { // Get the image byte[] image = (byte[])container.GetType().GetProperty("Image").GetValue(container); // Write the image to S3 S3Api.TransferFile(image, s3Path, "image.jpg", "image/jpeg"); // Create a new MagickImage using (MagickImage magickImage = new MagickImage(image)) { // Set the output format to WebP magickImage.Format = MagickFormat.WebP; // Create the raw WebP byte[] webpImage = magickImage.ToByteArray(); // Upload to S3 S3Api.TransferFile(webpImage, s3Path, "image.webp", "image/webp"); } } lock (WebFileHandler.Lock) { // Connect to the remote server if needed WebFileHandler.Connect(((SsbuBotConfiguration)Configuration.LoadedConfiguration).WebConfig); // Convert the Container to a StrippedContainer StrippedContainer strippedContainer = StrippedContainer.ConvertToStrippedContainer(container); // Declare a variable to hold the container list List <StrippedContainer> containerList; // Format the container list path string indexPath = string.Format(((SsbuBotConfiguration)Configuration.LoadedConfiguration).WebConfig.ContainerListPath, FileTypeExtensions.GetNamePrefixFromType(fileType)); // Check if the file exists if (WebFileHandler.Exists(indexPath)) { // Deserialize the List containerList = WebFileHandler.ReadAllText <List <StrippedContainer> >(indexPath); } else { // Create a new List containerList = new List <StrippedContainer>(); } // Check if the Container already exists in the list int index = containerList.FindIndex(x => x.Id == container.Id); // Check the index if (index == -1) { // Add the StrippedContainer to the List containerList.Insert(0, strippedContainer); } else { // Replace the item at the index containerList[index] = strippedContainer; } // Serialize and write the container list WebFileHandler.WriteAllText(indexPath, WebFileHandler.ToJson(containerList)); // Declare a variable to hold the ContainerIndex ContainerIndex containerIndex; // Check if the ContainerIndex exists if (!WebFileHandler.Exists(((SsbuBotConfiguration)Configuration.LoadedConfiguration).WebConfig.ContainerIndexPath)) { // Create a dummy StrippedContainer StrippedContainer dummyStrippedContainer = new StrippedContainer(); dummyStrippedContainer.Id = "-1"; dummyStrippedContainer.Text = new Dictionary <Nintendo.Bcat.Language, string>(); // Create a dummy ContainerIndex containerIndex = new ContainerIndex(); containerIndex.Event = dummyStrippedContainer; containerIndex.LineNews = dummyStrippedContainer; containerIndex.PopUpNews = dummyStrippedContainer; containerIndex.Present = dummyStrippedContainer; } else { // Read the file containerIndex = WebFileHandler.ReadAllText <ContainerIndex>(((SsbuBotConfiguration)Configuration.LoadedConfiguration).WebConfig.ContainerIndexPath); } // Get the correct property PropertyInfo propertyInfo = containerIndex.GetType().GetProperty(container.GetType().Name); // Set the value propertyInfo.SetValue(containerIndex, strippedContainer); // Write out the ContainerIndex WebFileHandler.WriteAllText(((SsbuBotConfiguration)Configuration.LoadedConfiguration).WebConfig.ContainerIndexPath, WebFileHandler.ToJson(containerIndex)); // Disconnect from the remote server WebFileHandler.Disconnect(); } }