public void RegisterFileTypeItem(IDocumentTypeItem fileType)
{
if (FileTypeExtensions == null)
{
FileTypeExtensions = new List <IDocumentTypeItem>();
}
FileTypeExtensions.Add(fileType);
}
protected override async Task Run()
{
await DiscordBot.LoggingChannel.SendMessageAsync($"**[WebPConversionOneTimeTask]** Starting WebP image conversion");
List <Container> allContainers = new List <Container>();
// Add all Containers to the List
allContainers.AddRange(ContainerCache.GetEvents());
allContainers.AddRange(ContainerCache.GetPopUpNews());
allContainers.AddRange(ContainerCache.GetPresents());
// Loop over every Container
foreach (Container container in allContainers)
{
// Get the FileType
FileType fileType = FileTypeExtensions.GetTypeFromContainer(container);
// Format the destination S3 path
string s3Path = $"/smash/{FileTypeExtensions.GetNamePrefixFromType(fileType)}/{container.Id}";
// Get the raw image
byte[] jpgImage = (byte[])container.GetType().GetProperty("Image").GetValue(container);
// Create a new MagickImage
using (MagickImage image = new MagickImage(jpgImage))
{
// Set the output format to WebP
image.Format = MagickFormat.WebP;
// Create the raw WebP
byte[] webpImage = image.ToByteArray();
await DiscordBot.LoggingChannel.SendMessageAsync($"**[WebPConversionOneTimeTask]** Uploading image for {fileType.ToString()} ID {container.Id}");
// Upload to S3
S3Api.TransferFile(webpImage, s3Path, "image.webp", "image/webp");
}
}
}
public static async Task <bool> ScheduleArchival(Container container, DateTime dateTime)
{
// Check if this is an event PopUpNews
PopUpNews popUpNews = container as PopUpNews;
if (popUpNews != null && popUpNews.IsPopUpForEvent)
{
// Skip
return(false);
}
// Create the job name
string jobName = container.GetType().Name + container.Id;
// Get the Url
string url = container.GetFormattedUrl();
if (url.Length == 0)
{
// Skip
return(false);
}
// Create the JobDataMap
JobDataMap dataMap = new JobDataMap();
dataMap.Put(ContainerArchivalJob.ARCHIVAL_DATA_PREFIX, FileTypeExtensions.GetNamePrefixFromType(FileTypeExtensions.GetTypeFromContainer(container)));
dataMap.Put(ContainerArchivalJob.ARCHIVAL_DATA_ID, container.Id);
dataMap.Put(ContainerArchivalJob.ARCHIVAL_DATA_URL, url);
// Schedule the job for this Container
await QuartzScheduler.ScheduleJob <ContainerArchivalJob>(jobName, dateTime, dataMap);
await DiscordBot.LoggingChannel.SendMessageAsync($"**[ArchivalHandler]** Scheduled archival job for {container.GetType().Name} ({container.Id})");
return(true);
}
public async Task Execute(IJobExecutionContext context)
{
try
{
// Log that we're about to begin a check
await DiscordBot.LoggingChannel.SendMessageAsync("**[BCAT]** Beginning check");
// Download the latest Topic
Topic topic = await BcatApi.GetDataTopic(Program.TITLE_ID, Program.PASSPHRASE);
// Create the target folder name
string targetFolder = string.Format(Program.LOCAL_OLD_DATA_DIRECTORY, DateTime.Now.ToString(Program.FOLDER_DATE_TIME_FORMAT));
// Check if this the first run
if (!Configuration.LoadedConfiguration.FirstRunCompleted)
{
// Log that this is the first run of BCAT
await DiscordBot.LoggingChannel.SendMessageAsync("**[BCAT]** First run");
// Download all data
Dictionary <string, byte[]> downloadedData = await BcatCheckerUtils.DownloadAllData(topic, Program.TITLE_ID, Program.PASSPHRASE, targetFolder);
// Loop over all data
foreach (KeyValuePair <string, byte[]> pair in downloadedData)
{
// Get the FileType
FileType fileType = FileTypeExtensions.GetTypeFromName(pair.Key);
// Check if this is a container
if (fileType.IsContainer())
{
// Create a Container instance
Nintendo.SmashUltimate.Bcat.Container container = CreateSmashContainerInstance(fileType, pair.Value);
// Add this to the container cache
ContainerCache.AddFile(container, Path.GetFileName(pair.Key), pair.Value);
}
else
{
// Write this file out to the common file cache
File.WriteAllBytes(Path.Combine(Program.LOCAL_COMMON_CACHE_DIRECTORY, Path.GetFileName(pair.Key)), pair.Value);
}
}
// Save the configuration
Configuration.LoadedConfiguration.FirstRunCompleted = true;
Configuration.LoadedConfiguration.Write();
// Write out the topic
File.WriteAllBytes(Program.LOCAL_LAST_TOPIC, MessagePackSerializer.Serialize(topic));
// Log that we're about to begin a check
await DiscordBot.LoggingChannel.SendMessageAsync("**[BCAT]** First run complete");
return;
}
// Load the old Topic
Topic oldTopic = MessagePackSerializer.Deserialize <Topic>(File.ReadAllBytes(Program.LOCAL_LAST_TOPIC));
#if DEBUG
if (!Configuration.LoadedConfiguration.IsProduction)
{
/*foreach (Bcat.Directory dir in oldTopic.Directories)
* {
* if (dir.Name == "line_news")
* {
* Data dbgData = dir.Data.FirstOrDefault();
* if (dbgData == null)
* {
* continue;
* }
* dir.Data.Remove(dbgData);
* //dbgData.Digest = "deadbeef";
* }
* }*/
}
#endif
// Get the differences
List <KeyValuePair <DifferenceType, string> > differences = BcatCheckerUtils.GetTopicChanges(oldTopic, topic);
// Check if there aren't any
if (differences.Count == 0)
{
// Nothing to do here
goto finished;
}
// Download all data
Dictionary <string, byte[]> data = await BcatCheckerUtils.DownloadAllData(topic, Program.TITLE_ID, Program.PASSPHRASE, targetFolder);
// Loop over every difference
foreach (KeyValuePair <DifferenceType, string> pair in differences)
{
// Log the difference
await DiscordBot.LoggingChannel.SendMessageAsync("**[BCAT]** diff: ``" + pair.Value + "`` (" + pair.Key.ToString() + ")");
// Get the FileType
FileType fileType = FileTypeExtensions.GetTypeFromName(pair.Value);
// Declare a variable to hold the method parameters
object[] parameters;
if (pair.Key == DifferenceType.Added)
{
// Get the raw file
byte[] rawFile = data[pair.Value];
// Check if this is a Container
if (fileType.IsContainer())
{
// Create a Container instance
Nintendo.SmashUltimate.Bcat.Container addedContainer = CreateSmashContainerInstance(fileType, data[pair.Value]);
// Add this to the container cache
ContainerCache.AddFile(addedContainer, Path.GetFileName(pair.Value), rawFile);
// Set the method parameters
parameters = new object[] { addedContainer };
}
else
{
// Write this file out to the common file cache
File.WriteAllBytes(Path.Combine(Program.LOCAL_COMMON_CACHE_DIRECTORY, Path.GetFileName(pair.Value)), data[pair.Value]);
// Set the method parameters
parameters = new object[] { rawFile };
}
}
else if (pair.Key == DifferenceType.Changed)
{
// Get the raw file
byte[] rawFile = data[pair.Value];
// Check if this is a Container
if (fileType.IsContainer())
{
// Create a Container instance
Nintendo.SmashUltimate.Bcat.Container addedContainer = CreateSmashContainerInstance(fileType, data[pair.Value]);
// Overwrite this to the container cache
Nintendo.SmashUltimate.Bcat.Container previousContainer = ContainerCache.OverwriteFile(addedContainer, Path.GetFileName(pair.Value), rawFile);
// Set the method parameters
parameters = new object[] { previousContainer, addedContainer };
}
else
{
// Construct the commoon cache path
string path = Path.Combine(Program.LOCAL_COMMON_CACHE_DIRECTORY, Path.GetFileName(pair.Value));
// Load the old file
byte[] previousRawFile = File.ReadAllBytes(path);
// Write this file out to the common file cache
File.WriteAllBytes(path, data[pair.Value]);
// Set the method parameters
parameters = new object[] { previousRawFile, rawFile };
}
}
else // Removed
{
// TODO: discord print
continue;
}
// Call every difference handler
await BcatCheckerUtils.CallDifferenceHandlers((int)fileType, pair.Key, parameters);
}
// Write out the Topic
File.WriteAllBytes(Program.LOCAL_LAST_TOPIC, MessagePackSerializer.Serialize(topic));
finished:
await DiscordBot.LoggingChannel.SendMessageAsync("**[BCAT]** Check complete");
}
catch (Exception exception)
{
// Notify the logging channel
await DiscordUtil.HandleException(exception, $"in ``BcatCheckerJob``");
}
}
public async Task Execute(IJobExecutionContext context)
{
try
{
if (Configuration.LoadedConfiguration.FirstRunCompleted)
{
throw new Exception("Attempting to do first run more than once");
}
foreach (KeyValuePair <RomType, BcatPair> bcatPairEntry in Program.BcatPairs)
{
// Log that we're about to begin a check
await DiscordBot.LoggingChannel.SendMessageAsync("**[BCAT]** Beginning topic download for " + bcatPairEntry.Key.ToString());
// Download the latest Topic
Topic topic = await BcatApi.GetDataTopic(bcatPairEntry.Value.TitleId, bcatPairEntry.Value.Passphrase);
// Create the target folder name
string targetFolder = string.Format(Program.LOCAL_OLD_DATA_DIRECTORY, DateTime.Now.ToString(Program.FOLDER_DATE_TIME_FORMAT), bcatPairEntry.Key.ToString());
// Download all data
Dictionary <string, byte[]> downloadedData = await BcatCheckerUtils.DownloadAllData(topic, bcatPairEntry.Value.TitleId, bcatPairEntry.Value.Passphrase, targetFolder);
// Loop over all data
foreach (KeyValuePair <string, byte[]> dataPair in downloadedData)
{
// Get the FileType
FileType fileType = FileTypeExtensions.GetTypeFromFilePath(dataPair.Key);
// Populate the FileCache directories based on the FileType
string path;
switch (fileType)
{
case FileType.VersusSetting:
path = string.Format(FileCache.VERSUS_SETTING_PATH, bcatPairEntry.Key.ToString());
break;
case FileType.CoopSetting:
// Only write the CoopSetting file once
if (bcatPairEntry.Key != RomType.NorthAmerica)
{
continue;
}
path = FileCache.COOP_SETTING_PATH;
break;
case FileType.FestivalByaml:
// Deserialize the byaml to get the ID
dynamic byaml = ByamlLoader.GetByamlDynamic(dataPair.Value);
// Generate the path
path = string.Format(FileCache.FESTIVAL_SETTING_PATH, bcatPairEntry.Key.ToString(), byaml["FestivalId"]);
break;
default:
continue;
}
// Create the directories if needed
System.IO.Directory.CreateDirectory(Path.GetDirectoryName(path));
// Write the file
File.WriteAllBytes(path, dataPair.Value);
}
// Write out the topic
File.WriteAllBytes(string.Format(Program.LOCAL_LAST_TOPIC, bcatPairEntry.Key.ToString()), MessagePackSerializer.Serialize(topic));
// Set the last data download directory
(Configuration.LoadedConfiguration as JelonzoBotConfiguration).LastDownloadPaths[bcatPairEntry.Key] = targetFolder;
// Log that the first run is done
await DiscordBot.LoggingChannel.SendMessageAsync($"**[BCAT]** First run complete for {bcatPairEntry.Key.ToString()}");
}
// Save the configuration
Configuration.LoadedConfiguration.FirstRunCompleted = true;
Configuration.LoadedConfiguration.Write();
// Initialize the FileCache
FileCache.Initialize();
}
catch (Exception exception)
{
// Notify the logging channel
await DiscordUtil.HandleException(exception, $"in ``BcatCheckerJob``");
}
await QuartzScheduler.ScheduleJob <BcatCheckerJob>("Normal", Configuration.LoadedConfiguration.JobSchedules["Bcat"]);
}
public static void HandleContainer(Container container)
{
// Get the FileType
FileType fileType = FileTypeExtensions.GetTypeFromContainer(container);
// Format the destination S3 path
string s3Path = $"/smash/{FileTypeExtensions.GetNamePrefixFromType(fileType)}/{container.Id}";
// Convert the Container into a JSON string
byte[] json = Encoding.UTF8.GetBytes(WebFileHandler.ToJson(container));
// Write the data to S3
S3Api.TransferFile(json, s3Path, "data.json", "application/json");
// Check if this has an image
if (fileType == FileType.Event || fileType == FileType.PopUpNews || fileType == FileType.Present)
{
// Get the image
byte[] image = (byte[])container.GetType().GetProperty("Image").GetValue(container);
// Write the image to S3
S3Api.TransferFile(image, s3Path, "image.jpg", "image/jpeg");
// Create a new MagickImage
using (MagickImage magickImage = new MagickImage(image))
{
// Set the output format to WebP
magickImage.Format = MagickFormat.WebP;
// Create the raw WebP
byte[] webpImage = magickImage.ToByteArray();
// Upload to S3
S3Api.TransferFile(webpImage, s3Path, "image.webp", "image/webp");
}
}
lock (WebFileHandler.Lock)
{
// Connect to the remote server if needed
WebFileHandler.Connect(((SsbuBotConfiguration)Configuration.LoadedConfiguration).WebConfig);
// Convert the Container to a StrippedContainer
StrippedContainer strippedContainer = StrippedContainer.ConvertToStrippedContainer(container);
// Declare a variable to hold the container list
List <StrippedContainer> containerList;
// Format the container list path
string indexPath = string.Format(((SsbuBotConfiguration)Configuration.LoadedConfiguration).WebConfig.ContainerListPath, FileTypeExtensions.GetNamePrefixFromType(fileType));
// Check if the file exists
if (WebFileHandler.Exists(indexPath))
{
// Deserialize the List
containerList = WebFileHandler.ReadAllText <List <StrippedContainer> >(indexPath);
}
else
{
// Create a new List
containerList = new List <StrippedContainer>();
}
// Check if the Container already exists in the list
int index = containerList.FindIndex(x => x.Id == container.Id);
// Check the index
if (index == -1)
{
// Add the StrippedContainer to the List
containerList.Insert(0, strippedContainer);
}
else
{
// Replace the item at the index
containerList[index] = strippedContainer;
}
// Serialize and write the container list
WebFileHandler.WriteAllText(indexPath, WebFileHandler.ToJson(containerList));
// Declare a variable to hold the ContainerIndex
ContainerIndex containerIndex;
// Check if the ContainerIndex exists
if (!WebFileHandler.Exists(((SsbuBotConfiguration)Configuration.LoadedConfiguration).WebConfig.ContainerIndexPath))
{
// Create a dummy StrippedContainer
StrippedContainer dummyStrippedContainer = new StrippedContainer();
dummyStrippedContainer.Id = "-1";
dummyStrippedContainer.Text = new Dictionary <Nintendo.Bcat.Language, string>();
// Create a dummy ContainerIndex
containerIndex = new ContainerIndex();
containerIndex.Event = dummyStrippedContainer;
containerIndex.LineNews = dummyStrippedContainer;
containerIndex.PopUpNews = dummyStrippedContainer;
containerIndex.Present = dummyStrippedContainer;
}
else
{
// Read the file
containerIndex = WebFileHandler.ReadAllText <ContainerIndex>(((SsbuBotConfiguration)Configuration.LoadedConfiguration).WebConfig.ContainerIndexPath);
}
// Get the correct property
PropertyInfo propertyInfo = containerIndex.GetType().GetProperty(container.GetType().Name);
// Set the value
propertyInfo.SetValue(containerIndex, strippedContainer);
// Write out the ContainerIndex
WebFileHandler.WriteAllText(((SsbuBotConfiguration)Configuration.LoadedConfiguration).WebConfig.ContainerIndexPath, WebFileHandler.ToJson(containerIndex));
// Disconnect from the remote server
WebFileHandler.Disconnect();
}
}
public async Task Execute(IJobExecutionContext context)
{
try
{
foreach (KeyValuePair <RomType, BcatPair> bcatPairEntry in Program.BcatPairs)
{
// Log that we're about to begin a check
await DiscordBot.LoggingChannel.SendMessageAsync("**[BCAT]** Beginning topic download for " + bcatPairEntry.Key.ToString());
// Download the latest Topic
Topic topic = await BcatApi.GetDataTopic(bcatPairEntry.Value.TitleId, bcatPairEntry.Value.Passphrase);
// Create the target folder name
string targetFolder = string.Format(Program.LOCAL_OLD_DATA_DIRECTORY, DateTime.Now.ToString(Program.FOLDER_DATE_TIME_FORMAT), bcatPairEntry.Key.ToString());
// Format the old Topic path
string oldTopicPath = string.Format(Program.LOCAL_LAST_TOPIC, bcatPairEntry.Key.ToString());
// Load the old Topic
Topic oldTopic = MessagePackSerializer.Deserialize <Topic>(File.ReadAllBytes(oldTopicPath));
#if DEBUG
if (!Configuration.LoadedConfiguration.IsProduction)
{
/*foreach (Nintendo.Bcat.Directory dir in oldTopic.Directories)
* {
* if (dir.Name == "fesdata")
* {
* Data dbgData = dir.Data.Where(x => x.Name == "Festival.byaml").FirstOrDefault();
* dbgData.Digest = "deadbeef";
* }
* }*/
}
#endif
// Get the differences
List <KeyValuePair <DifferenceType, string> > differences = BcatCheckerUtils.GetTopicChanges(oldTopic, topic);
// Check if there aren't any
if (differences.Count == 0)
{
// Nothing to do here
goto finished;
}
// Download all data
Dictionary <string, byte[]> data = await BcatCheckerUtils.DownloadAllData(topic, bcatPairEntry.Value.TitleId, bcatPairEntry.Value.Passphrase, targetFolder);
// Loop over every difference
foreach (KeyValuePair <DifferenceType, string> differencePair in differences)
{
// Log the difference
await DiscordBot.LoggingChannel.SendMessageAsync("**[BCAT]** diff: ``" + differencePair.Value + "`` (" + differencePair.Key.ToString() + ")");
// Get the FileType
FileType fileType = FileTypeExtensions.GetTypeFromFilePath(differencePair.Value);
// Create a ByamlSerializer instance
ByamlSerializer serializer = new ByamlSerializer(new ByamlSerializerSettings()
{
ByteOrder = ByteOrder.LittleEndian,
SupportPaths = false,
Version = ByamlVersion.Version1
});
// Create the difference handler parameters based off the FileType and difference type
object[] parameters;
if (differencePair.Key != DifferenceType.Removed)
{
// Get the raw file
byte[] rawFile = data[differencePair.Value];
// Deserialize the object from byaml if necessary
object deserializedObject;
using (MemoryStream memoryStream = new MemoryStream(rawFile))
{
switch (fileType)
{
case FileType.VersusSetting:
deserializedObject = serializer.Deserialize <VersusSetting>(memoryStream);
break;
case FileType.CoopSetting:
deserializedObject = serializer.Deserialize <CoopSetting>(memoryStream);
break;
case FileType.FestivalByaml:
deserializedObject = serializer.Deserialize <FestivalSetting>(memoryStream);
break;
default:
deserializedObject = data[differencePair.Value];
break;
}
}
if (differencePair.Key == DifferenceType.Added)
{
parameters = new object[] { bcatPairEntry.Key, data, deserializedObject, rawFile };
}
else // Changed
{
// Get the previous file
object previousFile;
switch (fileType)
{
case FileType.VersusSetting:
previousFile = FileCache.GetVersusSettingForRomType(bcatPairEntry.Key);
break;
case FileType.CoopSetting:
previousFile = FileCache.GetCoopSetting();
break;
case FileType.FestivalByaml:
previousFile = FileCache.GetLatestFestivalSettingForRomType(bcatPairEntry.Key);
break;
default:
// Construct the previous file path
string previousPath = Path.Combine((Configuration.LoadedConfiguration as JelonzoBotConfiguration).LastDownloadPaths[bcatPairEntry.Key], differencePair.Value.Replace('/', Path.DirectorySeparatorChar));
// Load it
previousFile = File.ReadAllBytes(previousPath);;
break;
}
// Create the parameters
parameters = new object[] { bcatPairEntry.Key, data, previousFile, deserializedObject, rawFile };
}
}
else
{
parameters = new object[] { bcatPairEntry.Key, data };
}
// Call every difference handler
await BcatCheckerUtils.CallDifferenceHandlers((int)fileType, differencePair.Key, parameters);
}
// Write out the Topic
File.WriteAllBytes(oldTopicPath, MessagePackSerializer.Serialize(topic));
// Set the last download path
(Configuration.LoadedConfiguration as JelonzoBotConfiguration).LastDownloadPaths[bcatPairEntry.Key] = targetFolder;
finished:
await DiscordBot.LoggingChannel.SendMessageAsync("**[BCAT]** Check complete for " + bcatPairEntry.Key.ToString());
}
}
catch (Exception exception)
{
// Notify the logging channel
await DiscordUtil.HandleException(exception, $"in ``BcatCheckerJob``");
}
}
public static void Initialize(string cachePath)
{
// Check initialization
if (Initialized)
{
throw new Exception("Cannot initialize when already initialized");
}
// Set path
CachePath = cachePath;
// Create Dictionary and Lists
Events = new List <Event>();
LineNews = new List <LineNews>();
PopUpNews = new List <PopUpNews>();
Presents = new List <Present>();
// Check if the path doesn't exist
if (!Directory.Exists(CachePath))
{
// Create the Directory
Directory.CreateDirectory(CachePath);
return;
}
// Load every file in the cache path
foreach (string path in Directory.GetFiles(CachePath))
{
// Get the FileType
FileType fileType = FileTypeExtensions.GetTypeFromName(Path.GetFileName(path));
// Open a FileStream
using (FileStream fileStream = new FileStream(path, FileMode.Open))
{
// Create the appropriate class and add to the correct List
switch (fileType)
{
case FileType.Event:
Events.Add(new Event(fileStream));
break;
case FileType.LineNews:
LineNews.Add(new LineNews(fileStream));
break;
case FileType.PopUpNews:
PopUpNews.Add(new PopUpNews(fileStream));
break;
case FileType.Present:
Presents.Add(new Present(fileStream));
break;
}
}
}
// Sort the containers
Events.Sort((x, y) => y.StartDateTime.CompareTo(x.StartDateTime));
LineNews.Sort((x, y) => int.Parse(y.Id).CompareTo(int.Parse(x.Id)));
PopUpNews.Sort((x, y) => y.StartDateTime.CompareTo(x.StartDateTime));
Presents.Sort((x, y) => y.StartDateTime.CompareTo(x.StartDateTime));
// Set initialized
Initialized = true;
}
