public static string FetchExeTransform(string name)
{
using var wc = new System.Net.WebClient();
string moddesc = wc.DownloadStringAwareOfEncoding(ExeTransformBaseURL + name);
return(moddesc);
}
/// <summary>
/// Fetch latest version information (manifest attribute) from ME3Tweaks Updater Service. This should not be used for true update checks, use CheckForModUpdates() for that purpose.
/// </summary>
/// <param name="updatecode">Code to check</param>
/// <returns>verison string if found and parsable, null otherwise</returns>
public static Version GetLatestVersionOfModOnUpdaterService(int updatecode)
{
if (updatecode <= 0)
{
return(null); //invalid
}
string updateFinalRequest = UpdaterServiceManifestEndpoint + "?classicupdatecode[]=" + updatecode;
using var wc = new System.Net.WebClient();
try
{
string updatexml = wc.DownloadStringAwareOfEncoding(updateFinalRequest);
XElement rootElement = XElement.Parse(updatexml);
var modUpdateInfos = (from e in rootElement.Elements("mod")
select new ModUpdateInfo
{
versionstr = (string)e.Attribute("version")
}).ToList();
if (modUpdateInfos.Count == 1 && Version.TryParse(modUpdateInfos[0].versionstr, out var ver))
{
return(ver);
}
}
catch (Exception e)
{
Log.Error("Unable to fetch latest version of mod on updater service: " + e.Message);
}
return(null);
}
public static string FetchThirdPartyModdesc(string name)
{
using var wc = new System.Net.WebClient();
string moddesc = wc.DownloadStringAwareOfEncoding(ThirdPartyModDescURL + name);
return(moddesc);
}
public static Dictionary <long, List <ThirdPartyServices.ThirdPartyImportingInfo> > FetchThirdPartyImportingService(bool overrideThrottling = false)
{
if (!File.Exists(Utilities.GetThirdPartyImportingCachedFile()) || overrideThrottling || Utilities.CanFetchContentThrottleCheck())
{
try
{
using var wc = new System.Net.WebClient();
string json = wc.DownloadStringAwareOfEncoding(ThirdPartyImportingServiceURL);
File.WriteAllText(Utilities.GetThirdPartyImportingCachedFile(), json);
return(JsonConvert.DeserializeObject <Dictionary <long, List <ThirdPartyServices.ThirdPartyImportingInfo> > >(json));
}
catch (Exception e)
{
//Unable to fetch latest help.
Log.Error("Error fetching latest tips service file: " + e.Message);
if (File.Exists(Utilities.GetThirdPartyImportingCachedFile()))
{
Log.Warning("Using cached third party importing service file instead");
return(JsonConvert.DeserializeObject <Dictionary <long, List <ThirdPartyServices.ThirdPartyImportingInfo> > >(File.ReadAllText(Utilities.GetThirdPartyImportingCachedFile())));
}
else
{
Log.Error("Unable to fetch latest third party importing service file from server and local file doesn't exist. Returning a blank copy.");
return(new Dictionary <long, List <ThirdPartyServices.ThirdPartyImportingInfo> >());
}
}
}
return(JsonConvert.DeserializeObject <Dictionary <long, List <ThirdPartyServices.ThirdPartyImportingInfo> > >(File.ReadAllText(Utilities.GetThirdPartyImportingCachedFile())));
}
public static List <SortableHelpElement> FetchLatestHelp(bool overrideThrottling = false)
{
if (!File.Exists(Utilities.GetLocalHelpFile()) || (!overrideThrottling && Utilities.CanFetchContentThrottleCheck()))
{
try
{
using var wc = new System.Net.WebClient();
string xml = wc.DownloadStringAwareOfEncoding(LatestHelpFileLink);
File.WriteAllText(Utilities.GetLocalHelpFile(), xml);
return(ParseLocalHelp(xml));
}
catch (Exception e)
{
//Unable to fetch latest help.
Log.Error("Error fetching online help: " + e.Message);
if (File.Exists(Utilities.GetLocalHelpFile()))
{
Log.Warning("Using cached help instead");
return(ParseLocalHelp(File.ReadAllText(Utilities.GetLocalHelpFile())));
}
else
{
Log.Error("Unable to display dynamic help: Could not fetch online asset and cached help asset does not exist.");
return(null);
}
}
}
return(ParseLocalHelp(File.ReadAllText(Utilities.GetLocalHelpFile())));
}
public static Dictionary <string, CaseInsensitiveDictionary <ThirdPartyServices.ThirdPartyModInfo> > FetchThirdPartyIdentificationManifest(bool overrideThrottling = false)
{
if (!File.Exists(Utilities.GetThirdPartyIdentificationCachedFile()) || (!overrideThrottling && Utilities.CanFetchContentThrottleCheck()))
{
try
{
using var wc = new System.Net.WebClient();
string json = wc.DownloadStringAwareOfEncoding(ThirdPartyIdentificationServiceURL);
File.WriteAllText(Utilities.GetThirdPartyIdentificationCachedFile(), json);
return(JsonConvert.DeserializeObject <Dictionary <string, CaseInsensitiveDictionary <ThirdPartyServices.ThirdPartyModInfo> > >(json));
}
catch (Exception e)
{
//Unable to fetch latest help.
Log.Error("Error fetching online third party identification service: " + e.Message);
if (File.Exists(Utilities.GetThirdPartyIdentificationCachedFile()))
{
Log.Warning("Using cached third party identification service instead");
return(JsonConvert.DeserializeObject <Dictionary <string, CaseInsensitiveDictionary <ThirdPartyServices.ThirdPartyModInfo> > >(File.ReadAllText(Utilities.GetThirdPartyIdentificationCachedFile())));
}
else
{
Log.Error("Unable to load third party identification service and local file doesn't exist. Returning a blank copy.");
Dictionary <string, CaseInsensitiveDictionary <ThirdPartyServices.ThirdPartyModInfo> > d = new Dictionary <string, CaseInsensitiveDictionary <ThirdPartyServices.ThirdPartyModInfo> >();
d["ME1"] = new CaseInsensitiveDictionary <ThirdPartyServices.ThirdPartyModInfo>();
d["ME2"] = new CaseInsensitiveDictionary <ThirdPartyServices.ThirdPartyModInfo>();
d["ME3"] = new CaseInsensitiveDictionary <ThirdPartyServices.ThirdPartyModInfo>();
return(d);
}
}
}
return(JsonConvert.DeserializeObject <Dictionary <string, CaseInsensitiveDictionary <ThirdPartyServices.ThirdPartyModInfo> > >(File.ReadAllText(Utilities.GetThirdPartyIdentificationCachedFile())));
}
public static string FetchRemoteString(string url)
{
try
{
using var wc = new System.Net.WebClient();
return(wc.DownloadStringAwareOfEncoding(url));
} catch (Exception e)
{
Log.Error("Error downloading string: " + e.Message);
return(null);
}
}
public static Dictionary <string, string> QueryModRelay(string md5, long size)
{
//Todo: Finish implementing relay service
string finalRelayURL = $"{ModInfoRelayEndpoint}?modmanagerversion={App.BuildNumber}&md5={md5.ToLowerInvariant()}&size={size}";
try
{
using (var wc = new System.Net.WebClient())
{
Debug.WriteLine(finalRelayURL);
string json = wc.DownloadStringAwareOfEncoding(finalRelayURL);
//todo: Implement response format serverside
return(JsonConvert.DeserializeObject <Dictionary <string, string> >(json));
}
}
catch (Exception e)
{
Log.Error("Error querying relay service from ME3Tweaks: " + App.FlattenException(e));
}
return(null);
}
/// <summary>
/// Checks mods for updates. ForceUpdateCheck will force the mod to validate against the server (essentially repair mode). It is not used for rate limiting!
/// </summary>
/// <param name="modsToCheck">Mods to have server send information about</param>
/// <param name="forceUpdateCheck">Force update check regardless of version</param>
/// <returns></returns>
public static List <ModUpdateInfo> CheckForModUpdates(List <Mod> modsToCheck, bool forceUpdateCheck)
{
string updateFinalRequest = UpdaterServiceManifestEndpoint;
bool first = true;
foreach (var mod in modsToCheck)
{
if (mod.ModModMakerID > 0)
{
//Modmaker style
if (first)
{
updateFinalRequest += "?";
first = false;
}
else
{
updateFinalRequest += "&";
}
updateFinalRequest += "modmakerupdatecode[]=" + mod.ModModMakerID;
}
else if (mod.ModClassicUpdateCode > 0)
{
//Classic style
if (first)
{
updateFinalRequest += "?";
first = false;
}
else
{
updateFinalRequest += "&";
}
updateFinalRequest += "classicupdatecode[]=" + mod.ModClassicUpdateCode;
}
else if (mod.NexusModID > 0)
{
//Nexus style
if (first)
{
updateFinalRequest += "?";
first = false;
}
else
{
updateFinalRequest += "&";
}
updateFinalRequest += "nexusupdatecode[]=" + mod.Game.ToString().Substring(2) + "-" + mod.NexusModID;
}
//else if (mod.NexusModID > 0)
//{
// //Classic style
// if (first)
// {
// updateFinalRequest += "?";
// first = false;
// }
// else
// {
// updateFinalRequest += "&";
// }
// updateFinalRequest += "nexusupdatecode[]=" + mod.ModClassicUpdateCode;
//}
}
using var wc = new System.Net.WebClient();
try
{
Debug.WriteLine(updateFinalRequest);
string updatexml = wc.DownloadStringAwareOfEncoding(updateFinalRequest);
XElement rootElement = XElement.Parse(updatexml);
#region classic mods
var modUpdateInfos = new List <ModUpdateInfo>();
var classicUpdateInfos = (from e in rootElement.Elements("mod")
select new ModUpdateInfo
{
changelog = (string)e.Attribute("changelog"),
versionstr = (string)e.Attribute("version"),
updatecode = (int)e.Attribute("updatecode"),
serverfolder = (string)e.Attribute("folder"),
sourceFiles = (from f in e.Elements("sourcefile")
select new SourceFile
{
lzmahash = (string)f.Attribute("lzmahash"),
hash = (string)f.Attribute("hash"),
size = (int)f.Attribute("size"),
lzmasize = (int)f.Attribute("lzmasize"),
relativefilepath = f.Value,
timestamp = (Int64?)f.Attribute("timestamp") ?? (Int64)0
}).ToList(),
blacklistedFiles = e.Elements("blacklistedfile").Select(x => x.Value).ToList()
}).ToList();
foreach (var modUpdateInfo in classicUpdateInfos)
{
modUpdateInfo.ResolveVersionVar();
//Calculate update information
var matchingMod = modsToCheck.FirstOrDefault(x => x.ModClassicUpdateCode == modUpdateInfo.updatecode);
if (matchingMod != null && (forceUpdateCheck || matchingMod.ParsedModVersion < modUpdateInfo.version))
{
modUpdateInfo.mod = matchingMod;
modUpdateInfo.SetLocalizedInfo();
string modBasepath = matchingMod.ModPath;
foreach (var serverFile in modUpdateInfo.sourceFiles)
{
var localFile = Path.Combine(modBasepath, serverFile.relativefilepath);
if (File.Exists(localFile))
{
var info = new FileInfo(localFile);
if (info.Length != serverFile.size)
{
modUpdateInfo.applicableUpdates.Add(serverFile);
}
else
{
//Check hash
CLog.Information("Hashing file for update check: " + localFile, Settings.LogModUpdater);
var md5 = Utilities.CalculateMD5(localFile);
if (md5 != serverFile.hash)
{
modUpdateInfo.applicableUpdates.Add(serverFile);
}
}
}
else
{
modUpdateInfo.applicableUpdates.Add(serverFile);
}
}
foreach (var blacklistedFile in modUpdateInfo.blacklistedFiles)
{
var localFile = Path.Combine(modBasepath, blacklistedFile);
if (File.Exists(localFile))
{
Log.Information(@"Blacklisted file marked for deletion: " + localFile);
modUpdateInfo.filesToDelete.Add(localFile);
}
}
//Files to remove calculation
var modFiles = Directory.GetFiles(modBasepath, "*", SearchOption.AllDirectories).Select(x => x.Substring(modBasepath.Length + 1)).ToList();
modUpdateInfo.filesToDelete.AddRange(modFiles.Except(modUpdateInfo.sourceFiles.Select(x => x.relativefilepath), StringComparer.InvariantCultureIgnoreCase).Distinct().ToList()); //Todo: Add security check here to prevent malicious values
modUpdateInfo.TotalBytesToDownload = modUpdateInfo.applicableUpdates.Sum(x => x.lzmasize);
}
}
modUpdateInfos.AddRange(classicUpdateInfos);
#endregion
#region modmaker mods
var modmakerModUpdateInfos = (from e in rootElement.Elements("modmakermod")
select new ModMakerModUpdateInfo
{
ModMakerId = (int)e.Attribute("id"),
versionstr = (string)e.Attribute("version"),
PublishDate = DateTime.ParseExact((string)e.Attribute("publishdate"), "yyyy-MM-dd", CultureInfo.InvariantCulture),
changelog = (string)e.Attribute("changelog")
}).ToList();
modUpdateInfos.AddRange(modmakerModUpdateInfos);
#endregion
#region Nexus Mod Third Party
var nexusModsUpdateInfo = (from e in rootElement.Elements("nexusmod")
select new NexusModUpdateInfo
{
NexusModsId = (int)e.Attribute("id"),
GameId = (int)e.Attribute("game"),
versionstr = (string)e.Attribute("version"),
UpdatedTime = DateTimeOffset.FromUnixTimeSeconds((long)e.Attribute("updated_timestamp")).DateTime
}).ToList();
modUpdateInfos.AddRange(nexusModsUpdateInfo);
#endregion
return(modUpdateInfos);
}
catch (Exception e)
{
Log.Error("Error checking for mod updates: " + App.FlattenException(e));
Crashes.TrackError(e, new Dictionary <string, string>()
{
{ "Update check URL", updateFinalRequest }
});
}
return(null);
}
//private static readonly string LatestHelpFileLink = StaticFilesBaseURL_Github + "dynamichelp/latesthelp-localized.xml";
//internal static readonly string HelpResourcesBaseURL = StaticFilesBaseURL_Github + "dynamichelp/resources";
public static List <SortableHelpElement> FetchLatestHelp(string language, bool preferLocal, bool overrideThrottling = false)
{
var localHelpExists = File.Exists(Utilities.GetLocalHelpFile());
string cached = null;
if (localHelpExists)
{
try
{
cached = File.ReadAllText(Utilities.GetLocalHelpFile());
}
catch (Exception e)
{
var attachments = new List <ErrorAttachmentLog>();
string log = LogCollector.CollectLatestLog(false);
if (log.Length < ByteSizeLib.ByteSize.BytesInMegaByte * 7)
{
attachments.Add(ErrorAttachmentLog.AttachmentWithText(log, "applog.txt"));
}
Crashes.TrackError(e, new Dictionary <string, string>()
{
{ "Error type", "Error reading cached online content" },
{ "Service", "Dynamic Help" },
{ "Message", e.Message }
}, attachments.ToArray());
}
}
if (localHelpExists && preferLocal)
{
return(ParseLocalHelp(cached, language));
}
if (!localHelpExists || overrideThrottling || OnlineContent.CanFetchContentThrottleCheck())
{
foreach (var staticendpoint in StaticFilesBaseEndpoints)
{
using var wc = new System.Net.WebClient();
try
{
string xml = wc.DownloadStringAwareOfEncoding(staticendpoint + @"dynamichelp/latesthelp-localized.xml");
File.WriteAllText(Utilities.GetLocalHelpFile(), xml);
return(ParseLocalHelp(xml, language));
}
catch (Exception e)
{
Log.Error($"Error fetching online help from endpoint {staticendpoint}: {e.Message}");
}
}
if (cached != null)
{
Log.Warning("Using cached help instead");
}
else
{
Log.Error("Unable to display dynamic help: Could not fetch online asset and cached help asset does not exist.");
return(null);
}
}
try
{
return(ParseLocalHelp(cached, language));
}
catch (Exception e)
{
Log.Error("Unable to parse local dynamic help file: " + e.Message);
return(null);
}
}
private UploadModResult UploadMod(Action <double> progressCallback = null, Action <TaskbarItemProgressState> setTaskbarProgressState = null)
{
#region online fetch
//Fetch current production manifest for mod (it may not exist)
setTaskbarProgressState?.Invoke(TaskbarItemProgressState.Indeterminate);
using var wc = new System.Net.WebClient();
try
{
CurrentActionText = M3L.GetString(M3L.string_checkingIfUpdaterServiceIsConfiguredForMod);
string validationUrl = $@"{UpdaterServiceCodeValidationEndpoint}?updatecode={mod.ModClassicUpdateCode}&updatexmlname={mod.UpdaterServiceServerFolderShortname}.xml";
string isBeingServed = wc.DownloadStringAwareOfEncoding(validationUrl);
if (string.IsNullOrWhiteSpace(isBeingServed) || isBeingServed != @"true") //we don't parse for bool because it might have a different text that is not specifically true or false. It might
// have an error for example
{
//Not being served
Log.Error(@"This mod is not configured for serving on the Updater Service. Please contact Mgamerz.");
CurrentActionText = M3L.GetString(M3L.string_serverNotConfiguredForModContactMgamerz);
HideChangelogArea();
return(UploadModResult.NOT_BEING_SERVED);
}
}
catch (Exception ex)
{
Log.Error(@"Error validating mod is configured on updater service: " + ex.Message);
CurrentActionText = M3L.GetString(M3L.string_interp_errorCheckingUpdaterServiceConfiguration, ex.Message);
HideChangelogArea();
return(UploadModResult.ERROR_VALIDATING_MOD_IS_CONFIGURED);
}
#endregion
#region get current production version to see if we should prompt user
var latestVersionOnServer = OnlineContent.GetLatestVersionOfModOnUpdaterService(mod.ModClassicUpdateCode);
if (latestVersionOnServer != null)
{
if (latestVersionOnServer >= mod.ParsedModVersion)
{
bool cancel = false;
setTaskbarProgressState?.Invoke(TaskbarItemProgressState.Paused);
Application.Current.Dispatcher.Invoke(delegate
{
// server is newer or same as version we are pushing
var response = M3L.ShowDialog(mainwindow, M3L.GetString(M3L.string_interp_dialog_serverVersionSameOrNewerThanLocal, mod.ParsedModVersion, latestVersionOnServer), M3L.GetString(M3L.string_serverVersionSameOrNewerThanLocal), MessageBoxButton.YesNo, MessageBoxImage.Warning);
if (response == MessageBoxResult.No)
{
CurrentActionText = M3L.GetString(M3L.string_uploadAbortedModOnServerIsSameOrNewerThanLocalOneBeingUploaded);
HideChangelogArea();
cancel = true;
return;
}
});
setTaskbarProgressState?.Invoke(TaskbarItemProgressState.Indeterminate);
if (cancel)
{
return(UploadModResult.ABORTED_BY_USER_SAME_VERSION_UPLOADED);
}
}
}
#endregion
#region mod variables
//get refs
var files = mod.GetAllRelativeReferences(true);
files = files.OrderByDescending(x => new FileInfo(Path.Combine(mod.ModPath, x)).Length).ToList();
long totalModSizeUncompressed = files.Sum(x => new FileInfo(Path.Combine(mod.ModPath, x)).Length);
#endregion
#region compress and stage mod
void updateCurrentTextCallback(string newText)
{
CurrentActionText = newText;
}
bool?canceledCheckCallback() => CancelOperations;
CurrentActionText = M3L.GetString(M3L.string_compressingModForUpdaterService);
progressCallback?.Invoke(0);
setTaskbarProgressState?.Invoke(TaskbarItemProgressState.Normal);
var lzmaStagingPath = OnlineContent.StageModForUploadToUpdaterService(mod, files, totalModSizeUncompressed, canceledCheckCallback, updateCurrentTextCallback, progressCallback);
#endregion
if (CancelOperations)
{
AbortUpload();
return(UploadModResult.ABORTED_BY_USER);
}
setTaskbarProgressState?.Invoke(TaskbarItemProgressState.Indeterminate);
#region hash mod and build server manifest
CurrentActionText = M3L.GetString(M3L.string_buildingServerManifest);
long amountHashed = 0;
ConcurrentDictionary <string, SourceFile> manifestFiles = new ConcurrentDictionary <string, SourceFile>();
Parallel.ForEach(files, new ParallelOptions()
{
MaxDegreeOfParallelism = Math.Max(1, Environment.ProcessorCount - 1)
}, x =>
{
if (CancelOperations)
{
return;
}
SourceFile sf = new SourceFile();
var sFile = Path.Combine(mod.ModPath, x);
var lFile = Path.Combine(lzmaStagingPath, x + @".lzma");
sf.hash = Utilities.CalculateMD5(sFile);
sf.lzmahash = Utilities.CalculateMD5(lFile);
var fileInfo = new FileInfo(sFile);
sf.size = fileInfo.Length;
sf.timestamp = fileInfo.LastWriteTimeUtc.Ticks;
sf.relativefilepath = x;
sf.lzmasize = new FileInfo(lFile).Length;
manifestFiles.TryAdd(x, sf);
var done = Interlocked.Add(ref amountHashed, sf.size);
CurrentActionText = M3L.GetString(M3L.string_buildingServerManifest) + $@" {Math.Round(done * 100.0 / totalModSizeUncompressed)}%";
});
if (CancelOperations)
{
AbortUpload();
return(UploadModResult.ABORTED_BY_USER);
}
//Build document
XmlDocument xmlDoc = new XmlDocument();
XmlNode rootNode = xmlDoc.CreateElement(@"mod");
xmlDoc.AppendChild(rootNode);
foreach (var mf in manifestFiles)
{
if (CancelOperations)
{
AbortUpload();
return(UploadModResult.ABORTED_BY_USER);
}
XmlNode sourceNode = xmlDoc.CreateElement(@"sourcefile");
var size = xmlDoc.CreateAttribute(@"size");
size.InnerText = mf.Value.size.ToString();
var hash = xmlDoc.CreateAttribute(@"hash");
hash.InnerText = mf.Value.hash;
var lzmasize = xmlDoc.CreateAttribute(@"lzmasize");
lzmasize.InnerText = mf.Value.lzmasize.ToString();
var lzmahash = xmlDoc.CreateAttribute(@"lzmahash");
lzmahash.InnerText = mf.Value.lzmahash;
var timestamp = xmlDoc.CreateAttribute(@"timestamp");
timestamp.InnerText = mf.Value.timestamp.ToString();
sourceNode.InnerText = mf.Key;
sourceNode.Attributes.Append(size);
sourceNode.Attributes.Append(hash);
sourceNode.Attributes.Append(lzmasize);
sourceNode.Attributes.Append(lzmahash);
sourceNode.Attributes.Append(timestamp);
rootNode.AppendChild(sourceNode);
}
if (CancelOperations)
{
AbortUpload();
return(UploadModResult.ABORTED_BY_USER);
}
foreach (var bf in mod.UpdaterServiceBlacklistedFiles)
{
if (CancelOperations)
{
AbortUpload();
return(UploadModResult.ABORTED_BY_USER);
}
var bfn = xmlDoc.CreateElement(@"blacklistedfile");
bfn.InnerText = bf;
rootNode.AppendChild(bfn);
}
if (CancelOperations)
{
AbortUpload();
return(UploadModResult.ABORTED_BY_USER);
}
var updatecode = xmlDoc.CreateAttribute(@"updatecode");
updatecode.InnerText = mod.ModClassicUpdateCode.ToString();
rootNode.Attributes.Append(updatecode);
var version = xmlDoc.CreateAttribute(@"version");
version.InnerText = mod.ParsedModVersion.ToString();
rootNode.Attributes.Append(version);
var serverfolder = xmlDoc.CreateAttribute(@"folder");
serverfolder.InnerText = mod.UpdaterServiceServerFolder;
rootNode.Attributes.Append(serverfolder);
setTaskbarProgressState?.Invoke(TaskbarItemProgressState.Indeterminate);
#endregion
//wait to ensure changelog is set.
while (ChangelogNotYetSet)
{
setTaskbarProgressState?.Invoke(TaskbarItemProgressState.Paused);
if (CancelOperations)
{
AbortUpload();
return(UploadModResult.ABORTED_BY_USER);
}
CurrentActionText = M3L.GetString(M3L.string_waitingForChangelogToBeSet);
Thread.Sleep(250); //wait for changelog to be set.
}
setTaskbarProgressState?.Invoke(TaskbarItemProgressState.Indeterminate);
#region Finish building manifest
var changelog = xmlDoc.CreateAttribute(@"changelog");
changelog.InnerText = ChangelogText;
rootNode.Attributes.Append(changelog);
using var stringWriter = new StringWriterWithEncoding(Encoding.UTF8);
XmlWriterSettings settings = new XmlWriterSettings();
settings.Indent = true;
settings.IndentChars = @" ";
settings.Encoding = Encoding.UTF8;
using var xmlTextWriter = XmlWriter.Create(stringWriter, settings);
xmlDoc.WriteTo(xmlTextWriter);
xmlTextWriter.Flush();
#endregion
var finalManifestText = stringWriter.GetStringBuilder().ToString();
#region Connect to ME3Tweaks
CurrentActionText = M3L.GetString(M3L.string_connectingToME3TweaksUpdaterService);
Log.Information(@"Connecting to ME3Tweaks as " + Username);
string host = @"ftp.me3tweaks.com";
string username = Username;
string password = Settings.DecryptUpdaterServicePassword();
using SftpClient sftp = new SftpClient(host, username, password);
sftp.Connect();
Log.Information(@"Connected to ME3Tweaks over SSH (SFTP)");
CurrentActionText = M3L.GetString(M3L.string_connectedToME3TweaksUpdaterService);
var serverFolderName = mod.UpdaterServiceServerFolderShortname;
//sftp.ChangeDirectory(LZMAStoragePath);
//Log.Information(@"Listing files/folders for " + LZMAStoragePath);
//var lzmaStorageDirectoryItems = sftp.ListDirectory(LZMAStoragePath);
var serverModPath = LZMAStoragePath + @"/" + serverFolderName;
bool justMadeFolder = false;
if (!sftp.Exists(serverModPath))
{
CurrentActionText = M3L.GetString(M3L.string_creatingServerFolderForMod);
Log.Information(@"Creating server folder for mod: " + serverModPath);
sftp.CreateDirectory(serverModPath);
justMadeFolder = true;
}
var dirContents = sftp.ListDirectory(serverModPath).ToList();
Dictionary <string, string> serverHashes = new Dictionary <string, string>();
//Open SSH connection as we will need to hash files out afterwards.
Log.Information(@"Connecting to ME3Tweaks Updater Service over SSH (SSH Shell)");
using SshClient sshClient = new SshClient(host, username, password);
sshClient.Connect();
Log.Information(@"Connected to ME3Tweaks Updater Service over SSH (SSH Shell)");
if (!justMadeFolder && dirContents.Any(x => x.Name != @"." && x.Name != @".."))
{
CurrentActionText = M3L.GetString(M3L.string_hashingFilesOnServerForDelta);
Log.Information(@"Hashing existing files on server to compare for delta");
serverHashes = getServerHashes(sshClient, serverFolderName, serverModPath);
}
//Calculate what needs to be updated or removed from server
List <string> filesToUploadToServer = new List <string>();
List <string> filesToDeleteOffServer = new List <string>();
//Files to upload
foreach (var sourceFile in manifestFiles)
{
//find matching server file
if (serverHashes.TryGetValue(sourceFile.Key.Replace('\\', '/') + @".lzma", out var matchingHash))
{
//exists on server, compare hash
if (matchingHash != sourceFile.Value.lzmahash)
{
//server hash is different! Upload new file.
Log.Information(@"Server version of file is different from local: " + sourceFile.Key);
filesToUploadToServer.Add(sourceFile.Key);
}
else
{
Log.Information(@"Server version of file is same as local: " + sourceFile.Key);
}
}
else
{
Log.Information(@"Server does not have file: " + sourceFile.Key);
filesToUploadToServer.Add(sourceFile.Key);
}
}
//Files to remove from server
foreach (var serverfile in serverHashes.Keys)
{
if (!manifestFiles.Any(x => (x.Key + @".lzma") == serverfile.Replace('/', '\\')))
{
Log.Information(@"File exists on server but not locally: " + serverfile);
filesToDeleteOffServer.Add(serverfile);
}
}
#endregion
long amountUploaded = 0, amountToUpload = 1;
//Confirm changes
if (filesToDeleteOffServer.Any() || filesToUploadToServer.Any())
{
var text = M3L.GetString(M3L.string_interp_updaterServiceDeltaConfirmationHeader, mod.ModName);
if (filesToUploadToServer.Any())
{
text += M3L.GetString(M3L.string_nnFilesToUploadToServern) + @" " + string.Join('\n' + @" - ", filesToUploadToServer); //weird stuff to deal with localizer
}
if (filesToDeleteOffServer.Any())
{
text += M3L.GetString(M3L.string_nnFilesToDeleteOffServern) + @" " + string.Join('\n' + @" - ", filesToDeleteOffServer); //weird stuff to deal with localizer
}
text += M3L.GetString(M3L.string_interp_updaterServiceDeltaConfirmationFooter);
bool performUpload = false;
Log.Information(@"Prompting user to accept server delta");
setTaskbarProgressState?.Invoke(TaskbarItemProgressState.Paused);
Application.Current.Dispatcher.Invoke(delegate { performUpload = M3L.ShowDialog(mainwindow, text, M3L.GetString(M3L.string_confirmChanges), MessageBoxButton.OKCancel, MessageBoxImage.Exclamation) == MessageBoxResult.OK; });
setTaskbarProgressState?.Invoke(TaskbarItemProgressState.Indeterminate);
if (performUpload)
{
Log.Information(@"User has accepted the delta, applying delta to server");
#region upload files
//Create directories
SortedSet <string> directoriesToCreate = new SortedSet <string>();
foreach (var f in filesToUploadToServer)
{
string foldername = f;
var lastIndex = foldername.LastIndexOf(@"\");
while (lastIndex > 0)
{
foldername = foldername.Substring(0, lastIndex);
directoriesToCreate.Add(foldername.Replace('\\', '/'));
lastIndex = foldername.LastIndexOf(@"\");
}
}
#endregion
//UploadDirectory(sftp, lzmaStagingPath, serverModPath, (ucb) => Debug.WriteLine("UCB: " + ucb));
var dirsToCreateOnServerSorted = directoriesToCreate.ToList();
dirsToCreateOnServerSorted.Sort((a, b) => a.Length.CompareTo(b.Length)); //short to longest so we create top levels first!
int numFoldersToCreate = dirsToCreateOnServerSorted.Count();
int numDone = 0;
if (dirsToCreateOnServerSorted.Count > 0)
{
CurrentActionText = M3L.GetString(M3L.string_creatingModDirectoriesOnServer);
foreach (var f in dirsToCreateOnServerSorted)
{
var serverFolderStr = serverModPath + @"/" + f;
if (!sftp.Exists(serverFolderStr))
{
Log.Information(@"Creating directory on server: " + serverFolderStr);
sftp.CreateDirectory(serverFolderStr);
}
else
{
Log.Information(@"Server folder already exists, skipping: " + serverFolderStr);
}
numDone++;
CurrentActionText = M3L.GetString(M3L.string_creatingModDirectoriesOnServer) + @" " + Math.Round(numDone * 100.0 / numFoldersToCreate) + @"%";
}
}
//Upload files
progressCallback?.Invoke(0);
setTaskbarProgressState?.Invoke(TaskbarItemProgressState.Normal);
amountToUpload = filesToUploadToServer.Sum(x => new FileInfo(Path.Combine(lzmaStagingPath, x + @".lzma")).Length);
foreach (var file in filesToUploadToServer)
{
if (CancelOperations)
{
AbortUpload();
return(UploadModResult.ABORTED_BY_USER);
}
var fullPath = Path.Combine(lzmaStagingPath, file + @".lzma");
var serverFilePath = serverModPath + @"/" + file.Replace(@"\", @"/") + @".lzma";
Log.Information(@"Uploading file " + fullPath + @" to " + serverFilePath);
long amountUploadedBeforeChunk = amountUploaded;
using Stream fileStream = new FileStream(fullPath, FileMode.Open);
sftp.UploadFile(fileStream, serverFilePath, true, (x) =>
{
if (CancelOperations)
{
CurrentActionText = M3L.GetString(M3L.string_abortingUpload);
return;
}
amountUploaded = amountUploadedBeforeChunk + (long)x;
var uploadedHR = ByteSize.FromBytes(amountUploaded).ToString(@"0.00");
var totalUploadHR = ByteSize.FromBytes(amountToUpload).ToString(@"0.00");
if (amountToUpload > 0)
{
progressCallback?.Invoke(amountUploaded * 1.0 / amountToUpload);
}
CurrentActionText = M3L.GetString(M3L.string_interp_uploadingFilesToServerXY, uploadedHR, totalUploadHR);
});
}
setTaskbarProgressState?.Invoke(TaskbarItemProgressState.Indeterminate);
if (CancelOperations)
{
AbortUpload();
return(UploadModResult.ABORTED_BY_USER);
}
//delete extra files
int numdone = 0;
foreach (var file in filesToDeleteOffServer)
{
CurrentActionText = M3L.GetString(M3L.string_interp_deletingObsoleteFiles, numdone, filesToDeleteOffServer.Count);
var fullPath = $@"{LZMAStoragePath}/{serverFolderName}/{file}";
Log.Information(@"Deleting unused file off server: " + fullPath);
sftp.DeleteFile(fullPath);
numdone++;
}
//Upload manifest
using var manifestStream = finalManifestText.ToStream();
var serverManifestPath = $@"{ManifestStoragePath}/{serverFolderName}.xml";
Log.Information(@"Uploading manifest to server: " + serverManifestPath);
sftp.UploadFile(manifestStream, serverManifestPath, true, (x) =>
{
var uploadedAmountHR = ByteSize.FromBytes(amountUploaded).ToString(@"0.00");
var uploadAmountTotalHR = ByteSize.FromBytes(amountToUpload).ToString(@"0.00");
CurrentActionText = M3L.GetString(M3L.string_uploadingUpdateManifestToServer) + $@"{uploadedAmountHR}/{uploadAmountTotalHR}";
});
}
else
{
Log.Warning(@"User has declined uploading the delta. We will not change anything on the server.");
CancelOperations = true;
AbortUpload();
return(UploadModResult.ABORTED_BY_USER);
}
CurrentActionText = M3L.GetString(M3L.string_validatingModOnServer);
Log.Information(@"Verifying hashes on server for new files");
var newServerhashes = getServerHashes(sshClient, serverFolderName, serverModPath);
var badHashes = verifyHashes(manifestFiles, newServerhashes);
if (badHashes.Any())
{
CurrentActionText = M3L.GetString(M3L.string_someHashesOnServerAreIncorrectContactMgamerz);
return(UploadModResult.BAD_SERVER_HASHES_AFTER_VALIDATION);
}
else
{
CurrentActionText = M3L.GetString(M3L.string_modUploadedToUpdaterService);
return(UploadModResult.UPLOAD_OK);
}
}
return(UploadModResult.ABORTED_BY_USER);
}
/// <summary>
/// Checks mods for updates. ForceUpdateCheck will force the mod to validate against the server (essentially repair mode). It is not used for rate limiting!
/// </summary>
/// <param name="modsToCheck">Mods to have server send information about</param>
/// <param name="forceUpdateCheck">Force update check regardless of version</param>
/// <returns></returns>
public static List <ModUpdateInfo> CheckForModUpdates(List <Mod> modsToCheck, bool forceUpdateCheck, Action <string> updateStatusCallback = null)
{
string updateFinalRequest = UpdaterServiceManifestEndpoint;
bool first = true;
foreach (var mod in modsToCheck)
{
if (mod.ModModMakerID > 0)
{
//Modmaker style
if (first)
{
updateFinalRequest += "?";
first = false;
}
else
{
updateFinalRequest += "&";
}
updateFinalRequest += "modmakerupdatecode[]=" + mod.ModModMakerID;
}
else if (mod.ModClassicUpdateCode > 0)
{
//Classic style
if (first)
{
updateFinalRequest += "?";
first = false;
}
else
{
updateFinalRequest += "&";
}
updateFinalRequest += "classicupdatecode[]=" + mod.ModClassicUpdateCode;
}
else if (mod.NexusModID > 0 && mod.NexusUpdateCheck)
{
//Nexus style
if (first)
{
updateFinalRequest += "?";
first = false;
}
else
{
updateFinalRequest += "&";
}
updateFinalRequest += "nexusupdatecode[]=" + mod.Game.ToString().Substring(2) + "-" + mod.NexusModID;
}
//else if (mod.NexusModID > 0)
//{
// //Classic style
// if (first)
// {
// updateFinalRequest += "?";
// first = false;
// }
// else
// {
// updateFinalRequest += "&";
// }
// updateFinalRequest += "nexusupdatecode[]=" + mod.ModClassicUpdateCode;
//}
}
using var wc = new System.Net.WebClient();
try
{
Debug.WriteLine(updateFinalRequest);
string updatexml = wc.DownloadStringAwareOfEncoding(updateFinalRequest);
XElement rootElement = XElement.Parse(updatexml);
#region classic mods
var modUpdateInfos = new List <ModUpdateInfo>();
var classicUpdateInfos = (from e in rootElement.Elements("mod")
select new ModUpdateInfo
{
changelog = (string)e.Attribute("changelog"),
versionstr = (string)e.Attribute("version"),
updatecode = (int)e.Attribute("updatecode"),
serverfolder = (string)e.Attribute("folder"),
sourceFiles = (from f in e.Elements("sourcefile")
select new SourceFile
{
lzmahash = (string)f.Attribute("lzmahash"),
hash = (string)f.Attribute("hash"),
size = (int)f.Attribute("size"),
lzmasize = (int)f.Attribute("lzmasize"),
relativefilepath = f.Value,
timestamp = (Int64?)f.Attribute("timestamp") ?? (Int64)0
}).ToList(),
blacklistedFiles = e.Elements("blacklistedfile").Select(x => x.Value).ToList()
}).ToList();
// CALCULATE UPDATE DELTA
CaseInsensitiveDictionary <USFileInfo> hashMap = new CaseInsensitiveDictionary <USFileInfo>(); //Used to rename files
foreach (var modUpdateInfo in classicUpdateInfos)
{
modUpdateInfo.ResolveVersionVar();
//Calculate update information
var matchingMod = modsToCheck.FirstOrDefault(x => x.ModClassicUpdateCode == modUpdateInfo.updatecode);
if (matchingMod != null && (forceUpdateCheck || matchingMod.ParsedModVersion < modUpdateInfo.version))
{
// The following line is left so we know that it was at one point considered implemented.
// This prevents updating copies of the same mod in the library. Cause it's just kind of a bandwidth waste.
//modsToCheck.Remove(matchingMod); //This makes it so we don't feed in multiple same-mods. For example, nexus check on 3 Project Variety downloads
modUpdateInfo.mod = matchingMod;
modUpdateInfo.SetLocalizedInfo();
string modBasepath = matchingMod.ModPath;
double i = 0;
List <string> references = null;
try
{
references = matchingMod.GetAllRelativeReferences(true);
}
catch
{
// There's an error. Underlying disk state may have changed since we originally loaded the mod
}
if (references == null || !matchingMod.ValidMod)
{
// The mod failed to load. We should just index everything the
// references will not be dfully parsed.
var localFiles = Directory.GetFiles(matchingMod.ModPath, "*", SearchOption.AllDirectories);
references = localFiles.Select(x => x.Substring(matchingMod.ModPath.Length + 1)).ToList();
}
int total = references.Count;
// Index existing files
foreach (var v in references)
{
updateStatusCallback?.Invoke(
$"Indexing {modUpdateInfo.mod.ModName} for updates {(int)(i * 100 / total)}%");
i++;
var fpath = Path.Combine(matchingMod.ModPath, v);
if (fpath.RepresentsPackageFilePath())
{
// We need to make sure it's decompressed
var qPackage = MEPackageHandler.QuickOpenMEPackage(fpath);
if (qPackage.IsCompressed)
{
CLog.Information(
$" >> Decompressing compressed package for update comparison check: {fpath}",
Settings.LogModUpdater);
qPackage = MEPackageHandler.OpenMEPackage(fpath);
MemoryStream tStream = new MemoryStream();
tStream = qPackage.SaveToStream(false);
hashMap[v] = new USFileInfo()
{
MD5 = Utilities.CalculateMD5(tStream),
CompressedMD5 = Utilities.CalculateMD5(fpath),
Filesize = tStream.Length,
RelativeFilepath = v
};
continue;
}
}
hashMap[v] = new USFileInfo()
{
MD5 = Utilities.CalculateMD5(fpath),
Filesize = new FileInfo(fpath).Length,
RelativeFilepath = v
};
}
i = 0;
total = modUpdateInfo.sourceFiles.Count;
foreach (var serverFile in modUpdateInfo.sourceFiles)
{
Log.Information($@"Checking {serverFile.relativefilepath} for update applicability");
updateStatusCallback?.Invoke(
$"Calculating update delta for {modUpdateInfo.mod.ModName} {(int)(i * 100 / total)}%");
i++;
bool calculatedOp = false;
if (hashMap.TryGetValue(serverFile.relativefilepath, out var indexInfo))
{
if (indexInfo.MD5 == serverFile.hash)
{
CLog.Information(@" >> File is up to date", Settings.LogModUpdater);
calculatedOp = true;
}
else if (indexInfo.CompressedMD5 != null && indexInfo.CompressedMD5 == serverFile.hash)
{
CLog.Information(@" >> Compressed package file is up to date",
Settings.LogModUpdater);
calculatedOp = true;
}
}
if (!calculatedOp)
{
// File is missing or hash was wrong. We should try to map it to another existing file
// to save bandwidth
var existingFilesThatMatchServerHash =
hashMap.Where(x =>
x.Value.MD5 == serverFile.hash || (x.Value.CompressedMD5 != null &&
x.Value.CompressedMD5 ==
serverFile.hash))
.ToList();
if (existingFilesThatMatchServerHash.Any())
{
CLog.Information(
$" >> Server file can be cloned from local file {existingFilesThatMatchServerHash[0].Value.RelativeFilepath} as it has same hash",
Settings.LogModUpdater);
modUpdateInfo.cloneOperations[serverFile] =
existingFilesThatMatchServerHash[0]
.Value; // Server file can be sourced from the value
}
else if (indexInfo == null)
{
// we don't have file hashed (new file)
CLog.Information(
$" >> Applicable for updates, File does not exist locally",
Settings.LogModUpdater);
modUpdateInfo.applicableUpdates.Add(serverFile);
}
else
{
// Existing file has wrong hash
CLog.Information($" >> Applicable for updates, hash has changed",
Settings.LogModUpdater);
modUpdateInfo.applicableUpdates.Add(serverFile);
}
}
}
foreach (var blacklistedFile in modUpdateInfo.blacklistedFiles)
{
var blLocalFile = Path.Combine(modBasepath, blacklistedFile);
if (File.Exists(blLocalFile))
{
Log.Information(@"Blacklisted file marked for deletion: " + blLocalFile);
modUpdateInfo.filesToDelete.Add(blLocalFile);
}
}
// alphabetize files
modUpdateInfo.applicableUpdates.Sort(x => x.relativefilepath);
//Files to remove calculation
var modFiles = Directory.GetFiles(modBasepath, "*", SearchOption.AllDirectories)
.Select(x => x.Substring(modBasepath.Length + 1)).ToList();
var additionalFilesToDelete = modFiles.Except(
modUpdateInfo.sourceFiles.Select(x => x.relativefilepath),
StringComparer.InvariantCultureIgnoreCase).Distinct().ToList();
modUpdateInfo.filesToDelete.AddRange(
additionalFilesToDelete); //Todo: Add security check here to prevent malicious
modUpdateInfo.TotalBytesToDownload = modUpdateInfo.applicableUpdates.Sum(x => x.lzmasize);
}
}
modUpdateInfos.AddRange(classicUpdateInfos);
#endregion
#region modmaker mods
var modmakerModUpdateInfos = (from e in rootElement.Elements("modmakermod")
select new ModMakerModUpdateInfo
{
ModMakerId = (int)e.Attribute("id"),
versionstr = (string)e.Attribute("version"),
PublishDate = DateTime.ParseExact((string)e.Attribute("publishdate"), "yyyy-MM-dd",
CultureInfo.InvariantCulture),
changelog = (string)e.Attribute("changelog")
}).ToList();
modUpdateInfos.AddRange(modmakerModUpdateInfos);
#endregion
#region Nexus Mod Third Party
var nexusModsUpdateInfo = (from e in rootElement.Elements("nexusmod")
select new NexusModUpdateInfo
{
NexusModsId = (int)e.Attribute("id"),
GameId = (int)e.Attribute("game"),
versionstr = (string)e.Attribute("version"),
UpdatedTime = DateTimeOffset.FromUnixTimeSeconds((long)e.Attribute("updated_timestamp"))
.DateTime
}).ToList();
modUpdateInfos.AddRange(nexusModsUpdateInfo);
#endregion
return(modUpdateInfos);
}
catch (Exception e)
{
Log.Error("Error checking for mod updates: " + App.FlattenException(e));
Crashes.TrackError(e, new Dictionary <string, string>()
{
{ "Update check URL", updateFinalRequest }
});
}
return(null);
}
