public static string FetchExeTransform(string name) { using var wc = new System.Net.WebClient(); string moddesc = wc.DownloadStringAwareOfEncoding(ExeTransformBaseURL + name); return(moddesc); }
/// <summary> /// Fetch latest version information (manifest attribute) from ME3Tweaks Updater Service. This should not be used for true update checks, use CheckForModUpdates() for that purpose. /// </summary> /// <param name="updatecode">Code to check</param> /// <returns>verison string if found and parsable, null otherwise</returns> public static Version GetLatestVersionOfModOnUpdaterService(int updatecode) { if (updatecode <= 0) { return(null); //invalid } string updateFinalRequest = UpdaterServiceManifestEndpoint + "?classicupdatecode[]=" + updatecode; using var wc = new System.Net.WebClient(); try { string updatexml = wc.DownloadStringAwareOfEncoding(updateFinalRequest); XElement rootElement = XElement.Parse(updatexml); var modUpdateInfos = (from e in rootElement.Elements("mod") select new ModUpdateInfo { versionstr = (string)e.Attribute("version") }).ToList(); if (modUpdateInfos.Count == 1 && Version.TryParse(modUpdateInfos[0].versionstr, out var ver)) { return(ver); } } catch (Exception e) { Log.Error("Unable to fetch latest version of mod on updater service: " + e.Message); } return(null); }
public static string FetchThirdPartyModdesc(string name) { using var wc = new System.Net.WebClient(); string moddesc = wc.DownloadStringAwareOfEncoding(ThirdPartyModDescURL + name); return(moddesc); }
public static Dictionary <long, List <ThirdPartyServices.ThirdPartyImportingInfo> > FetchThirdPartyImportingService(bool overrideThrottling = false) { if (!File.Exists(Utilities.GetThirdPartyImportingCachedFile()) || overrideThrottling || Utilities.CanFetchContentThrottleCheck()) { try { using var wc = new System.Net.WebClient(); string json = wc.DownloadStringAwareOfEncoding(ThirdPartyImportingServiceURL); File.WriteAllText(Utilities.GetThirdPartyImportingCachedFile(), json); return(JsonConvert.DeserializeObject <Dictionary <long, List <ThirdPartyServices.ThirdPartyImportingInfo> > >(json)); } catch (Exception e) { //Unable to fetch latest help. Log.Error("Error fetching latest tips service file: " + e.Message); if (File.Exists(Utilities.GetThirdPartyImportingCachedFile())) { Log.Warning("Using cached third party importing service file instead"); return(JsonConvert.DeserializeObject <Dictionary <long, List <ThirdPartyServices.ThirdPartyImportingInfo> > >(File.ReadAllText(Utilities.GetThirdPartyImportingCachedFile()))); } else { Log.Error("Unable to fetch latest third party importing service file from server and local file doesn't exist. Returning a blank copy."); return(new Dictionary <long, List <ThirdPartyServices.ThirdPartyImportingInfo> >()); } } } return(JsonConvert.DeserializeObject <Dictionary <long, List <ThirdPartyServices.ThirdPartyImportingInfo> > >(File.ReadAllText(Utilities.GetThirdPartyImportingCachedFile()))); }
public static List <SortableHelpElement> FetchLatestHelp(bool overrideThrottling = false) { if (!File.Exists(Utilities.GetLocalHelpFile()) || (!overrideThrottling && Utilities.CanFetchContentThrottleCheck())) { try { using var wc = new System.Net.WebClient(); string xml = wc.DownloadStringAwareOfEncoding(LatestHelpFileLink); File.WriteAllText(Utilities.GetLocalHelpFile(), xml); return(ParseLocalHelp(xml)); } catch (Exception e) { //Unable to fetch latest help. Log.Error("Error fetching online help: " + e.Message); if (File.Exists(Utilities.GetLocalHelpFile())) { Log.Warning("Using cached help instead"); return(ParseLocalHelp(File.ReadAllText(Utilities.GetLocalHelpFile()))); } else { Log.Error("Unable to display dynamic help: Could not fetch online asset and cached help asset does not exist."); return(null); } } } return(ParseLocalHelp(File.ReadAllText(Utilities.GetLocalHelpFile()))); }
public static Dictionary <string, CaseInsensitiveDictionary <ThirdPartyServices.ThirdPartyModInfo> > FetchThirdPartyIdentificationManifest(bool overrideThrottling = false) { if (!File.Exists(Utilities.GetThirdPartyIdentificationCachedFile()) || (!overrideThrottling && Utilities.CanFetchContentThrottleCheck())) { try { using var wc = new System.Net.WebClient(); string json = wc.DownloadStringAwareOfEncoding(ThirdPartyIdentificationServiceURL); File.WriteAllText(Utilities.GetThirdPartyIdentificationCachedFile(), json); return(JsonConvert.DeserializeObject <Dictionary <string, CaseInsensitiveDictionary <ThirdPartyServices.ThirdPartyModInfo> > >(json)); } catch (Exception e) { //Unable to fetch latest help. Log.Error("Error fetching online third party identification service: " + e.Message); if (File.Exists(Utilities.GetThirdPartyIdentificationCachedFile())) { Log.Warning("Using cached third party identification service instead"); return(JsonConvert.DeserializeObject <Dictionary <string, CaseInsensitiveDictionary <ThirdPartyServices.ThirdPartyModInfo> > >(File.ReadAllText(Utilities.GetThirdPartyIdentificationCachedFile()))); } else { Log.Error("Unable to load third party identification service and local file doesn't exist. Returning a blank copy."); Dictionary <string, CaseInsensitiveDictionary <ThirdPartyServices.ThirdPartyModInfo> > d = new Dictionary <string, CaseInsensitiveDictionary <ThirdPartyServices.ThirdPartyModInfo> >(); d["ME1"] = new CaseInsensitiveDictionary <ThirdPartyServices.ThirdPartyModInfo>(); d["ME2"] = new CaseInsensitiveDictionary <ThirdPartyServices.ThirdPartyModInfo>(); d["ME3"] = new CaseInsensitiveDictionary <ThirdPartyServices.ThirdPartyModInfo>(); return(d); } } } return(JsonConvert.DeserializeObject <Dictionary <string, CaseInsensitiveDictionary <ThirdPartyServices.ThirdPartyModInfo> > >(File.ReadAllText(Utilities.GetThirdPartyIdentificationCachedFile()))); }
public static string FetchRemoteString(string url) { try { using var wc = new System.Net.WebClient(); return(wc.DownloadStringAwareOfEncoding(url)); } catch (Exception e) { Log.Error("Error downloading string: " + e.Message); return(null); } }
public static Dictionary <string, string> QueryModRelay(string md5, long size) { //Todo: Finish implementing relay service string finalRelayURL = $"{ModInfoRelayEndpoint}?modmanagerversion={App.BuildNumber}&md5={md5.ToLowerInvariant()}&size={size}"; try { using (var wc = new System.Net.WebClient()) { Debug.WriteLine(finalRelayURL); string json = wc.DownloadStringAwareOfEncoding(finalRelayURL); //todo: Implement response format serverside return(JsonConvert.DeserializeObject <Dictionary <string, string> >(json)); } } catch (Exception e) { Log.Error("Error querying relay service from ME3Tweaks: " + App.FlattenException(e)); } return(null); }
/// <summary> /// Checks mods for updates. ForceUpdateCheck will force the mod to validate against the server (essentially repair mode). It is not used for rate limiting! /// </summary> /// <param name="modsToCheck">Mods to have server send information about</param> /// <param name="forceUpdateCheck">Force update check regardless of version</param> /// <returns></returns> public static List <ModUpdateInfo> CheckForModUpdates(List <Mod> modsToCheck, bool forceUpdateCheck) { string updateFinalRequest = UpdaterServiceManifestEndpoint; bool first = true; foreach (var mod in modsToCheck) { if (mod.ModModMakerID > 0) { //Modmaker style if (first) { updateFinalRequest += "?"; first = false; } else { updateFinalRequest += "&"; } updateFinalRequest += "modmakerupdatecode[]=" + mod.ModModMakerID; } else if (mod.ModClassicUpdateCode > 0) { //Classic style if (first) { updateFinalRequest += "?"; first = false; } else { updateFinalRequest += "&"; } updateFinalRequest += "classicupdatecode[]=" + mod.ModClassicUpdateCode; } else if (mod.NexusModID > 0) { //Nexus style if (first) { updateFinalRequest += "?"; first = false; } else { updateFinalRequest += "&"; } updateFinalRequest += "nexusupdatecode[]=" + mod.Game.ToString().Substring(2) + "-" + mod.NexusModID; } //else if (mod.NexusModID > 0) //{ // //Classic style // if (first) // { // updateFinalRequest += "?"; // first = false; // } // else // { // updateFinalRequest += "&"; // } // updateFinalRequest += "nexusupdatecode[]=" + mod.ModClassicUpdateCode; //} } using var wc = new System.Net.WebClient(); try { Debug.WriteLine(updateFinalRequest); string updatexml = wc.DownloadStringAwareOfEncoding(updateFinalRequest); XElement rootElement = XElement.Parse(updatexml); #region classic mods var modUpdateInfos = new List <ModUpdateInfo>(); var classicUpdateInfos = (from e in rootElement.Elements("mod") select new ModUpdateInfo { changelog = (string)e.Attribute("changelog"), versionstr = (string)e.Attribute("version"), updatecode = (int)e.Attribute("updatecode"), serverfolder = (string)e.Attribute("folder"), sourceFiles = (from f in e.Elements("sourcefile") select new SourceFile { lzmahash = (string)f.Attribute("lzmahash"), hash = (string)f.Attribute("hash"), size = (int)f.Attribute("size"), lzmasize = (int)f.Attribute("lzmasize"), relativefilepath = f.Value, timestamp = (Int64?)f.Attribute("timestamp") ?? (Int64)0 }).ToList(), blacklistedFiles = e.Elements("blacklistedfile").Select(x => x.Value).ToList() }).ToList(); foreach (var modUpdateInfo in classicUpdateInfos) { modUpdateInfo.ResolveVersionVar(); //Calculate update information var matchingMod = modsToCheck.FirstOrDefault(x => x.ModClassicUpdateCode == modUpdateInfo.updatecode); if (matchingMod != null && (forceUpdateCheck || matchingMod.ParsedModVersion < modUpdateInfo.version)) { modUpdateInfo.mod = matchingMod; modUpdateInfo.SetLocalizedInfo(); string modBasepath = matchingMod.ModPath; foreach (var serverFile in modUpdateInfo.sourceFiles) { var localFile = Path.Combine(modBasepath, serverFile.relativefilepath); if (File.Exists(localFile)) { var info = new FileInfo(localFile); if (info.Length != serverFile.size) { modUpdateInfo.applicableUpdates.Add(serverFile); } else { //Check hash CLog.Information("Hashing file for update check: " + localFile, Settings.LogModUpdater); var md5 = Utilities.CalculateMD5(localFile); if (md5 != serverFile.hash) { modUpdateInfo.applicableUpdates.Add(serverFile); } } } else { modUpdateInfo.applicableUpdates.Add(serverFile); } } foreach (var blacklistedFile in modUpdateInfo.blacklistedFiles) { var localFile = Path.Combine(modBasepath, blacklistedFile); if (File.Exists(localFile)) { Log.Information(@"Blacklisted file marked for deletion: " + localFile); modUpdateInfo.filesToDelete.Add(localFile); } } //Files to remove calculation var modFiles = Directory.GetFiles(modBasepath, "*", SearchOption.AllDirectories).Select(x => x.Substring(modBasepath.Length + 1)).ToList(); modUpdateInfo.filesToDelete.AddRange(modFiles.Except(modUpdateInfo.sourceFiles.Select(x => x.relativefilepath), StringComparer.InvariantCultureIgnoreCase).Distinct().ToList()); //Todo: Add security check here to prevent malicious values modUpdateInfo.TotalBytesToDownload = modUpdateInfo.applicableUpdates.Sum(x => x.lzmasize); } } modUpdateInfos.AddRange(classicUpdateInfos); #endregion #region modmaker mods var modmakerModUpdateInfos = (from e in rootElement.Elements("modmakermod") select new ModMakerModUpdateInfo { ModMakerId = (int)e.Attribute("id"), versionstr = (string)e.Attribute("version"), PublishDate = DateTime.ParseExact((string)e.Attribute("publishdate"), "yyyy-MM-dd", CultureInfo.InvariantCulture), changelog = (string)e.Attribute("changelog") }).ToList(); modUpdateInfos.AddRange(modmakerModUpdateInfos); #endregion #region Nexus Mod Third Party var nexusModsUpdateInfo = (from e in rootElement.Elements("nexusmod") select new NexusModUpdateInfo { NexusModsId = (int)e.Attribute("id"), GameId = (int)e.Attribute("game"), versionstr = (string)e.Attribute("version"), UpdatedTime = DateTimeOffset.FromUnixTimeSeconds((long)e.Attribute("updated_timestamp")).DateTime }).ToList(); modUpdateInfos.AddRange(nexusModsUpdateInfo); #endregion return(modUpdateInfos); } catch (Exception e) { Log.Error("Error checking for mod updates: " + App.FlattenException(e)); Crashes.TrackError(e, new Dictionary <string, string>() { { "Update check URL", updateFinalRequest } }); } return(null); }
//private static readonly string LatestHelpFileLink = StaticFilesBaseURL_Github + "dynamichelp/latesthelp-localized.xml"; //internal static readonly string HelpResourcesBaseURL = StaticFilesBaseURL_Github + "dynamichelp/resources"; public static List <SortableHelpElement> FetchLatestHelp(string language, bool preferLocal, bool overrideThrottling = false) { var localHelpExists = File.Exists(Utilities.GetLocalHelpFile()); string cached = null; if (localHelpExists) { try { cached = File.ReadAllText(Utilities.GetLocalHelpFile()); } catch (Exception e) { var attachments = new List <ErrorAttachmentLog>(); string log = LogCollector.CollectLatestLog(false); if (log.Length < ByteSizeLib.ByteSize.BytesInMegaByte * 7) { attachments.Add(ErrorAttachmentLog.AttachmentWithText(log, "applog.txt")); } Crashes.TrackError(e, new Dictionary <string, string>() { { "Error type", "Error reading cached online content" }, { "Service", "Dynamic Help" }, { "Message", e.Message } }, attachments.ToArray()); } } if (localHelpExists && preferLocal) { return(ParseLocalHelp(cached, language)); } if (!localHelpExists || overrideThrottling || OnlineContent.CanFetchContentThrottleCheck()) { foreach (var staticendpoint in StaticFilesBaseEndpoints) { using var wc = new System.Net.WebClient(); try { string xml = wc.DownloadStringAwareOfEncoding(staticendpoint + @"dynamichelp/latesthelp-localized.xml"); File.WriteAllText(Utilities.GetLocalHelpFile(), xml); return(ParseLocalHelp(xml, language)); } catch (Exception e) { Log.Error($"Error fetching online help from endpoint {staticendpoint}: {e.Message}"); } } if (cached != null) { Log.Warning("Using cached help instead"); } else { Log.Error("Unable to display dynamic help: Could not fetch online asset and cached help asset does not exist."); return(null); } } try { return(ParseLocalHelp(cached, language)); } catch (Exception e) { Log.Error("Unable to parse local dynamic help file: " + e.Message); return(null); } }
private UploadModResult UploadMod(Action <double> progressCallback = null, Action <TaskbarItemProgressState> setTaskbarProgressState = null) { #region online fetch //Fetch current production manifest for mod (it may not exist) setTaskbarProgressState?.Invoke(TaskbarItemProgressState.Indeterminate); using var wc = new System.Net.WebClient(); try { CurrentActionText = M3L.GetString(M3L.string_checkingIfUpdaterServiceIsConfiguredForMod); string validationUrl = $@"{UpdaterServiceCodeValidationEndpoint}?updatecode={mod.ModClassicUpdateCode}&updatexmlname={mod.UpdaterServiceServerFolderShortname}.xml"; string isBeingServed = wc.DownloadStringAwareOfEncoding(validationUrl); if (string.IsNullOrWhiteSpace(isBeingServed) || isBeingServed != @"true") //we don't parse for bool because it might have a different text that is not specifically true or false. It might // have an error for example { //Not being served Log.Error(@"This mod is not configured for serving on the Updater Service. Please contact Mgamerz."); CurrentActionText = M3L.GetString(M3L.string_serverNotConfiguredForModContactMgamerz); HideChangelogArea(); return(UploadModResult.NOT_BEING_SERVED); } } catch (Exception ex) { Log.Error(@"Error validating mod is configured on updater service: " + ex.Message); CurrentActionText = M3L.GetString(M3L.string_interp_errorCheckingUpdaterServiceConfiguration, ex.Message); HideChangelogArea(); return(UploadModResult.ERROR_VALIDATING_MOD_IS_CONFIGURED); } #endregion #region get current production version to see if we should prompt user var latestVersionOnServer = OnlineContent.GetLatestVersionOfModOnUpdaterService(mod.ModClassicUpdateCode); if (latestVersionOnServer != null) { if (latestVersionOnServer >= mod.ParsedModVersion) { bool cancel = false; setTaskbarProgressState?.Invoke(TaskbarItemProgressState.Paused); Application.Current.Dispatcher.Invoke(delegate { // server is newer or same as version we are pushing var response = M3L.ShowDialog(mainwindow, M3L.GetString(M3L.string_interp_dialog_serverVersionSameOrNewerThanLocal, mod.ParsedModVersion, latestVersionOnServer), M3L.GetString(M3L.string_serverVersionSameOrNewerThanLocal), MessageBoxButton.YesNo, MessageBoxImage.Warning); if (response == MessageBoxResult.No) { CurrentActionText = M3L.GetString(M3L.string_uploadAbortedModOnServerIsSameOrNewerThanLocalOneBeingUploaded); HideChangelogArea(); cancel = true; return; } }); setTaskbarProgressState?.Invoke(TaskbarItemProgressState.Indeterminate); if (cancel) { return(UploadModResult.ABORTED_BY_USER_SAME_VERSION_UPLOADED); } } } #endregion #region mod variables //get refs var files = mod.GetAllRelativeReferences(true); files = files.OrderByDescending(x => new FileInfo(Path.Combine(mod.ModPath, x)).Length).ToList(); long totalModSizeUncompressed = files.Sum(x => new FileInfo(Path.Combine(mod.ModPath, x)).Length); #endregion #region compress and stage mod void updateCurrentTextCallback(string newText) { CurrentActionText = newText; } bool?canceledCheckCallback() => CancelOperations; CurrentActionText = M3L.GetString(M3L.string_compressingModForUpdaterService); progressCallback?.Invoke(0); setTaskbarProgressState?.Invoke(TaskbarItemProgressState.Normal); var lzmaStagingPath = OnlineContent.StageModForUploadToUpdaterService(mod, files, totalModSizeUncompressed, canceledCheckCallback, updateCurrentTextCallback, progressCallback); #endregion if (CancelOperations) { AbortUpload(); return(UploadModResult.ABORTED_BY_USER); } setTaskbarProgressState?.Invoke(TaskbarItemProgressState.Indeterminate); #region hash mod and build server manifest CurrentActionText = M3L.GetString(M3L.string_buildingServerManifest); long amountHashed = 0; ConcurrentDictionary <string, SourceFile> manifestFiles = new ConcurrentDictionary <string, SourceFile>(); Parallel.ForEach(files, new ParallelOptions() { MaxDegreeOfParallelism = Math.Max(1, Environment.ProcessorCount - 1) }, x => { if (CancelOperations) { return; } SourceFile sf = new SourceFile(); var sFile = Path.Combine(mod.ModPath, x); var lFile = Path.Combine(lzmaStagingPath, x + @".lzma"); sf.hash = Utilities.CalculateMD5(sFile); sf.lzmahash = Utilities.CalculateMD5(lFile); var fileInfo = new FileInfo(sFile); sf.size = fileInfo.Length; sf.timestamp = fileInfo.LastWriteTimeUtc.Ticks; sf.relativefilepath = x; sf.lzmasize = new FileInfo(lFile).Length; manifestFiles.TryAdd(x, sf); var done = Interlocked.Add(ref amountHashed, sf.size); CurrentActionText = M3L.GetString(M3L.string_buildingServerManifest) + $@" {Math.Round(done * 100.0 / totalModSizeUncompressed)}%"; }); if (CancelOperations) { AbortUpload(); return(UploadModResult.ABORTED_BY_USER); } //Build document XmlDocument xmlDoc = new XmlDocument(); XmlNode rootNode = xmlDoc.CreateElement(@"mod"); xmlDoc.AppendChild(rootNode); foreach (var mf in manifestFiles) { if (CancelOperations) { AbortUpload(); return(UploadModResult.ABORTED_BY_USER); } XmlNode sourceNode = xmlDoc.CreateElement(@"sourcefile"); var size = xmlDoc.CreateAttribute(@"size"); size.InnerText = mf.Value.size.ToString(); var hash = xmlDoc.CreateAttribute(@"hash"); hash.InnerText = mf.Value.hash; var lzmasize = xmlDoc.CreateAttribute(@"lzmasize"); lzmasize.InnerText = mf.Value.lzmasize.ToString(); var lzmahash = xmlDoc.CreateAttribute(@"lzmahash"); lzmahash.InnerText = mf.Value.lzmahash; var timestamp = xmlDoc.CreateAttribute(@"timestamp"); timestamp.InnerText = mf.Value.timestamp.ToString(); sourceNode.InnerText = mf.Key; sourceNode.Attributes.Append(size); sourceNode.Attributes.Append(hash); sourceNode.Attributes.Append(lzmasize); sourceNode.Attributes.Append(lzmahash); sourceNode.Attributes.Append(timestamp); rootNode.AppendChild(sourceNode); } if (CancelOperations) { AbortUpload(); return(UploadModResult.ABORTED_BY_USER); } foreach (var bf in mod.UpdaterServiceBlacklistedFiles) { if (CancelOperations) { AbortUpload(); return(UploadModResult.ABORTED_BY_USER); } var bfn = xmlDoc.CreateElement(@"blacklistedfile"); bfn.InnerText = bf; rootNode.AppendChild(bfn); } if (CancelOperations) { AbortUpload(); return(UploadModResult.ABORTED_BY_USER); } var updatecode = xmlDoc.CreateAttribute(@"updatecode"); updatecode.InnerText = mod.ModClassicUpdateCode.ToString(); rootNode.Attributes.Append(updatecode); var version = xmlDoc.CreateAttribute(@"version"); version.InnerText = mod.ParsedModVersion.ToString(); rootNode.Attributes.Append(version); var serverfolder = xmlDoc.CreateAttribute(@"folder"); serverfolder.InnerText = mod.UpdaterServiceServerFolder; rootNode.Attributes.Append(serverfolder); setTaskbarProgressState?.Invoke(TaskbarItemProgressState.Indeterminate); #endregion //wait to ensure changelog is set. while (ChangelogNotYetSet) { setTaskbarProgressState?.Invoke(TaskbarItemProgressState.Paused); if (CancelOperations) { AbortUpload(); return(UploadModResult.ABORTED_BY_USER); } CurrentActionText = M3L.GetString(M3L.string_waitingForChangelogToBeSet); Thread.Sleep(250); //wait for changelog to be set. } setTaskbarProgressState?.Invoke(TaskbarItemProgressState.Indeterminate); #region Finish building manifest var changelog = xmlDoc.CreateAttribute(@"changelog"); changelog.InnerText = ChangelogText; rootNode.Attributes.Append(changelog); using var stringWriter = new StringWriterWithEncoding(Encoding.UTF8); XmlWriterSettings settings = new XmlWriterSettings(); settings.Indent = true; settings.IndentChars = @" "; settings.Encoding = Encoding.UTF8; using var xmlTextWriter = XmlWriter.Create(stringWriter, settings); xmlDoc.WriteTo(xmlTextWriter); xmlTextWriter.Flush(); #endregion var finalManifestText = stringWriter.GetStringBuilder().ToString(); #region Connect to ME3Tweaks CurrentActionText = M3L.GetString(M3L.string_connectingToME3TweaksUpdaterService); Log.Information(@"Connecting to ME3Tweaks as " + Username); string host = @"ftp.me3tweaks.com"; string username = Username; string password = Settings.DecryptUpdaterServicePassword(); using SftpClient sftp = new SftpClient(host, username, password); sftp.Connect(); Log.Information(@"Connected to ME3Tweaks over SSH (SFTP)"); CurrentActionText = M3L.GetString(M3L.string_connectedToME3TweaksUpdaterService); var serverFolderName = mod.UpdaterServiceServerFolderShortname; //sftp.ChangeDirectory(LZMAStoragePath); //Log.Information(@"Listing files/folders for " + LZMAStoragePath); //var lzmaStorageDirectoryItems = sftp.ListDirectory(LZMAStoragePath); var serverModPath = LZMAStoragePath + @"/" + serverFolderName; bool justMadeFolder = false; if (!sftp.Exists(serverModPath)) { CurrentActionText = M3L.GetString(M3L.string_creatingServerFolderForMod); Log.Information(@"Creating server folder for mod: " + serverModPath); sftp.CreateDirectory(serverModPath); justMadeFolder = true; } var dirContents = sftp.ListDirectory(serverModPath).ToList(); Dictionary <string, string> serverHashes = new Dictionary <string, string>(); //Open SSH connection as we will need to hash files out afterwards. Log.Information(@"Connecting to ME3Tweaks Updater Service over SSH (SSH Shell)"); using SshClient sshClient = new SshClient(host, username, password); sshClient.Connect(); Log.Information(@"Connected to ME3Tweaks Updater Service over SSH (SSH Shell)"); if (!justMadeFolder && dirContents.Any(x => x.Name != @"." && x.Name != @"..")) { CurrentActionText = M3L.GetString(M3L.string_hashingFilesOnServerForDelta); Log.Information(@"Hashing existing files on server to compare for delta"); serverHashes = getServerHashes(sshClient, serverFolderName, serverModPath); } //Calculate what needs to be updated or removed from server List <string> filesToUploadToServer = new List <string>(); List <string> filesToDeleteOffServer = new List <string>(); //Files to upload foreach (var sourceFile in manifestFiles) { //find matching server file if (serverHashes.TryGetValue(sourceFile.Key.Replace('\\', '/') + @".lzma", out var matchingHash)) { //exists on server, compare hash if (matchingHash != sourceFile.Value.lzmahash) { //server hash is different! Upload new file. Log.Information(@"Server version of file is different from local: " + sourceFile.Key); filesToUploadToServer.Add(sourceFile.Key); } else { Log.Information(@"Server version of file is same as local: " + sourceFile.Key); } } else { Log.Information(@"Server does not have file: " + sourceFile.Key); filesToUploadToServer.Add(sourceFile.Key); } } //Files to remove from server foreach (var serverfile in serverHashes.Keys) { if (!manifestFiles.Any(x => (x.Key + @".lzma") == serverfile.Replace('/', '\\'))) { Log.Information(@"File exists on server but not locally: " + serverfile); filesToDeleteOffServer.Add(serverfile); } } #endregion long amountUploaded = 0, amountToUpload = 1; //Confirm changes if (filesToDeleteOffServer.Any() || filesToUploadToServer.Any()) { var text = M3L.GetString(M3L.string_interp_updaterServiceDeltaConfirmationHeader, mod.ModName); if (filesToUploadToServer.Any()) { text += M3L.GetString(M3L.string_nnFilesToUploadToServern) + @" " + string.Join('\n' + @" - ", filesToUploadToServer); //weird stuff to deal with localizer } if (filesToDeleteOffServer.Any()) { text += M3L.GetString(M3L.string_nnFilesToDeleteOffServern) + @" " + string.Join('\n' + @" - ", filesToDeleteOffServer); //weird stuff to deal with localizer } text += M3L.GetString(M3L.string_interp_updaterServiceDeltaConfirmationFooter); bool performUpload = false; Log.Information(@"Prompting user to accept server delta"); setTaskbarProgressState?.Invoke(TaskbarItemProgressState.Paused); Application.Current.Dispatcher.Invoke(delegate { performUpload = M3L.ShowDialog(mainwindow, text, M3L.GetString(M3L.string_confirmChanges), MessageBoxButton.OKCancel, MessageBoxImage.Exclamation) == MessageBoxResult.OK; }); setTaskbarProgressState?.Invoke(TaskbarItemProgressState.Indeterminate); if (performUpload) { Log.Information(@"User has accepted the delta, applying delta to server"); #region upload files //Create directories SortedSet <string> directoriesToCreate = new SortedSet <string>(); foreach (var f in filesToUploadToServer) { string foldername = f; var lastIndex = foldername.LastIndexOf(@"\"); while (lastIndex > 0) { foldername = foldername.Substring(0, lastIndex); directoriesToCreate.Add(foldername.Replace('\\', '/')); lastIndex = foldername.LastIndexOf(@"\"); } } #endregion //UploadDirectory(sftp, lzmaStagingPath, serverModPath, (ucb) => Debug.WriteLine("UCB: " + ucb)); var dirsToCreateOnServerSorted = directoriesToCreate.ToList(); dirsToCreateOnServerSorted.Sort((a, b) => a.Length.CompareTo(b.Length)); //short to longest so we create top levels first! int numFoldersToCreate = dirsToCreateOnServerSorted.Count(); int numDone = 0; if (dirsToCreateOnServerSorted.Count > 0) { CurrentActionText = M3L.GetString(M3L.string_creatingModDirectoriesOnServer); foreach (var f in dirsToCreateOnServerSorted) { var serverFolderStr = serverModPath + @"/" + f; if (!sftp.Exists(serverFolderStr)) { Log.Information(@"Creating directory on server: " + serverFolderStr); sftp.CreateDirectory(serverFolderStr); } else { Log.Information(@"Server folder already exists, skipping: " + serverFolderStr); } numDone++; CurrentActionText = M3L.GetString(M3L.string_creatingModDirectoriesOnServer) + @" " + Math.Round(numDone * 100.0 / numFoldersToCreate) + @"%"; } } //Upload files progressCallback?.Invoke(0); setTaskbarProgressState?.Invoke(TaskbarItemProgressState.Normal); amountToUpload = filesToUploadToServer.Sum(x => new FileInfo(Path.Combine(lzmaStagingPath, x + @".lzma")).Length); foreach (var file in filesToUploadToServer) { if (CancelOperations) { AbortUpload(); return(UploadModResult.ABORTED_BY_USER); } var fullPath = Path.Combine(lzmaStagingPath, file + @".lzma"); var serverFilePath = serverModPath + @"/" + file.Replace(@"\", @"/") + @".lzma"; Log.Information(@"Uploading file " + fullPath + @" to " + serverFilePath); long amountUploadedBeforeChunk = amountUploaded; using Stream fileStream = new FileStream(fullPath, FileMode.Open); sftp.UploadFile(fileStream, serverFilePath, true, (x) => { if (CancelOperations) { CurrentActionText = M3L.GetString(M3L.string_abortingUpload); return; } amountUploaded = amountUploadedBeforeChunk + (long)x; var uploadedHR = ByteSize.FromBytes(amountUploaded).ToString(@"0.00"); var totalUploadHR = ByteSize.FromBytes(amountToUpload).ToString(@"0.00"); if (amountToUpload > 0) { progressCallback?.Invoke(amountUploaded * 1.0 / amountToUpload); } CurrentActionText = M3L.GetString(M3L.string_interp_uploadingFilesToServerXY, uploadedHR, totalUploadHR); }); } setTaskbarProgressState?.Invoke(TaskbarItemProgressState.Indeterminate); if (CancelOperations) { AbortUpload(); return(UploadModResult.ABORTED_BY_USER); } //delete extra files int numdone = 0; foreach (var file in filesToDeleteOffServer) { CurrentActionText = M3L.GetString(M3L.string_interp_deletingObsoleteFiles, numdone, filesToDeleteOffServer.Count); var fullPath = $@"{LZMAStoragePath}/{serverFolderName}/{file}"; Log.Information(@"Deleting unused file off server: " + fullPath); sftp.DeleteFile(fullPath); numdone++; } //Upload manifest using var manifestStream = finalManifestText.ToStream(); var serverManifestPath = $@"{ManifestStoragePath}/{serverFolderName}.xml"; Log.Information(@"Uploading manifest to server: " + serverManifestPath); sftp.UploadFile(manifestStream, serverManifestPath, true, (x) => { var uploadedAmountHR = ByteSize.FromBytes(amountUploaded).ToString(@"0.00"); var uploadAmountTotalHR = ByteSize.FromBytes(amountToUpload).ToString(@"0.00"); CurrentActionText = M3L.GetString(M3L.string_uploadingUpdateManifestToServer) + $@"{uploadedAmountHR}/{uploadAmountTotalHR}"; }); } else { Log.Warning(@"User has declined uploading the delta. We will not change anything on the server."); CancelOperations = true; AbortUpload(); return(UploadModResult.ABORTED_BY_USER); } CurrentActionText = M3L.GetString(M3L.string_validatingModOnServer); Log.Information(@"Verifying hashes on server for new files"); var newServerhashes = getServerHashes(sshClient, serverFolderName, serverModPath); var badHashes = verifyHashes(manifestFiles, newServerhashes); if (badHashes.Any()) { CurrentActionText = M3L.GetString(M3L.string_someHashesOnServerAreIncorrectContactMgamerz); return(UploadModResult.BAD_SERVER_HASHES_AFTER_VALIDATION); } else { CurrentActionText = M3L.GetString(M3L.string_modUploadedToUpdaterService); return(UploadModResult.UPLOAD_OK); } } return(UploadModResult.ABORTED_BY_USER); }
/// <summary> /// Checks mods for updates. ForceUpdateCheck will force the mod to validate against the server (essentially repair mode). It is not used for rate limiting! /// </summary> /// <param name="modsToCheck">Mods to have server send information about</param> /// <param name="forceUpdateCheck">Force update check regardless of version</param> /// <returns></returns> public static List <ModUpdateInfo> CheckForModUpdates(List <Mod> modsToCheck, bool forceUpdateCheck, Action <string> updateStatusCallback = null) { string updateFinalRequest = UpdaterServiceManifestEndpoint; bool first = true; foreach (var mod in modsToCheck) { if (mod.ModModMakerID > 0) { //Modmaker style if (first) { updateFinalRequest += "?"; first = false; } else { updateFinalRequest += "&"; } updateFinalRequest += "modmakerupdatecode[]=" + mod.ModModMakerID; } else if (mod.ModClassicUpdateCode > 0) { //Classic style if (first) { updateFinalRequest += "?"; first = false; } else { updateFinalRequest += "&"; } updateFinalRequest += "classicupdatecode[]=" + mod.ModClassicUpdateCode; } else if (mod.NexusModID > 0 && mod.NexusUpdateCheck) { //Nexus style if (first) { updateFinalRequest += "?"; first = false; } else { updateFinalRequest += "&"; } updateFinalRequest += "nexusupdatecode[]=" + mod.Game.ToString().Substring(2) + "-" + mod.NexusModID; } //else if (mod.NexusModID > 0) //{ // //Classic style // if (first) // { // updateFinalRequest += "?"; // first = false; // } // else // { // updateFinalRequest += "&"; // } // updateFinalRequest += "nexusupdatecode[]=" + mod.ModClassicUpdateCode; //} } using var wc = new System.Net.WebClient(); try { Debug.WriteLine(updateFinalRequest); string updatexml = wc.DownloadStringAwareOfEncoding(updateFinalRequest); XElement rootElement = XElement.Parse(updatexml); #region classic mods var modUpdateInfos = new List <ModUpdateInfo>(); var classicUpdateInfos = (from e in rootElement.Elements("mod") select new ModUpdateInfo { changelog = (string)e.Attribute("changelog"), versionstr = (string)e.Attribute("version"), updatecode = (int)e.Attribute("updatecode"), serverfolder = (string)e.Attribute("folder"), sourceFiles = (from f in e.Elements("sourcefile") select new SourceFile { lzmahash = (string)f.Attribute("lzmahash"), hash = (string)f.Attribute("hash"), size = (int)f.Attribute("size"), lzmasize = (int)f.Attribute("lzmasize"), relativefilepath = f.Value, timestamp = (Int64?)f.Attribute("timestamp") ?? (Int64)0 }).ToList(), blacklistedFiles = e.Elements("blacklistedfile").Select(x => x.Value).ToList() }).ToList(); // CALCULATE UPDATE DELTA CaseInsensitiveDictionary <USFileInfo> hashMap = new CaseInsensitiveDictionary <USFileInfo>(); //Used to rename files foreach (var modUpdateInfo in classicUpdateInfos) { modUpdateInfo.ResolveVersionVar(); //Calculate update information var matchingMod = modsToCheck.FirstOrDefault(x => x.ModClassicUpdateCode == modUpdateInfo.updatecode); if (matchingMod != null && (forceUpdateCheck || matchingMod.ParsedModVersion < modUpdateInfo.version)) { // The following line is left so we know that it was at one point considered implemented. // This prevents updating copies of the same mod in the library. Cause it's just kind of a bandwidth waste. //modsToCheck.Remove(matchingMod); //This makes it so we don't feed in multiple same-mods. For example, nexus check on 3 Project Variety downloads modUpdateInfo.mod = matchingMod; modUpdateInfo.SetLocalizedInfo(); string modBasepath = matchingMod.ModPath; double i = 0; List <string> references = null; try { references = matchingMod.GetAllRelativeReferences(true); } catch { // There's an error. Underlying disk state may have changed since we originally loaded the mod } if (references == null || !matchingMod.ValidMod) { // The mod failed to load. We should just index everything the // references will not be dfully parsed. var localFiles = Directory.GetFiles(matchingMod.ModPath, "*", SearchOption.AllDirectories); references = localFiles.Select(x => x.Substring(matchingMod.ModPath.Length + 1)).ToList(); } int total = references.Count; // Index existing files foreach (var v in references) { updateStatusCallback?.Invoke( $"Indexing {modUpdateInfo.mod.ModName} for updates {(int)(i * 100 / total)}%"); i++; var fpath = Path.Combine(matchingMod.ModPath, v); if (fpath.RepresentsPackageFilePath()) { // We need to make sure it's decompressed var qPackage = MEPackageHandler.QuickOpenMEPackage(fpath); if (qPackage.IsCompressed) { CLog.Information( $" >> Decompressing compressed package for update comparison check: {fpath}", Settings.LogModUpdater); qPackage = MEPackageHandler.OpenMEPackage(fpath); MemoryStream tStream = new MemoryStream(); tStream = qPackage.SaveToStream(false); hashMap[v] = new USFileInfo() { MD5 = Utilities.CalculateMD5(tStream), CompressedMD5 = Utilities.CalculateMD5(fpath), Filesize = tStream.Length, RelativeFilepath = v }; continue; } } hashMap[v] = new USFileInfo() { MD5 = Utilities.CalculateMD5(fpath), Filesize = new FileInfo(fpath).Length, RelativeFilepath = v }; } i = 0; total = modUpdateInfo.sourceFiles.Count; foreach (var serverFile in modUpdateInfo.sourceFiles) { Log.Information($@"Checking {serverFile.relativefilepath} for update applicability"); updateStatusCallback?.Invoke( $"Calculating update delta for {modUpdateInfo.mod.ModName} {(int)(i * 100 / total)}%"); i++; bool calculatedOp = false; if (hashMap.TryGetValue(serverFile.relativefilepath, out var indexInfo)) { if (indexInfo.MD5 == serverFile.hash) { CLog.Information(@" >> File is up to date", Settings.LogModUpdater); calculatedOp = true; } else if (indexInfo.CompressedMD5 != null && indexInfo.CompressedMD5 == serverFile.hash) { CLog.Information(@" >> Compressed package file is up to date", Settings.LogModUpdater); calculatedOp = true; } } if (!calculatedOp) { // File is missing or hash was wrong. We should try to map it to another existing file // to save bandwidth var existingFilesThatMatchServerHash = hashMap.Where(x => x.Value.MD5 == serverFile.hash || (x.Value.CompressedMD5 != null && x.Value.CompressedMD5 == serverFile.hash)) .ToList(); if (existingFilesThatMatchServerHash.Any()) { CLog.Information( $" >> Server file can be cloned from local file {existingFilesThatMatchServerHash[0].Value.RelativeFilepath} as it has same hash", Settings.LogModUpdater); modUpdateInfo.cloneOperations[serverFile] = existingFilesThatMatchServerHash[0] .Value; // Server file can be sourced from the value } else if (indexInfo == null) { // we don't have file hashed (new file) CLog.Information( $" >> Applicable for updates, File does not exist locally", Settings.LogModUpdater); modUpdateInfo.applicableUpdates.Add(serverFile); } else { // Existing file has wrong hash CLog.Information($" >> Applicable for updates, hash has changed", Settings.LogModUpdater); modUpdateInfo.applicableUpdates.Add(serverFile); } } } foreach (var blacklistedFile in modUpdateInfo.blacklistedFiles) { var blLocalFile = Path.Combine(modBasepath, blacklistedFile); if (File.Exists(blLocalFile)) { Log.Information(@"Blacklisted file marked for deletion: " + blLocalFile); modUpdateInfo.filesToDelete.Add(blLocalFile); } } // alphabetize files modUpdateInfo.applicableUpdates.Sort(x => x.relativefilepath); //Files to remove calculation var modFiles = Directory.GetFiles(modBasepath, "*", SearchOption.AllDirectories) .Select(x => x.Substring(modBasepath.Length + 1)).ToList(); var additionalFilesToDelete = modFiles.Except( modUpdateInfo.sourceFiles.Select(x => x.relativefilepath), StringComparer.InvariantCultureIgnoreCase).Distinct().ToList(); modUpdateInfo.filesToDelete.AddRange( additionalFilesToDelete); //Todo: Add security check here to prevent malicious modUpdateInfo.TotalBytesToDownload = modUpdateInfo.applicableUpdates.Sum(x => x.lzmasize); } } modUpdateInfos.AddRange(classicUpdateInfos); #endregion #region modmaker mods var modmakerModUpdateInfos = (from e in rootElement.Elements("modmakermod") select new ModMakerModUpdateInfo { ModMakerId = (int)e.Attribute("id"), versionstr = (string)e.Attribute("version"), PublishDate = DateTime.ParseExact((string)e.Attribute("publishdate"), "yyyy-MM-dd", CultureInfo.InvariantCulture), changelog = (string)e.Attribute("changelog") }).ToList(); modUpdateInfos.AddRange(modmakerModUpdateInfos); #endregion #region Nexus Mod Third Party var nexusModsUpdateInfo = (from e in rootElement.Elements("nexusmod") select new NexusModUpdateInfo { NexusModsId = (int)e.Attribute("id"), GameId = (int)e.Attribute("game"), versionstr = (string)e.Attribute("version"), UpdatedTime = DateTimeOffset.FromUnixTimeSeconds((long)e.Attribute("updated_timestamp")) .DateTime }).ToList(); modUpdateInfos.AddRange(nexusModsUpdateInfo); #endregion return(modUpdateInfos); } catch (Exception e) { Log.Error("Error checking for mod updates: " + App.FlattenException(e)); Crashes.TrackError(e, new Dictionary <string, string>() { { "Update check URL", updateFinalRequest } }); } return(null); }