public static string FetchExeTransform(string name) { using var wc = new ShortTimeoutWebClient(); string moddesc = wc.DownloadStringAwareOfEncoding(ExeTransformBaseURL + name); return(moddesc); }
public static (MemoryStream result, string errorMessage) FetchString(string url) { using var wc = new ShortTimeoutWebClient(); string downloadError = null; MemoryStream responseStream = null; wc.DownloadDataCompleted += (a, args) => { downloadError = args.Error?.Message; if (downloadError == null) { responseStream = new MemoryStream(args.Result); } lock (args.UserState) { //releases blocked thread Monitor.Pulse(args.UserState); } }; var syncObject = new Object(); lock (syncObject) { Debug.WriteLine(@"Download file to memory: " + url); wc.DownloadDataAsync(new Uri(url), syncObject); //This will block the thread until download completes Monitor.Wait(syncObject); } return(responseStream, downloadError); }
public static string FetchThirdPartyModdesc(string name) { using var wc = new ShortTimeoutWebClient(); string moddesc = wc.DownloadStringAwareOfEncoding(ThirdPartyModDescURL + name); return(moddesc); }
public static (MemoryStream download, string errorMessage) DownloadStaticAsset(string assetName, Action <long, long> progressCallback = null) { (MemoryStream, string)result = (null, @"Could not download file: No attempt was made, or errors occurred!"); foreach (var staticurl in StaticFilesBaseEndpoints) { try { using var wc = new ShortTimeoutWebClient(); { var fullURL = staticurl + assetName; result = DownloadToMemory(fullURL, logDownload: true, progressCallback: progressCallback); if (result.Item2 == null) { return(result); } } } catch (Exception e) { Log.Error($@"Could not download {assetName} from endpoint {staticurl}: {e.Message}"); } } return(result); }
public static Dictionary <string, string> FetchOnlineStartupManifest(bool betamode) { string[] ulrs = new[] { StartupManifestURL, StartupManifestBackupURL }; foreach (var staticurl in ulrs) { Uri myUri = new Uri(staticurl); string host = myUri.Host; var fetchUrl = staticurl; if (betamode && host == @"me3tweaks.com") { fetchUrl += @"&beta=true"; //only me3tweaks source supports beta. fallback will always just use whatever was live when it synced } try { using var wc = new ShortTimeoutWebClient(); string json = wc.DownloadString(fetchUrl); App.ServerManifest = JsonConvert.DeserializeObject <Dictionary <string, string> >(json); Log.Information($@"Fetched startup manifest from endpoint {host}"); return(App.ServerManifest); } catch (Exception e) { Log.Error($@"Unable to fetch startup manifest from endpoint {host}: {e.Message}"); } } Log.Error(@"Failed to fetch startup manifest."); return(new Dictionary <string, string>()); }
/// <summary> /// Downloads a static asset that is mirrored onto the ME3Tweaks Assets repo. This is not the same as the github version of staticfiles. /// </summary> /// <param name="assetName">The asset filename. Do not include any path information.</param> /// <returns></returns> public static (MemoryStream download, string errorMessage) DownloadME3TweaksStaticAsset(string assetName) { (MemoryStream, string)result = (null, @"Could not download file: No attempt was made, or errors occurred!"); foreach (var staticurl in ME3TweaksStaticFilesBaseEndpoints) { try { using var wc = new ShortTimeoutWebClient(); { var fullURL = staticurl + Path.GetFileNameWithoutExtension(assetName) + "/" + assetName; result = DownloadToMemory(fullURL, logDownload: true); if (result.Item2 == null) { return(result); } } } catch (Exception e) { Log.Error($@"Could not download {assetName} from endpoint {staticurl}: {e.Message}"); } } return(result); }
public static string FetchRemoteString(string url) { try { using var wc = new ShortTimeoutWebClient(); return(wc.DownloadStringAwareOfEncoding(url)); } catch (Exception e) { Log.Error("Error downloading string: " + e.Message); return(null); } }
public static Dictionary <string, string> FetchOnlineStartupManifest(bool betamode) { using var wc = new ShortTimeoutWebClient(); var fetchUrl = StartupManifestURL; if (betamode) { fetchUrl += "&beta=true"; } string json = wc.DownloadString(fetchUrl); App.ServerManifest = JsonConvert.DeserializeObject <Dictionary <string, string> >(json); return(App.ServerManifest); }
public static bool EnsureCriticalFiles() { //7-zip try { string sevenZDLL = Utilities.Get7zDllPath(); if (!File.Exists(sevenZDLL) || Utilities.CalculateMD5(sevenZDLL) != "72491c7b87a7c2dd350b727444f13bb4") { foreach (var staticurl in StaticFilesBaseEndpoints) { try { using var wc = new ShortTimeoutWebClient(); { var fullURL = staticurl + "7z.dll"; Log.Information("Downloading 7z.dll: " + fullURL); wc.DownloadFile(fullURL, sevenZDLL); break; //No more loops } } catch (Exception e) { Log.Error($"Could not download 7z.dll from endpoint {staticurl} {e.Message}"); } } } if (File.Exists(sevenZDLL)) { Log.Information("Setting 7z dll path: " + sevenZDLL); var p = Path.GetFullPath(sevenZDLL); SevenZip.SevenZipBase.SetLibraryPath(sevenZDLL); } else { Log.Fatal("Unable to load 7z dll! File doesn't exist: " + sevenZDLL); return(false); } } catch (Exception e) { Log.Error("Exception ensuring critical files: " + App.FlattenException(e)); return(false); } return(true); }
public static string FetchRemoteString(string url, string authorizationToken = null) { try { using var wc = new ShortTimeoutWebClient(); if (authorizationToken != null) { wc.Headers.Add(@"Authorization", authorizationToken); } return(wc.DownloadStringAwareOfEncoding(url)); } catch (Exception e) { Log.Error(@"Error downloading string: " + e.Message); return(null); } }
/// <summary> /// Downloads from a URL to memory. This is a blocking call and should be done on a background thread. /// </summary> /// <param name="url">URL to download from</param> /// <param name="progressCallback">Progress information clalback</param> /// <param name="hash">Hash check value (md5). Leave null if no hash check</param> /// <returns></returns> public static (MemoryStream result, string errorMessage) DownloadToMemory(string url, Action <long, long> progressCallback = null, string hash = null) { using var wc = new ShortTimeoutWebClient(); string downloadError = null; MemoryStream responseStream = null; wc.DownloadProgressChanged += (a, args) => { progressCallback?.Invoke(args.BytesReceived, args.TotalBytesToReceive); }; wc.DownloadDataCompleted += (a, args) => { downloadError = args.Error?.Message; if (downloadError == null) { responseStream = new MemoryStream(args.Result); if (hash != null) { var md5 = Utilities.CalculateMD5(responseStream); responseStream.Position = 0; if (md5 != hash) { responseStream = null; downloadError = $"Hash of downloaded item ({url}) does not match expected hash. Expected: {hash}, got: {md5}"; //needs localized } } } lock (args.UserState) { //releases blocked thread Monitor.Pulse(args.UserState); } }; var syncObject = new Object(); lock (syncObject) { Debug.WriteLine("Download file to memory: " + url); wc.DownloadDataAsync(new Uri(url), syncObject); //This will block the thread until download completes Monitor.Wait(syncObject); } return(responseStream, downloadError); }
public static bool EnsureStaticAssets() { string[] objectInfoFiles = { "ME1ObjectInfo.json", "ME2ObjectInfo.json", "ME3ObjectInfo.json" }; string localBaseDir = Utilities.GetObjectInfoFolder(); try { foreach (var info in objectInfoFiles) { var localPath = Path.Combine(localBaseDir, info); if (!File.Exists(localPath)) { foreach (var staticurl in StaticFilesBaseEndpoints) { var fullURL = staticurl + "objectinfos/" + info; try { using var wc = new ShortTimeoutWebClient(); Log.Information("Downloading static asset: " + fullURL); wc.DownloadFile(fullURL, localPath); break; } catch (Exception e) { Log.Error($"Could not download {info} from endpoint {fullURL} {e.Message}"); } } } } } catch (Exception e) { Log.Error("Exception trying to ensure static assets: " + e.Message); Crashes.TrackError(new Exception(@"Could not download static supporting files: " + e.Message)); return(false); } return(true); }
public static Dictionary <string, string> QueryModRelay(string md5, long size) { //Todo: Finish implementing relay service string finalRelayURL = $"{ModInfoRelayEndpoint}?modmanagerversion={App.BuildNumber}&md5={md5.ToLowerInvariant()}&size={size}"; try { using (var wc = new ShortTimeoutWebClient()) { Debug.WriteLine(finalRelayURL); string json = wc.DownloadStringAwareOfEncoding(finalRelayURL); //todo: Implement response format serverside return(JsonConvert.DeserializeObject <Dictionary <string, string> >(json)); } } catch (Exception e) { Log.Error("Error querying relay service from ME3Tweaks: " + App.FlattenException(e)); } return(null); }
public static Dictionary <string, CaseInsensitiveDictionary <List <BasegameFileIdentificationService.BasegameCloudDBFile> > > FetchBasegameFileIdentificationServiceManifest(bool overrideThrottling = false) { Log.Information(@"Fetching basegame file identification manifest"); //read cached first. string cached = null; if (File.Exists(Utilities.GetBasegameIdentificationCacheFile())) { try { cached = File.ReadAllText(Utilities.GetBasegameIdentificationCacheFile()); } catch (Exception e) { var attachments = new List <ErrorAttachmentLog>(); string log = LogCollector.CollectLatestLog(true); if (log != null && log.Length < ByteSizeLib.ByteSize.BytesInMegaByte * 7) { attachments.Add(ErrorAttachmentLog.AttachmentWithText(log, @"applog.txt")); } Crashes.TrackError(e, new Dictionary <string, string>() { { @"Error type", @"Error reading cached online content" }, { @"Service", @"Basegame File Identification Service" }, { @"Message", e.Message } }, attachments.ToArray()); } } if (!File.Exists(Utilities.GetBasegameIdentificationCacheFile()) || overrideThrottling || OnlineContent.CanFetchContentThrottleCheck()) { var urls = new[] { BasegameFileIdentificationServiceURL, BasegameFileIdentificationServiceBackupURL }; foreach (var staticurl in urls) { Uri myUri = new Uri(staticurl); string host = myUri.Host; try { using var wc = new ShortTimeoutWebClient(); string json = wc.DownloadStringAwareOfEncoding(staticurl); File.WriteAllText(Utilities.GetBasegameIdentificationCacheFile(), json); return(JsonConvert.DeserializeObject <Dictionary <string, CaseInsensitiveDictionary <List <BasegameFileIdentificationService.BasegameCloudDBFile> > > >(json)); } catch (Exception e) { //Unable to fetch latest help. Log.Error($"Error fetching online basegame file identification service from endpoint {host}: {e.Message}"); } } if (cached == null) { Log.Error("Unable to load basegame file identification service and local file doesn't exist. Returning a blank copy."); Dictionary <string, CaseInsensitiveDictionary <List <BasegameFileIdentificationService.BasegameCloudDBFile> > > d = new Dictionary <string, CaseInsensitiveDictionary <List <BasegameFileIdentificationService.BasegameCloudDBFile> > > { ["ME1"] = new CaseInsensitiveDictionary <List <BasegameFileIdentificationService.BasegameCloudDBFile> >(), ["ME2"] = new CaseInsensitiveDictionary <List <BasegameFileIdentificationService.BasegameCloudDBFile> >(), ["ME3"] = new CaseInsensitiveDictionary <List <BasegameFileIdentificationService.BasegameCloudDBFile> >() }; return(d); } } Log.Information("Using cached BGFIS instead"); try { return(JsonConvert.DeserializeObject <Dictionary <string, CaseInsensitiveDictionary <List <BasegameFileIdentificationService.BasegameCloudDBFile> > > >(cached)); } catch (Exception e) { Log.Error("Could not parse cached basegame file identification service file. Returning blank BFIS data instead. Reason: " + e.Message); return(new Dictionary <string, CaseInsensitiveDictionary <List <BasegameFileIdentificationService.BasegameCloudDBFile> > > { ["ME1"] = new CaseInsensitiveDictionary <List <BasegameFileIdentificationService.BasegameCloudDBFile> >(), ["ME2"] = new CaseInsensitiveDictionary <List <BasegameFileIdentificationService.BasegameCloudDBFile> >(), ["ME3"] = new CaseInsensitiveDictionary <List <BasegameFileIdentificationService.BasegameCloudDBFile> >() }); } }
public static Dictionary <long, List <ThirdPartyServices.ThirdPartyImportingInfo> > FetchThirdPartyImportingService(bool overrideThrottling = false) { string cached = null; if (File.Exists(Utilities.GetThirdPartyImportingCachedFile())) { try { cached = File.ReadAllText(Utilities.GetThirdPartyImportingCachedFile()); } catch (Exception e) { var attachments = new List <ErrorAttachmentLog>(); string log = LogCollector.CollectLatestLog(true); if (log != null && log.Length < ByteSizeLib.ByteSize.BytesInMegaByte * 7) { attachments.Add(ErrorAttachmentLog.AttachmentWithText(log, "applog.txt")); } Crashes.TrackError(e, new Dictionary <string, string>() { { "Error type", "Error reading cached online content" }, { "Service", "Third Party Importing Service" }, { "Message", e.Message } }, attachments.ToArray()); } } if (!File.Exists(Utilities.GetThirdPartyImportingCachedFile()) || overrideThrottling || OnlineContent.CanFetchContentThrottleCheck()) { try { using var wc = new ShortTimeoutWebClient(); string json = wc.DownloadStringAwareOfEncoding(ThirdPartyImportingServiceURL); File.WriteAllText(Utilities.GetThirdPartyImportingCachedFile(), json); return(JsonConvert.DeserializeObject <Dictionary <long, List <ThirdPartyServices.ThirdPartyImportingInfo> > >(json)); } catch (Exception e) { //Unable to fetch latest help. Log.Error("Error fetching latest importing service file: " + e.Message); if (cached != null) { Log.Warning("Using cached third party importing service file instead"); } else { Log.Error("Unable to fetch latest third party importing service file from server and local file doesn't exist. Returning a blank copy."); return(new Dictionary <long, List <ThirdPartyServices.ThirdPartyImportingInfo> >()); } } } try { return(JsonConvert.DeserializeObject <Dictionary <long, List <ThirdPartyServices.ThirdPartyImportingInfo> > >(cached)); } catch (Exception e) { Log.Error("Unable to parse cached importing service file: " + e.Message); return(new Dictionary <long, List <ThirdPartyServices.ThirdPartyImportingInfo> >()); } }
public static List <IntroTutorial.TutorialStep> FetchTutorialManifest(bool overrideThrottling = false) { Log.Information(@"Fetching tutorial manifest"); string cached = null; // Read cached first. if (File.Exists(Utilities.GetTutorialServiceCacheFile())) { try { cached = File.ReadAllText(Utilities.GetTutorialServiceCacheFile()); } catch (Exception e) { var attachments = new List <ErrorAttachmentLog>(); string log = LogCollector.CollectLatestLog(true); if (log != null && log.Length < FileSize.MebiByte * 7) { attachments.Add(ErrorAttachmentLog.AttachmentWithText(log, @"applog.txt")); } Crashes.TrackError(e, new Dictionary <string, string>() { { @"Error type", @"Error reading cached online content" }, { @"Service", @"Tutorial Service" }, { @"Message", e.Message } }, attachments.ToArray()); } } if (!File.Exists(Utilities.GetTutorialServiceCacheFile()) || overrideThrottling || OnlineContent.CanFetchContentThrottleCheck()) { string[] urls = new[] { TutorialServiceURL, TutorialServiceBackupURL }; foreach (var staticurl in urls) { Uri myUri = new Uri(staticurl); string host = myUri.Host; try { using var wc = new ShortTimeoutWebClient(); string json = wc.DownloadStringAwareOfEncoding(staticurl); File.WriteAllText(Utilities.GetTutorialServiceCacheFile(), json); return(JsonConvert.DeserializeObject <List <IntroTutorial.TutorialStep> >(json)); } catch (Exception e) { //Unable to fetch latest help. Log.Error($@"Error fetching latest tutorial service file from endpoint {host}: {e.Message}"); } } if (cached == null) { Log.Error(@"Unable to fetch latest tutorial service file from server and local file doesn't exist. Returning a blank copy."); return(new List <IntroTutorial.TutorialStep>()); } } Log.Information(@"Using cached tutorial service file"); try { return(JsonConvert.DeserializeObject <List <IntroTutorial.TutorialStep> >(cached)); } catch (Exception e) { Log.Error(@"Unable to parse cached importing service file: " + e.Message); return(new List <IntroTutorial.TutorialStep>()); } }
private static (Stream result, string errorMessage) DownloadToStreamInternal(string url, Action <long, long> progressCallback = null, string hash = null, bool logDownload = false, Stream destStreamOverride = null, CancellationToken cancellationToken = default) { using var wc = new ShortTimeoutWebClient(); string downloadError = null; string destType = destStreamOverride != null ? @"stream" : @"memory"; Stream responseStream = destStreamOverride ?? new MemoryStream(); var syncObject = new Object(); lock (syncObject) { if (logDownload) { Log.Information($@"Downloading to {destType}: " + url); } else { Debug.WriteLine($@"Downloading to {destType}: " + url); } try { using var remoteStream = wc.OpenRead(new Uri(url)); long.TryParse(wc.ResponseHeaders[@"Content-Length"], out var totalSize); var buffer = new byte[4096]; int bytesReceived; while ((bytesReceived = remoteStream.Read(buffer, 0, buffer.Length)) != 0) { if (cancellationToken.IsCancellationRequested) { downloadError = M3L.GetString(M3L.string_theDownloadWasCanceled); return(responseStream, downloadError); } responseStream.Write(buffer, 0, bytesReceived); progressCallback?.Invoke(responseStream.Position, totalSize); // Progress } // Check hash if (hash != null) { responseStream.Position = 0; var md5 = MD5.Create().ComputeHashAsync(responseStream, cancellationToken, x => progressCallback?.Invoke(x, 100)).Result; responseStream.Position = 0; if (md5 != hash) { responseStream = null; downloadError = M3L.GetString(M3L.string_interp_onlineContentHashWrong, url, hash, md5); //needs localized } } } catch (Exception e) { downloadError = e.Message; } } return(responseStream, downloadError); }
public static Dictionary <string, CaseInsensitiveDictionary <ThirdPartyServices.ThirdPartyModInfo> > FetchThirdPartyIdentificationManifest(bool overrideThrottling = false) { string cached = null; if (File.Exists(Utilities.GetThirdPartyIdentificationCachedFile())) { try { cached = File.ReadAllText(Utilities.GetThirdPartyIdentificationCachedFile()); } catch (Exception e) { var attachments = new List <ErrorAttachmentLog>(); string log = LogCollector.CollectLatestLog(true); if (log != null && log.Length < FileSize.MebiByte * 7) { attachments.Add(ErrorAttachmentLog.AttachmentWithText(log, @"applog.txt")); } Crashes.TrackError(e, new Dictionary <string, string>() { { @"Error type", @"Error reading cached online content" }, { @"Service", @"Third Party Identification Service" }, { @"Message", e.Message } }, attachments.ToArray()); } } if (!File.Exists(Utilities.GetThirdPartyIdentificationCachedFile()) || overrideThrottling || OnlineContent.CanFetchContentThrottleCheck()) { try { using var wc = new ShortTimeoutWebClient(); string json = wc.DownloadStringAwareOfEncoding(ThirdPartyIdentificationServiceURL); File.WriteAllText(Utilities.GetThirdPartyIdentificationCachedFile(), json); return(JsonConvert.DeserializeObject <Dictionary <string, CaseInsensitiveDictionary <ThirdPartyServices.ThirdPartyModInfo> > >(json)); } catch (Exception e) { //Unable to fetch latest help. Log.Error(@"Error fetching online third party identification service: " + e.Message); if (cached != null) { Log.Warning(@"Using cached third party identification service file instead"); } else { Log.Error(@"Unable to load third party identification service and local file doesn't exist. Returning a blank copy."); Dictionary <string, CaseInsensitiveDictionary <ThirdPartyServices.ThirdPartyModInfo> > d = new Dictionary <string, CaseInsensitiveDictionary <ThirdPartyServices.ThirdPartyModInfo> > { [@"ME1"] = new CaseInsensitiveDictionary <ThirdPartyServices.ThirdPartyModInfo>(), [@"ME2"] = new CaseInsensitiveDictionary <ThirdPartyServices.ThirdPartyModInfo>(), [@"ME3"] = new CaseInsensitiveDictionary <ThirdPartyServices.ThirdPartyModInfo>() }; return(d); } } } try { return(JsonConvert.DeserializeObject <Dictionary <string, CaseInsensitiveDictionary <ThirdPartyServices.ThirdPartyModInfo> > >(cached)); } catch (Exception e) { Log.Error(@"Could not parse cached third party identification service file. Returning blank TPMI data instead. Reason: " + e.Message); return(new Dictionary <string, CaseInsensitiveDictionary <ThirdPartyServices.ThirdPartyModInfo> > { [@"ME1"] = new CaseInsensitiveDictionary <ThirdPartyServices.ThirdPartyModInfo>(), [@"ME2"] = new CaseInsensitiveDictionary <ThirdPartyServices.ThirdPartyModInfo>(), [@"ME3"] = new CaseInsensitiveDictionary <ThirdPartyServices.ThirdPartyModInfo>() }); } }
public static Dictionary <string, CaseInsensitiveDictionary <List <BasegameFileIdentificationService.BasegameCloudDBFile> > > FetchBasegameFileIdentificationServiceManifest(bool overrideThrottling = false) { string cached = null; if (File.Exists(Utilities.GetBasegameIdentificationCacheFile())) { try { cached = File.ReadAllText(Utilities.GetBasegameIdentificationCacheFile()); } catch (Exception e) { var attachments = new List <ErrorAttachmentLog>(); string log = LogCollector.CollectLatestLog(true); if (log.Length < ByteSizeLib.ByteSize.BytesInMegaByte * 7) { attachments.Add(ErrorAttachmentLog.AttachmentWithText(log, "applog.txt")); } Crashes.TrackError(e, new Dictionary <string, string>() { { "Error type", "Error reading cached online content" }, { "Service", "Basegame File Identification Service" }, { "Message", e.Message } }, attachments.ToArray()); } } if (!File.Exists(Utilities.GetBasegameIdentificationCacheFile()) || overrideThrottling || Utilities.CanFetchContentThrottleCheck()) { try { using var wc = new ShortTimeoutWebClient(); string json = wc.DownloadStringAwareOfEncoding(BasegameFileIdentificationServiceURL); File.WriteAllText(Utilities.GetBasegameIdentificationCacheFile(), json); return(JsonConvert.DeserializeObject <Dictionary <string, CaseInsensitiveDictionary <List <BasegameFileIdentificationService.BasegameCloudDBFile> > > >(json)); } catch (Exception e) { //Unable to fetch latest help. Log.Error("Error fetching online basegame file identification service: " + e.Message); if (cached != null) { Log.Warning("Using cached basegame file identification service file instead"); } else { Log.Error("Unable to load basegame file identification service and local file doesn't exist. Returning a blank copy."); Dictionary <string, CaseInsensitiveDictionary <List <BasegameFileIdentificationService.BasegameCloudDBFile> > > d = new Dictionary <string, CaseInsensitiveDictionary <List <BasegameFileIdentificationService.BasegameCloudDBFile> > > { ["ME1"] = new CaseInsensitiveDictionary <List <BasegameFileIdentificationService.BasegameCloudDBFile> >(), ["ME2"] = new CaseInsensitiveDictionary <List <BasegameFileIdentificationService.BasegameCloudDBFile> >(), ["ME3"] = new CaseInsensitiveDictionary <List <BasegameFileIdentificationService.BasegameCloudDBFile> >() }; return(d); } } } try { return(JsonConvert.DeserializeObject <Dictionary <string, CaseInsensitiveDictionary <List <BasegameFileIdentificationService.BasegameCloudDBFile> > > >(cached)); } catch (Exception e) { Log.Error("Could not parse cached basegame file identification service file. Returning blank BFIS data instead. Reason: " + e.Message); return(new Dictionary <string, CaseInsensitiveDictionary <List <BasegameFileIdentificationService.BasegameCloudDBFile> > > { ["ME1"] = new CaseInsensitiveDictionary <List <BasegameFileIdentificationService.BasegameCloudDBFile> >(), ["ME2"] = new CaseInsensitiveDictionary <List <BasegameFileIdentificationService.BasegameCloudDBFile> >(), ["ME3"] = new CaseInsensitiveDictionary <List <BasegameFileIdentificationService.BasegameCloudDBFile> >() }); } }