public async Task <string[]> GetCDNMungedNames() { using var client = await(await BunnyCdnFtpInfo.GetCreds(StorageSpace.AuthoredFiles)).GetClient(); var lst = await client.GetListingAsync(@"\"); return(lst.Select(l => l.Name).ToArray()); }
private static async Task <FtpClient> GetClient(BunnyCdnFtpInfo creds) { var ftpClient = new FtpClient(creds.Hostname, new NetworkCredential(creds.Username, creds.Password)); await ftpClient.ConnectAsync(); return(ftpClient); }
public ModUpgrade(ILogger <ModUpgrade> logger, SqlService sql, DiscordWebHook discord, QuickSync quickSync, AppSettings settings) { _logger = logger; _sql = sql; _discord = discord; _settings = settings; _quickSync = quickSync; _creds = BunnyCdnFtpInfo.GetCreds(StorageSpace.Patches); _mirrorCreds = BunnyCdnFtpInfo.GetCreds(StorageSpace.Mirrors); }
private async Task <FtpClient> GetBunnyCdnFtpClient() { var info = await BunnyCdnFtpInfo.GetCreds(StorageSpace.AuthoredFiles); var client = new FtpClient(info.Hostname) { Credentials = new NetworkCredential(info.Username, info.Password) }; await client.ConnectAsync(); return(client); }
private static async Task <FtpClient> GetClient(BunnyCdnFtpInfo creds = null) { return(await CircuitBreaker.WithAutoRetryAllAsync <FtpClient>(async() => { creds ??= await BunnyCdnFtpInfo.GetCreds(StorageSpace.Mirrors); var ftpClient = new FtpClient(creds.Hostname, new NetworkCredential(creds.Username, creds.Password)); ftpClient.DataConnectionType = FtpDataConnectionType.EPSV; await ftpClient.ConnectAsync(); return ftpClient; })); }
public override async Task <int> Execute() { var toDelete = await FindFilesToDelete(); var log = new[] { $"CDNDelete ({toDelete.CDNDelete.Length}):\n\n" } .Concat(toDelete.CDNDelete) .Concat(new[] { $"SQLDelete ({toDelete.SQLDelete.Length}" }) .Concat(toDelete.SQLDelete) .Concat(new[] { $"CDNRemain ({toDelete.CDNNotDeleted.Length}" }) .Concat(toDelete.CDNNotDeleted) .Concat(new[] { $"SQLRemain ({toDelete.SQLNotDeleted.Length}" }) .Concat(toDelete.SQLNotDeleted) .ToArray(); //await AbsolutePath.EntryPoint.Combine("cdn_delete_log.txt").WriteAllLinesAsync(log); foreach (var sqlFile in toDelete.SQLDelete) { Utils.Log($"Deleting {sqlFile} from SQL"); await _sql.DeleteFileDefinition(await _sql.GetCDNFileDefinition(sqlFile)); } using var queue = new WorkQueue(6); await toDelete.CDNDelete.Select((d, idx) => (d, idx)).PMap(queue, async cdnFile => { using var conn = await(await BunnyCdnFtpInfo.GetCreds(StorageSpace.AuthoredFiles)).GetClient(); Utils.Log($"Deleting {cdnFile} from CDN"); await _discord.Send(Channel.Ham, new DiscordMessage { Content = $"({cdnFile.idx}/{toDelete.CDNDelete.Length}) {cdnFile.d} is no longer referenced by any modlist and will be removed from the CDN" }); if (await conn.DirectoryExistsAsync(cdnFile.d)) { await conn.DeleteDirectoryAsync(cdnFile.d); } if (await conn.FileExistsAsync(cdnFile.d)) { await conn.DeleteFileAsync(cdnFile.d); } }); return(toDelete.CDNDelete.Length + toDelete.SQLDelete.Length); }
public override async Task <int> Execute() { int uploaded = 0; TOP: var toUpload = await _sql.GetNextMirroredFile(); if (toUpload == default) { return(uploaded); } uploaded += 1; try { var creds = await BunnyCdnFtpInfo.GetCreds(StorageSpace.Mirrors); using var queue = new WorkQueue(); if (_archives.TryGetPath(toUpload.Hash, out var path)) { _logger.LogInformation($"Uploading mirror file {toUpload.Hash} {path.Size.FileSizeToString()}"); bool exists = false; using (var client = await GetClient(creds)) { exists = await client.FileExistsAsync($"{toUpload.Hash.ToHex()}/definition.json.gz"); } if (exists) { _logger.LogInformation($"Skipping {toUpload.Hash} it's already on the server"); await toUpload.Finish(_sql); goto TOP; } await _discord.Send(Channel.Spam, new DiscordMessage { Content = $"Uploading {toUpload.Hash} - {toUpload.Created} because {toUpload.Rationale}" }); var definition = await Client.GenerateFileDefinition(queue, path, (s, percent) => { }); using (var client = await GetClient(creds)) { await client.CreateDirectoryAsync($"{definition.Hash.ToHex()}"); await client.CreateDirectoryAsync($"{definition.Hash.ToHex()}/parts"); } string MakePath(long idx) { return($"{definition.Hash.ToHex()}/parts/{idx}"); } await definition.Parts.PMap(queue, async part => { _logger.LogInformation($"Uploading mirror part ({part.Index}/{definition.Parts.Length})"); var buffer = new byte[part.Size]; await using (var fs = await path.OpenShared()) { fs.Position = part.Offset; await fs.ReadAsync(buffer); } using var client = await GetClient(creds); var name = MakePath(part.Index); await client.UploadAsync(new MemoryStream(buffer), name); }); using (var client = await GetClient(creds)) { _logger.LogInformation($"Finishing mirror upload"); await using var ms = new MemoryStream(); await using (var gz = new GZipStream(ms, CompressionLevel.Optimal, true)) { definition.ToJson(gz); } ms.Position = 0; var remoteName = $"{definition.Hash.ToHex()}/definition.json.gz"; await client.UploadAsync(ms, remoteName); } await toUpload.Finish(_sql); } else { await toUpload.Fail(_sql, "Archive not found"); } } catch (Exception ex) { _logger.LogInformation($"{toUpload.Created} {toUpload.Uploaded}"); _logger.LogError(ex, "Error uploading"); await toUpload.Fail(_sql, ex.ToString()); } goto TOP; }
public SqlService(AppSettings settings) { _settings = settings; _mirrorCreds = BunnyCdnFtpInfo.GetCreds(StorageSpace.Mirrors); }