public async Task <string[]> GetCDNMungedNames()
{
using var client = await(await BunnyCdnFtpInfo.GetCreds(StorageSpace.AuthoredFiles)).GetClient();
var lst = await client.GetListingAsync(@"\");
return(lst.Select(l => l.Name).ToArray());
}
private static async Task <FtpClient> GetClient(BunnyCdnFtpInfo creds)
{
var ftpClient = new FtpClient(creds.Hostname, new NetworkCredential(creds.Username, creds.Password));
await ftpClient.ConnectAsync();
return(ftpClient);
}
public ModUpgrade(ILogger <ModUpgrade> logger, SqlService sql, DiscordWebHook discord, QuickSync quickSync, AppSettings settings)
{
_logger = logger;
_sql = sql;
_discord = discord;
_settings = settings;
_quickSync = quickSync;
_creds = BunnyCdnFtpInfo.GetCreds(StorageSpace.Patches);
_mirrorCreds = BunnyCdnFtpInfo.GetCreds(StorageSpace.Mirrors);
}
private async Task <FtpClient> GetBunnyCdnFtpClient()
{
var info = await BunnyCdnFtpInfo.GetCreds(StorageSpace.AuthoredFiles);
var client = new FtpClient(info.Hostname)
{
Credentials = new NetworkCredential(info.Username, info.Password)
};
await client.ConnectAsync();
return(client);
}
private static async Task <FtpClient> GetClient(BunnyCdnFtpInfo creds = null)
{
return(await CircuitBreaker.WithAutoRetryAllAsync <FtpClient>(async() =>
{
creds ??= await BunnyCdnFtpInfo.GetCreds(StorageSpace.Mirrors);
var ftpClient = new FtpClient(creds.Hostname, new NetworkCredential(creds.Username, creds.Password));
ftpClient.DataConnectionType = FtpDataConnectionType.EPSV;
await ftpClient.ConnectAsync();
return ftpClient;
}));
}
public override async Task <int> Execute()
{
var toDelete = await FindFilesToDelete();
var log = new[] { $"CDNDelete ({toDelete.CDNDelete.Length}):\n\n" }
.Concat(toDelete.CDNDelete)
.Concat(new[] { $"SQLDelete ({toDelete.SQLDelete.Length}" })
.Concat(toDelete.SQLDelete)
.Concat(new[] { $"CDNRemain ({toDelete.CDNNotDeleted.Length}" })
.Concat(toDelete.CDNNotDeleted)
.Concat(new[] { $"SQLRemain ({toDelete.SQLNotDeleted.Length}" })
.Concat(toDelete.SQLNotDeleted)
.ToArray();
//await AbsolutePath.EntryPoint.Combine("cdn_delete_log.txt").WriteAllLinesAsync(log);
foreach (var sqlFile in toDelete.SQLDelete)
{
Utils.Log($"Deleting {sqlFile} from SQL");
await _sql.DeleteFileDefinition(await _sql.GetCDNFileDefinition(sqlFile));
}
using var queue = new WorkQueue(6);
await toDelete.CDNDelete.Select((d, idx) => (d, idx)).PMap(queue, async cdnFile =>
{
using var conn = await(await BunnyCdnFtpInfo.GetCreds(StorageSpace.AuthoredFiles)).GetClient();
Utils.Log($"Deleting {cdnFile} from CDN");
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Content =
$"({cdnFile.idx}/{toDelete.CDNDelete.Length}) {cdnFile.d} is no longer referenced by any modlist and will be removed from the CDN"
});
if (await conn.DirectoryExistsAsync(cdnFile.d))
{
await conn.DeleteDirectoryAsync(cdnFile.d);
}
if (await conn.FileExistsAsync(cdnFile.d))
{
await conn.DeleteFileAsync(cdnFile.d);
}
});
return(toDelete.CDNDelete.Length + toDelete.SQLDelete.Length);
}
public override async Task <int> Execute()
{
int uploaded = 0;
TOP:
var toUpload = await _sql.GetNextMirroredFile();
if (toUpload == default)
{
return(uploaded);
}
uploaded += 1;
try
{
var creds = await BunnyCdnFtpInfo.GetCreds(StorageSpace.Mirrors);
using var queue = new WorkQueue();
if (_archives.TryGetPath(toUpload.Hash, out var path))
{
_logger.LogInformation($"Uploading mirror file {toUpload.Hash} {path.Size.FileSizeToString()}");
bool exists = false;
using (var client = await GetClient(creds))
{
exists = await client.FileExistsAsync($"{toUpload.Hash.ToHex()}/definition.json.gz");
}
if (exists)
{
_logger.LogInformation($"Skipping {toUpload.Hash} it's already on the server");
await toUpload.Finish(_sql);
goto TOP;
}
await _discord.Send(Channel.Spam,
new DiscordMessage
{
Content = $"Uploading {toUpload.Hash} - {toUpload.Created} because {toUpload.Rationale}"
});
var definition = await Client.GenerateFileDefinition(queue, path, (s, percent) => { });
using (var client = await GetClient(creds))
{
await client.CreateDirectoryAsync($"{definition.Hash.ToHex()}");
await client.CreateDirectoryAsync($"{definition.Hash.ToHex()}/parts");
}
string MakePath(long idx)
{
return($"{definition.Hash.ToHex()}/parts/{idx}");
}
await definition.Parts.PMap(queue, async part =>
{
_logger.LogInformation($"Uploading mirror part ({part.Index}/{definition.Parts.Length})");
var buffer = new byte[part.Size];
await using (var fs = await path.OpenShared())
{
fs.Position = part.Offset;
await fs.ReadAsync(buffer);
}
using var client = await GetClient(creds);
var name = MakePath(part.Index);
await client.UploadAsync(new MemoryStream(buffer), name);
});
using (var client = await GetClient(creds))
{
_logger.LogInformation($"Finishing mirror upload");
await using var ms = new MemoryStream();
await using (var gz = new GZipStream(ms, CompressionLevel.Optimal, true))
{
definition.ToJson(gz);
}
ms.Position = 0;
var remoteName = $"{definition.Hash.ToHex()}/definition.json.gz";
await client.UploadAsync(ms, remoteName);
}
await toUpload.Finish(_sql);
}
else
{
await toUpload.Fail(_sql, "Archive not found");
}
}
catch (Exception ex)
{
_logger.LogInformation($"{toUpload.Created} {toUpload.Uploaded}");
_logger.LogError(ex, "Error uploading");
await toUpload.Fail(_sql, ex.ToString());
}
goto TOP;
}
public SqlService(AppSettings settings)
{
_settings = settings;
_mirrorCreds = BunnyCdnFtpInfo.GetCreds(StorageSpace.Mirrors);
}
