public override async Task <int> ExecuteAsync(CommandContext context, T settings) => await _console.Progress() .Columns(Columns) .StartAsync(async ctx => await Run(ctx, settings));
public override async Task <int> ExecuteAsync(CommandContext context, BrentOptions settings)
{
var archives = new List <BrentArchive>()
{
new("https://downloads.brentozar.com/StackOverflow-SQL-Server-2010.torrent",
"Small: 10GB database as of 2010", "small"),
new("https://downloads.brentozar.com/StackOverflow2013.torrent", "Medium: 50GB database as of 2013",
"medium"),
new("https://downloads.brentozar.com/StackOverflowCore.torrent", "Large 150GB database as of 2019",
"large"),
new("https://downloads.brentozar.com/StackOverflow-SQL-Server-202006.torrent",
"Extra-Large: current 381GB database as of 2020/06", "extra-large"),
};
var archiveName = settings.Archive;
if (string.IsNullOrWhiteSpace(archiveName))
{
var choice = _console.Prompt(
new SelectionPrompt <BrentArchive>()
.PageSize(10)
.Title("Pick an archive to download")
.AddChoices(archives));
archiveName = choice.ShortName;
}
var archive = archives.FirstOrDefault(a =>
a.ShortName.Equals(archiveName, StringComparison.InvariantCultureIgnoreCase));
if (archive == null)
{
throw new SoddiException($"Could not find an archive matching \"{archiveName}\"");
}
var outputPath = settings.Output;
if (string.IsNullOrWhiteSpace(outputPath))
{
outputPath = _fileSystem.Directory.GetCurrentDirectory();
}
if (!_fileSystem.Directory.Exists(outputPath))
{
throw new SoddiException($"Output path {outputPath} not found");
}
var downloadedFiles = await _torrentDownloader.Download(archive.Url, settings.EnablePortForwarding,
outputPath,
CancellationToken.None);
var sevenZipFiles = downloadedFiles.Where(i =>
_fileSystem.Path.GetExtension(i).Equals(".7z", StringComparison.InvariantCultureIgnoreCase));
var stopWatch = Stopwatch.StartNew();
var progressBar = _console.Progress()
.AutoClear(false)
.Columns(new ProgressColumn[]
{
new SpinnerColumn {
CompletedText = Emoji.Known.CheckMark
}, new TaskDescriptionColumn(),
new ProgressBarColumn(), new PercentageColumn(), new TransferSpeedColumn(),
new RemainingTimeColumn()
});
progressBar.Start(ctx =>
{
foreach (var sevenZipFile in sevenZipFiles)
{
using var stream = _fileSystem.File.OpenRead(sevenZipFile);
using var sevenZipArchive = SevenZipArchive.Open(stream);
var tasks = sevenZipArchive.Entries.ToImmutableDictionary(
e => e.Key,
e => ctx.AddTask(e.Key, new ProgressTaskSettings {
MaxValue = e.Size, AutoStart = false
}));
foreach (var entry in sevenZipArchive.Entries)
{
var currentTask = tasks[entry.Key];
currentTask.StartTask();
var totalRead = 0L;
void Handler(object?sender, CompressedBytesReadEventArgs args)
{
var diff = args.CurrentFilePartCompressedBytesRead - totalRead;
currentTask.Increment(diff);
totalRead = args.CurrentFilePartCompressedBytesRead;
}
sevenZipArchive.CompressedBytesRead += Handler;
entry.WriteToDirectory(outputPath, new ExtractionOptions {
Overwrite = true
});
sevenZipArchive.CompressedBytesRead -= Handler;
}
}
});
stopWatch.Stop();
_console.MarkupLine($"Extraction complete in [blue]{stopWatch.Elapsed.Humanize()}[/].");
return(0);
}
public async Task <ImmutableList <string> > Download(string url,
List <string>?potentialArchives,
bool enablePortForwarding,
string outputPath,
CancellationToken cancellationToken)
{
var stopWatch = Stopwatch.StartNew();
var paddingFolder = _fileSystem.Path.Combine(outputPath, ".____padding_file");
var doesPaddingFolderExistToStart = _fileSystem.Directory.Exists(paddingFolder);
var progressBar = _console.Progress()
.AutoClear(false)
.Columns(new ProgressColumn[]
{
new SpinnerColumn {
CompletedText = Emoji.Known.CheckMark
}, new DownloadedColumnExtended(),
new TaskDescriptionColumn(), new TorrentProgressBarColumn(), new PercentageColumn(),
new TransferSpeedColumn(), new RemainingTimeColumn()
});
ImmutableList <TorrentFile>?downloadedFiles = null;
await progressBar.StartAsync(async ctx =>
{
_console.WriteLine("Loading torrent...");
var httpClient = new HttpClient();
var torrentContents = await httpClient.GetByteArrayAsync(url, cancellationToken);
var settings = new EngineSettings
{
AllowedEncryption = EncryptionTypes.All, SavePath = outputPath, MaximumHalfOpenConnections = 16
};
_console.WriteLine("Initializing BitTorrent engine...");
var engine = new ClientEngine(settings);
if (enablePortForwarding)
{
_console.WriteLine("Attempting to forward ports");
await engine.EnablePortForwardingAsync(cancellationToken);
}
var torrent = await Torrent.LoadAsync(torrentContents);
if (potentialArchives != null)
{
foreach (var torrentFile in torrent.Files)
{
if (!potentialArchives.Contains(torrentFile.Path))
{
torrentFile.Priority = Priority.DoNotDownload;
}
}
}
var manager = new TorrentManager(
torrent,
outputPath,
new TorrentSettings {
MaximumConnections = 250
}, string.Empty);
await engine.Register(manager);
await engine.StartAll();
downloadedFiles = manager.Torrent.Files
.Where(i => i.Priority != Priority.DoNotDownload).ToImmutableList();
var fileTasks = downloadedFiles
.ToDictionary(
i => i.Path,
file => ctx.AddTask(file.Path,
new ProgressTaskSettings {
MaxValue = file.Length, AutoStart = false
})
);
while (manager.State != TorrentState.Stopped && manager.State != TorrentState.Seeding)
{
foreach (var torrentFile in downloadedFiles)
{
var progressTask = fileTasks[torrentFile.Path];
if (torrentFile.BytesDownloaded > 0 && progressTask.IsStarted == false)
{
progressTask.StartTask();
}
progressTask.Increment(torrentFile.BytesDownloaded - progressTask.Value);
progressTask.State.Update <BitSmuggler>("torrentBits",
_ => new BitSmuggler(torrentFile.BitField));
}
await Task.Delay(100, cancellationToken);
}
await manager.StopAsync();
await engine.StopAllAsync();
try
{
// the stackoverflow torrent files, and I think all of archive.org
// seem to have these padding files that sneak into the download even
// if they aren't included in the file list. not quite sure how to prevent that
// so I'm gonna delete them after the fact I guess
if (!doesPaddingFolderExistToStart && _fileSystem.Directory.Exists(paddingFolder))
{
_fileSystem.Directory.Delete(paddingFolder, true);
}
}
catch
{
/* swallow */
}
foreach (var progressTask in fileTasks)
{
progressTask.Value.StopTask();
}
});
stopWatch.Stop();
_console.MarkupLine($"Download complete in [blue]{stopWatch.Elapsed.Humanize()}[/].");
return(downloadedFiles?.Select(i => _fileSystem.Path.Combine(outputPath, i.Path)).ToImmutableList() ??
ImmutableList <string> .Empty);
}
public override async Task <int> ExecuteAsync(CommandContext context, ImportOptions request)
{
var cancellationToken = CancellationToken.None;
var requestPath = await CheckAndFixupPath(request.Path, cancellationToken);
var dbName = _databaseHelper.GetDbNameFromPathOption(request.DatabaseName, requestPath);
var tasks = new Queue <(string name, ITask task)>();
var(masterConnectionString, databaseConnectionString) =
_databaseHelper.GetMasterAndDbConnectionStrings(request.ConnectionString, dbName);
ImmutableDictionary <string, long>?insertReport = null;
var processor = _processorFactory.VerifyAndCreateProcessor(requestPath);
if (request.DropAndRecreate)
{
tasks.Enqueue(("Create new database", new CreateDatabase(masterConnectionString, dbName)));
}
else
{
tasks.Enqueue(("Verify database exists", new VerifyDatabaseExists(masterConnectionString, dbName)));
}
tasks.Enqueue(("Create schema", new CreateSchema(databaseConnectionString, !request.SkipTags)));
if (!request.SkipPrimaryKeys)
{
tasks.Enqueue(("Add constraints", new AddConstraints(databaseConnectionString)));
}
tasks.Enqueue(("Insert type values", new InsertTypeValues(databaseConnectionString)));
tasks.Enqueue(("Insert data from archive",
new InsertData(
databaseConnectionString,
dbName,
processor,
!request.SkipTags,
d => insertReport = d,
request.BlockSize)));
var progressBar = _console.Progress()
.AutoClear(false)
.Columns(new ProgressColumn[]
{
new SpinnerColumn {
CompletedText = Emoji.Known.CheckMark
}, new FixedTaskDescriptionColumn(40),
new ProgressBarColumn(), new PercentageColumn(), new RemainingTimeColumn(),
});
var stopWatch = Stopwatch.StartNew();
progressBar.Start(ctx =>
{
foreach (var(description, task) in tasks)
{
var progressTasks = new ConcurrentDictionary <string, ProgressTask>();
var progress = new Progress <(string taskId, string message, double weight, double maxValue)>(i =>
{
var(taskId, message, weight, maxValue) = i;
var progressBarTask = progressTasks.GetOrAdd(taskId, _ => ctx.AddTask(description));
progressBarTask.MaxValue(maxValue);
progressBarTask.Increment(weight);
progressBarTask.Description(message);
});
task.Go(progress);
foreach (var progressTask in progressTasks.Values)
{
progressTask.Increment(progressTask.MaxValue - progressTask.Value);
}
}
});
stopWatch.Stop();
if (insertReport != null)
{
var counter = 1;
var chart = new BreakdownChart()
.Compact()
.Width(60)
.UseValueFormatter(d => d.ToMetric())
.FullSize()
.AddItems(insertReport,
pair => new BreakdownChartItem(pair.Key, pair.Value, counter++)
);
_console.MarkupLine("[blue]Rows inserted[/]");
_console.Render(chart);
}
_console.WriteLine();
_console.MarkupLine($"Import complete in [blue]{stopWatch.Elapsed.Humanize()}[/].");
return(0);
}
public override async Task <int> ExecuteAsync(CommandContext context, DownloadOptions request)
{
var cancellationToken = CancellationToken.None;
var outputPath = request.Output;
if (string.IsNullOrWhiteSpace(outputPath))
{
outputPath = _fileSystem.Directory.GetCurrentDirectory();
}
if (!_fileSystem.Directory.Exists(outputPath))
{
throw new SoddiException($"Output path {outputPath} not found");
}
var archiveUrl =
await _availableArchiveParser.FindOrPickArchive(request.Archive, request.Pick, cancellationToken);
var stopWatch = Stopwatch.StartNew();
await _console.Progress()
.AutoClear(false)
.Columns(new ProgressColumn[]
{
new SpinnerColumn {
CompletedText = Emoji.Known.CheckMark
}, new DownloadedColumn(),
new TaskDescriptionColumn(), new ProgressBarColumn(), new PercentageColumn(),
new TransferSpeedColumn(), new RemainingTimeColumn(),
}).StartAsync(async ctx =>
{
var tasks = new List <(ProgressTask Task, Archive.UriWithSize UriWithSize)>();
foreach (var archive in archiveUrl)
{
tasks.AddRange(archive.Uris
.Select(uriWithSize => (ctx.AddTask(uriWithSize.Description(false)), uriWithSize))
.ToList());
}
while (!ctx.IsFinished)
{
foreach (var(task, uriWithSize) in tasks)
{
var progress = new Progress <(int downloadedInBytes, int totalSizeInBytes)>(i =>
{
var progressTask = task;
var(downloadedInBytes, totalSizeInBytes) = i;
progressTask.Increment(downloadedInBytes);
progressTask.MaxValue(totalSizeInBytes);
}
);
var downloader = new ArchiveDownloader(outputPath, progress);
await downloader.Go(uriWithSize.Uri, cancellationToken);
}
}
});
stopWatch.Stop();
_console.MarkupLine($"Download complete in [blue]{stopWatch.Elapsed.Humanize()}[/].");
return(await Task.FromResult(0));
}