private async Task <OpenLibraryDownload> GetDownload(DateTime date, OpenLibraryArchiveType archiveType, CancellationToken ct)
{
// https://archive.org/download/ol_dump_2021-03-19/ol_dump_editions_2021-03-19.txt.gz
var formattedDate = date.ToIsoDateString();
var url = $"https://archive.org/download/ol_dump_{formattedDate}/ol_dump_{archiveType.GetKey()}_{formattedDate}.txt.gz";
_logger.LogInformation($"Checking {url}");
var timer = Stopwatch.StartNew();
OpenLibraryDownload dl = null;
using (var response = await _client.GetAsync(url, HttpCompletionOption.ResponseHeadersRead, ct))
{
timer.Stop();
if (response.IsSuccessStatusCode)
{
dl = new OpenLibraryDownload
{
Datestamp = date,
Source = url,
ArchiveType = archiveType,
};
}
}
timer.Stop();
_logger.LogInformation($"Checked {url} in {timer.ElapsedMilliseconds}ms. Exists = {dl is not null}");
return(dl);
}
public async Task <OpenLibraryVersion> FindLatestArchiveEntry(OpenLibraryArchiveType archiveType, CancellationToken ct)
{
var attributeValuesMap = new Dictionary <string, AttributeValue>
{
{ ":kind", new AttributeValue {
S = archiveType.GetKey()
} },
{ ":date", new AttributeValue {
S = _clock.UtcNow().ToIsoDateString()
} },
};
var queryReq = new QueryRequest
{
TableName = _openLibVersionsTable,
KeyConditionExpression = "Kind = :kind AND PublishDate <= :date",
ScanIndexForward = false, // False = sort by descending
ExpressionAttributeValues = attributeValuesMap,
Limit = 1,
};
var queryResults = await _dynamoClient.QueryAsync(queryReq, ct);
var theResult = queryResults.Items?.SingleOrDefault();
if (theResult is null)
{
return(null);
}
var doc = Document.FromAttributeMap(theResult);
var typed = _pocoClient.FromDocument <OpenLibraryVersion>(doc);
return(typed);
}
public static string GetKey(this OpenLibraryArchiveType archiveTypeArchiveType)
{
return(archiveTypeArchiveType switch
{
OpenLibraryArchiveType.Authors => "authors",
OpenLibraryArchiveType.Editions => "editions",
_ => throw new ArgumentException($"{archiveTypeArchiveType} is not a supported archive type"),
});
public async Task <IReadOnlyCollection <OpenLibraryVersion> > FindArchiveEntries(
DateTime searchStart,
DateTime searchEnd,
OpenLibraryArchiveType archiveType,
CancellationToken ct)
{
if (searchEnd <= searchStart)
{
throw new ArgumentException($"Search start ({searchStart:O}) must come before search end ({searchEnd:O})");
}
var range = new object[] { searchStart, searchEnd };
var query = _pocoClient.QueryAsync <OpenLibraryVersion>(archiveType.GetKey(), QueryOperator.Between, range);
var results = await query.GetNextSetAsync(ct);
return(results);
}
public Task <Stream> GetArchive(DateTime date, OpenLibraryArchiveType archiveType, CancellationToken ct)
{
var key = OpenLibraryDownloadExtensions.GetObjectName(date, archiveType);
return(_s3.GetObjectStreamAsync(_openLibVersionsBucket, key, additionalProperties: null, ct));
}
public async Task <OpenLibraryVersion> GetArchiveEntry(DateTime date, OpenLibraryArchiveType archiveType, CancellationToken ct)
{
var q = await _pocoClient.LoadAsync <OpenLibraryVersion>(archiveType.GetKey(), date, ct);
return(q);
}
public static string GetObjectName(DateTime dt, OpenLibraryArchiveType archiveType)
=> $"{dt.ToIsoDateString()}-{archiveType.GetKey()}-orig.txt.gz";
public async Task <OpenLibraryDownload> GetLatestVersionForType(OpenLibraryArchiveType archiveType, CancellationToken ct)
{
var plural = await GetLatestVersionForTypes(new[] { archiveType }, ct);
return(plural.SingleOrDefault());
}
public async Task <IReadOnlyCollection <OpenLibraryDownload> > GetDownloadsForType(OpenLibraryArchiveType archiveType, CancellationToken ct)
{
var feedItems = await GetFeedTimestamps(ct);
var downloadsByDateTasks = feedItems
.Select(d => GetDownload(d, archiveType, ct))
.ToList();
await Task.WhenAll(downloadsByDateTasks);
var availableForType = downloadsByDateTasks
.Where(t => t.IsCompletedSuccessfully)
.Select(t => t.Result)
.Where(dl => dl is not null)
.ToList();
return(availableForType);
}
