public async Task <bool> DownloadAsync(Uri uri, string outputFile)
{
if (uri == null)
{
throw new ArgumentNullException(nameof(uri));
}
if (!uri.IsAbsoluteUri)
{
throw new ArgumentException("Relative path is not allowed.", nameof(uri));
}
using (new LoggerPhaseScope(PhaseName))
using (new LoggerFileScope(outputFile))
{
Logger.LogInfo($"Creating xref archive file...");
try
{
using (var xa = XRefArchive.Open(outputFile, XRefArchiveMode.Create))
{
await DownloadCoreAsync(uri, xa, true);
}
}
catch (Exception ex)
{
Logger.LogError($"Unable to create archive: {ex.Message}");
return(false);
}
Logger.LogInfo($"Xref archive file created.");
return(true);
}
}
private async Task <string> DownloadCoreAsync(Uri uri, XRefArchive xa, bool isMajor)
{
XRefMap map;
map = await DownloadBySchemeAsync(uri);
if (map.Redirections?.Count > 0)
{
await RewriteRedirections(uri, xa, map);
}
if (map.References?.Count > 0 && map.HrefUpdated != true)
{
if (string.IsNullOrEmpty(map.BaseUrl))
{
UpdateHref(map, uri);
}
}
lock (_syncRoot)
{
if (isMajor)
{
return(xa.CreateMajor(map));
}
else
{
return(xa.CreateMinor(map, GetNames(uri, map)));
}
}
}
private async Task<string> DownloadCoreAsync(Uri uri, XRefArchive xa, bool isMajor)
{
IXRefContainer container;
container = await _downloader.DownloadAsync(uri);
var map = container as XRefMap;
if (map == null)
{
// not support download an xref archive, or reference to an xref archive
return null;
}
if (map.Redirections?.Count > 0)
{
await RewriteRedirections(uri, xa, map);
}
if (map.References?.Count > 0 && map.HrefUpdated != true)
{
if (string.IsNullOrEmpty(map.BaseUrl))
{
XRefMapDownloader.UpdateHref(map, uri);
}
}
lock (_syncRoot)
{
if (isMajor)
{
return xa.CreateMajor(map);
}
else
{
return xa.CreateMinor(map, GetNames(uri, map));
}
}
}
private async Task <string> DownloadCoreAsync(Uri uri, XRefArchive xa, bool isMajor)
{
IXRefContainer container;
container = await _downloader.DownloadAsync(uri);
if (!(container is XRefMap map))
{
// not support download an xref archive, or reference to an xref archive
return(null);
}
if (map.Redirections?.Count > 0)
{
await RewriteRedirections(uri, xa, map);
}
if (map.References?.Count > 0 && map.HrefUpdated != true)
{
if (string.IsNullOrEmpty(map.BaseUrl))
{
XRefMapDownloader.UpdateHref(map, uri);
}
}
lock (_syncRoot)
{
if (isMajor)
{
return(xa.CreateMajor(map));
}
else
{
return(xa.CreateMinor(map, GetNames(uri, map)));
}
}
}
public XRefArchiveReader(XRefArchive archive)
: base(XRefArchive.MajorFileName, new HashSet <string>(archive.Entries))
{
if (archive == null)
{
throw new ArgumentNullException(nameof(archive));
}
_archive = archive;
_lru = LruList <Tuple <string, XRefMap> > .Create(0x10, comparer : new TupleComparer());
}
private static IXRefContainer ReadLocalFile(string filePath)
{
if (".zip".Equals(Path.GetExtension(filePath), StringComparison.OrdinalIgnoreCase))
{
return(XRefArchive.Open(filePath, XRefArchiveMode.Read));
}
using var sr = File.OpenText(filePath);
return(YamlUtility.Deserialize <XRefMap>(sr));
}
private async Task <List <XRefMapRedirection> > RewriteRedirections(Uri uri, XRefArchive xa, XRefMap map) =>
(from list in
await Task.WhenAll(
from r in map.Redirections
where !string.IsNullOrEmpty(r.Href) || !string.IsNullOrEmpty(r.UidPrefix)
group r by r.Href into g
let href = GetHrefUri(uri, g.Key)
where href != null
select RewriteRedirectionsCore(g.ToList(), href, xa))
from r in list
orderby r.UidPrefix.Length descending, r.UidPrefix
select r).ToList();
protected static IXRefContainer DownloadFromLocal(Uri uri)
{
var filePath = uri.LocalPath;
if (".zip".Equals(Path.GetExtension(filePath), StringComparison.OrdinalIgnoreCase))
{
return(XRefArchive.Open(filePath, XRefArchiveMode.Read));
}
using (var sr = File.OpenText(filePath))
{
return(YamlUtility.Deserialize <XRefMap>(sr));
}
}
private async Task <List <XRefMapRedirection> > RewriteRedirectionsCore(List <XRefMapRedirection> redirections, Uri uri, XRefArchive xa)
{
var fileRef = await DownloadCoreAsync(uri, xa, false);
return((from r in redirections
select new XRefMapRedirection {
UidPrefix = r.UidPrefix, Href = fileRef
}).ToList());
}
private async Task<List<XRefMapRedirection>> RewriteRedirections(Uri uri, XRefArchive xa, XRefMap map) =>
(from list in
await Task.WhenAll(
from r in map.Redirections
where !string.IsNullOrEmpty(r.Href)
group r by r.Href into g
let href = GetHrefUri(uri, g.Key)
where href != null
select RewriteRedirectionsCore(g.ToList(), href, xa))
from r in list
orderby (r.UidPrefix ?? string.Empty).Length descending, (r.UidPrefix ?? string.Empty)
select r).ToList();
private async Task<List<XRefMapRedirection>> RewriteRedirectionsCore(List<XRefMapRedirection> redirections, Uri uri, XRefArchive xa)
{
var fileRef = await DownloadCoreAsync(uri, xa, false);
if (fileRef == null)
{
return new List<XRefMapRedirection>();
}
return (from r in redirections
select new XRefMapRedirection { UidPrefix = r.UidPrefix, Href = fileRef }).ToList();
}
private async Task PackOneReferenceAsync(string file, XRefArchive writer)
{
var currentPackage = Path.GetFileName(file);
lock (_currentPackages)
{
_currentPackages[Array.IndexOf(_currentPackages, null)] = currentPackage;
}
var entries = new List<XRefMapRedirection>();
await GetItems(file).ForEachAsync(pair =>
{
lock (writer)
{
entries.Add(
new XRefMapRedirection
{
UidPrefix = pair.EntryName,
Href = writer.CreateMinor(
new XRefMap
{
Sorted = true,
HrefUpdated = true,
References = pair.ViewModel,
},
new[] { pair.EntryName })
});
_entryCount++;
_apiCount += pair.ViewModel.Count;
}
});
lock (writer)
{
var map = writer.GetMajor();
map.Redirections = (from r in map.Redirections.Concat(entries)
orderby r.UidPrefix.Length descending
select r).ToList();
writer.UpdateMajor(map);
}
lock (_currentPackages)
{
_currentPackages[Array.IndexOf(_currentPackages, currentPackage)] = null;
}
}
