private string getPathToCachedOrDownloadedFile(IUser user, string url, string filenameCacheDescription)
{
string fullPath = cache.GetCachedFilename(new Uri(url));
if (fullPath != null)
{
if (isPlaceholderErrorFile(fullPath))
{
throw new CachedPreviousDownloadErrorKraken("File omitted because of previous error.");
}
user.RaiseMessage("Using {0} (cached)", filenameCacheDescription);
return(fullPath);
}
string tmpFile;
try
{
tmpFile = Net.Download(url, null, user);
}
catch (WebException e)
{
HttpWebResponse response = e.Response as HttpWebResponse;
if (response != null)
{
HttpWebResponse webResponse = response;
tmpFile = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
File.WriteAllText(tmpFile, DOWNLOAD_FILE_ERROR_TEXT, Encoding.ASCII);
cache.Store(new Uri(url), tmpFile, filenameCacheDescription + "." + ((int)webResponse.StatusCode).ToString(), true);
throw new CachedPreviousDownloadErrorKraken(e.Message);
}
throw;
}
return(cache.Store(new Uri(url), tmpFile, filenameCacheDescription, true));
}
public string DownloadPackage(Uri url, string identifier, DateTime?updated)
{
_requestedURLs.Add(url);
var cachedFile = _cache.GetCachedFilename(url, updated);
if (!string.IsNullOrWhiteSpace(cachedFile))
{
return(cachedFile);
}
else
{
var downloadedFile = Net.Download(url);
string extension;
switch (FileIdentifier.IdentifyFile(downloadedFile))
{
case FileType.ASCII:
extension = "txt";
break;
case FileType.GZip:
extension = "gz";
break;
case FileType.Tar:
extension = "tar";
break;
case FileType.TarGz:
extension = "tar.gz";
break;
case FileType.Zip:
extension = "zip";
string invalidReason;
if (!NetFileCache.ZipValid(downloadedFile, out invalidReason))
{
throw new Kraken($"{downloadedFile} is not a valid ZIP file: {invalidReason}");
}
break;
default:
extension = "ckan-package";
break;
}
return(_cache.Store(
url,
downloadedFile,
string.Format("netkan-{0}.{1}", identifier, extension),
move: true
));
}
}
public void StoreRetrieve()
{
Uri url = new Uri("http://example.com/");
string file = TestData.DogeCoinFlagZip();
// Our URL shouldn't be cached to begin with.
Assert.IsFalse(cache.IsCached(url));
// Store our file.
cache.Store(url, file);
// Now it should be cached.
Assert.IsTrue(cache.IsCached(url));
// Check contents match.
string cached_file = cache.GetCachedFilename(url);
FileAssert.AreEqual(file, cached_file);
}
public string DownloadPackage(Uri url, string identifier)
{
EnsureNotDisposed();
var cachedFile = _cache.GetCachedFilename(url);
if (!string.IsNullOrWhiteSpace(cachedFile))
{
return(cachedFile);
}
else
{
var downloadedFile = Net.Download(url);
string extension;
switch (FileIdentifier.IdentifyFile(downloadedFile))
{
case FileType.ASCII:
extension = "txt";
break;
case FileType.GZip:
extension = "gz";
break;
case FileType.Tar:
extension = "tar";
break;
case FileType.TarGz:
extension = "tar.gz";
break;
case FileType.Zip:
extension = "zip";
break;
default:
extension = "ckan-package";
break;
}
return(_cache.Store(
url,
downloadedFile,
string.Format("netkan-{0}.{1}", identifier, extension),
move: true
));
}
}
private string DownloadPackage(Uri url, string identifier, DateTime?updated, Uri primaryUrl = null)
{
if (primaryUrl == null)
{
primaryUrl = url;
}
if (_overwriteCache && !_requestedURLs.Contains(url))
{
// Discard cached file if command line says so,
// but only the first time in each run
_cache.Remove(url);
}
_requestedURLs.Add(url);
var cachedFile = _cache.GetCachedFilename(primaryUrl, updated);
if (!string.IsNullOrWhiteSpace(cachedFile))
{
return(cachedFile);
}
else
{
var downloadedFile = Net.Download(url);
string extension;
switch (FileIdentifier.IdentifyFile(downloadedFile))
{
case FileType.ASCII:
extension = "txt";
break;
case FileType.GZip:
extension = "gz";
break;
case FileType.Tar:
extension = "tar";
break;
case FileType.TarGz:
extension = "tar.gz";
break;
case FileType.Zip:
extension = "zip";
string invalidReason;
if (!NetFileCache.ZipValid(downloadedFile, out invalidReason))
{
log.Debug($"{downloadedFile} is not a valid ZIP file: {invalidReason}");
throw new Kraken($"{url} is not a valid ZIP file: {invalidReason}");
}
break;
default:
extension = "ckan-package";
break;
}
return(_cache.Store(
primaryUrl,
downloadedFile,
string.Format("netkan-{0}.{1}", identifier, extension),
move: true
));
}
}