internal static string GetPrefetchedFilePath(LazyUri url, bool checkExistence)
{
if (Caching.AzureApi != null)
{
Sanity.NotImplemented();
}
var path = Caching.GetFileCachePath(url);
if (checkExistence)
{
if (!BlobStore.Exists(path))
{
return(null);
}
if (BlobStore.GetLength(path) == 0)
{
if (BlobStore.Exists(Path.ChangeExtension(path, ".err")))
{
return(null);
}
}
}
return(path);
}
internal static WebCache TryReadCacheFile(string path, bool onlyIfFailed = false, bool fromFileSystem = false)
{
#if STANDALONE
HttpUtils.EnsureInitialized();
#else
Utils.EnsureInitialized();
#endif
Stream stream;
if (fromFileSystem)
{
if (!File.Exists(path))
{
return(null);
}
try
{
stream = File.Open(path, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.Read);
}
catch (FileNotFoundException)
{
return(null);
}
}
else
{
if (!BlobStore.Exists(path))
{
return(null);
}
try
{
stream = BlobStore.OpenRead(path);
}
catch (FileNotFoundException)
{
return(null);
}
}
Sanity.AssertFastReadByte(stream);
using (stream)
{
var q = stream.ReadByte();
if (q == 0xEF)
{
stream.ReadByte();
stream.ReadByte();
using (var sr = new StreamReader(stream, Encoding.UTF8))
{
var qq = JsonConvert.DeserializeObject <WebCache>(sr.ReadToEnd());
stream.Dispose();
SaveCache(path, qq);
return(qq);
}
}
if (q != 0x1F)
{
throw new ArgumentException("Invalid cache file.");
}
stream.Seek(0, SeekOrigin.Begin);
var gz = new GZipStream2(stream, CompressionMode.Decompress);
q = gz.ReadByte();
if (q < 50 || q > 80)
{
throw new ArgumentException("Invalid cache file.");
}
using (var br = new BinaryReader(gz, Encoding.UTF8))
{
Sanity.AssertFastReadByte(br.BaseStream);
var cache = new WebCache();
cache.ContentType = br.ReadNullableString();
cache.DateRetrieved = new DateTime(br.ReadInt64(), DateTimeKind.Utc);
cache.ErrorCode = br.ReadInt32();
cache.ExceptionMessage = br.ReadNullableString();
cache.ExceptionType = br.ReadNullableString();
if (onlyIfFailed && cache.ExceptionType == null)
{
return(null);
}
var headerCount = br.ReadInt32();
cache.Headers = new Dictionary <string, string>(headerCount);
for (int i = 0; i < headerCount; i++)
{
var name = br.ReadString();
var value = br.ReadString();
cache.Headers[name] = value;
}
var cookieCount = br.ReadInt32();
cache.Cookies = new Dictionary <string, string>(cookieCount);
for (int i = 0; i < cookieCount; i++)
{
var name = br.ReadString();
var value = br.ReadString();
cache.Cookies[name] = value;
}
cache.DataType = (WebCacheDataType)br.ReadByte();
cache.RedirectUrl = br.ReadNullableString()?.AsLazyUri();
var p = br.ReadNullableString();
cache.Url = p != null ? new LazyUri(p) : null;
cache.Result = br.ReadNullableString();
if (q >= 51)
{
cache.JsExecutionResults = br.ReadNullableString();
if (q >= 52)
{
var pp = br.ReadNullableString();
cache.PageUrl = pp != null ? new LazyUri(pp) : null;
}
}
return(cache);
}
}
}
internal static string GetFileSystemName(LazyUri url, string cacheFolder, string extension, bool createLng, bool partition = true)
{
var hashed = false;
var isazure = cacheFolder != null && cacheFolder.StartsWith("azure:");
if (cacheFolder == null)
{
return(null);
}
if (!isazure && cacheFolder.Length > CacheFolderMaxLength)
{
throw new ArgumentException("The path of the file cache folder must not be longer than " + CacheFolderMaxLength + " characters.");
}
var str = url.AbsoluteUri;
var hashcode = Math.Abs((long)str.GetHashCode());
var sb = ReseekableStringBuilder.AcquirePooledStringBuilder();
if (url.Scheme != "http" && url.Scheme != "https")
{
throw new NotSupportedException("URI scheme is not supported.");
}
var https = url.Scheme == "https";
sb.Append((string)url.DnsSafeHost);
if (!url.IsDefaultPort)
{
sb.Append("∴");
sb.AppendFast((int)url.Port);
}
sb.Append(https ? "₰" : "ℓ");
var abspath = url.AbsolutePath;
sb.Append(abspath, 1, abspath.Length - 1);
sb.Append((string)url.Query);
sb.Append((string)url.Fragment);
if (sb.Length <= CacheFileNameMaxLength)
{
FixupFabulousUrl(sb);
foreach (var item in Path.GetInvalidFileNameChars())
{
if (sb.IndexOf(item) != -1)
{
sb.Replace(item.ToString(), "℅" + ((int)item).ToString("X2"));
}
}
sb.Append(extension);
}
var folder = isazure ? null : partition?Path.Combine(cacheFolder, (hashcode % 1000).ToString("000")) : cacheFolder;
if (sb.Length > CacheFileNameMaxLength)
{
#if NET35
sb.Length = 0;
#else
sb.Clear();
#endif
sb.Append(url.DnsSafeHost.TrimSize(60, 0, false));
if (!url.IsDefaultPort)
{
sb.Append("∴");
sb.AppendFast((int)url.Port);
}
sb.Append(https ? "₰" : "ℓ");
sb.Append("↔");
using (var hashAlgorithm =
#if NATIVE_HTTP
System.Security.Cryptography.SHA256.Create()
#else
System.Security.Cryptography.Reimpl.SHA256.Create()
#endif
)
{
byte[] inputBytes = System.Text.Encoding.UTF8.GetBytes(str);
byte[] hash = hashAlgorithm.ComputeHash(inputBytes);
for (int i = 0; i < hash.Length; i++)
{
sb.Append(hash[i].ToString("x2"));
}
}
// IPv6 addresses
FixupFabulousUrl(sb);
if (!isazure)
{
sb.Append(extension);
}
hashed = true;
}
if (isazure)
{
sb.Length -= 4; // remove .dat
sb.Replace("₰", "/s/");
sb.Replace("ℓ", "/h/");
sb.Replace("↑", "");
sb.Replace('\u222F', '/'); // ∯
return(ReseekableStringBuilder.GetValueAndRelease(sb));
}
var path = Path.Combine(folder, ReseekableStringBuilder.GetValueAndRelease(sb));
if (createLng)
{
Directory.CreateDirectory(folder);
if (hashed)
{
var p = Path.ChangeExtension(path, ".lng");
if (!BlobStore.Exists(p))
{
BlobStore.WriteAllText(p, (string)url.AbsoluteUri, Encoding.UTF8);
}
}
}
return(path);
}
public static void MigrateFolder(string folder)
{
if (folder == null)
{
throw new ArgumentNullException();
}
folder = Path.GetFullPath(folder);
#if NET35
foreach (var file in Directory.GetFiles(folder))
#else
foreach (var file in Directory.EnumerateFiles(folder))
#endif
{
var ext = Path.GetExtension(file);
if (ext == ".shaman-blobs")
{
continue;
}
if (ext == ".shaman-blob-index")
{
continue;
}
if (BlobStore.Exists(file))
{
continue;
}
Console.WriteLine(file);
using (var fs = File.Open(file, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.Read))
{
using (var blob = OpenWrite(file))
{
#if NET35
InternalCopyTo(fs, blob, 81920);
#else
fs.CopyTo(blob);
#endif
blob.Commit();
}
}
}
var dir = GetDirectory(folder);
dir.Flush(true);
Console.WriteLine("Migration done, deleting files...");
foreach (var file in Directory
#if NET35
.GetFiles(
#else
.EnumerateFiles(
#endif
folder))
{
var ext = Path.GetExtension(file);
if (ext == ".shaman-blobs")
{
continue;
}
if (ext == ".shaman-blob-index")
{
continue;
}
File.Delete(file);
}
Console.WriteLine("Files deleted.");
}