internal static string GetPrefetchedFilePath(LazyUri url, bool checkExistence)
{
if (Caching.AzureApi != null)
{
Sanity.NotImplemented();
}
var path = Caching.GetFileCachePath(url);
if (checkExistence)
{
if (!BlobStore.Exists(path))
{
return(null);
}
if (BlobStore.GetLength(path) == 0)
{
if (BlobStore.Exists(Path.ChangeExtension(path, ".err")))
{
return(null);
}
}
}
return(path);
}
public Stream OpenRead()
{
if (fileStream == null)
{
return(BlobStore.OpenRead(this.Path));
}
int len;
if (Length != null)
{
len = Length.Value;
}
else
{
throw new Exception("Length should've been available, since fileStream is not null.");
}
fileStream.Seek(DataStartOffset, SeekOrigin.Begin);
var f = new BlobStream(fileStream, len, true);
fileStream = null;
return(f);
}
private static async Task DeleteFileCacheAsync(LazyUri url, string cachedir, string extension, bool fileCache)
{
if (cachedir == null)
{
throw new InvalidOperationException("The cache path is not configured.");
}
var path = GetFileSystemName(url, cachedir, extension, false, false);
#if !STANDALONE
if (fileCache && AzureApi != null)
{
await DeleteAzureBlobAsync(url);
}
else
#endif
{
BlobStore.Delete(path);
if (path.Contains('\u2194')) // ↔
{
BlobStore.Delete(Path.ChangeExtension(path, ".lng"));
}
BlobStore.Delete(Path.ChangeExtension(path, ".err"));
}
}
public static IEnumerable <KeyValuePair <LazyUri, string> > GetCachedFiles(string cachedir, string extension)
{
if (cachedir.StartsWith("azure:"))
{
throw new NotSupportedException();
}
foreach (var dir in new[] { cachedir })
{
if (extension == ".awc")
{
foreach (var file in BlobStore.EnumerateFiles(dir, "*" + extension))
{
var url = GetUrlFromPath(file.Path, true);
yield return(new KeyValuePair <LazyUri, string>(url, file.Path));
}
}
else
{
#if NET35
foreach (var file in Directory.GetFiles(dir, "*" + extension))
#else
foreach (var file in Directory.EnumerateFiles(dir, "*" + extension))
#endif
{
var url = GetUrlFromPath(file, true);
yield return(new KeyValuePair <LazyUri, string>(url, file));
}
}
}
}
public static LazyUri GetUrlFromPath(string file, bool tryOnly)
{
var name = Path.GetFileName(file);
if (name.Contains("↔"))
{
try
{
return(BlobStore.ReadAllText(Path.ChangeExtension(file, ".lng")).AsLazyUri());
}
catch when(!tryOnly)
{
return(null);
}
}
else
{
var fileName = Path.GetFileNameWithoutExtension(file);
var sb = new StringBuilder(fileName);
sb.Replace("↑", string.Empty);
sb.Replace('\u203D', '?'); // ‽
sb.Replace('\u222F', '/'); // ∯
sb.Replace('\u223A', ':'); // ∺
sb.Replace('\u2234', ':'); // ∴
if (fileName.Contains('\u20B0')) // ₰
{
sb.Insert(0, "https://");
sb.Replace('\u20B0', '/'); // ₰
}
else if (fileName.Contains('\u2113')) // ℓ
{
sb.Insert(0, "http://");
sb.Replace('\u2113', '/'); // ℓ
}
else if (tryOnly)
{
return(null);
}
else
{
throw new FormatException();
}
var str = sb.ToString();
if (str.Contains('\u2105')) // ℅
{
str = Regex.Replace(str, @"℅[0-9a-f]{2}", x =>
{
var ch = int.Parse(x.Value.Substring(1), NumberStyles.HexNumber);
return(((char)ch).ToString());
}
);
}
return(str.AsLazyUri());
}
}
public static void EnableWebCache(string path)
{
if (WebCachePath != null && (path != WebCachePath && path != null))
{
BlobStore.FlushDirectory(WebCachePath);
BlobStore.CloseDirectory(WebCachePath);
}
if (path != null)
{
path = Path.GetFullPath(path);
Directory.CreateDirectory(path);
}
WebCachePath = path;
}
internal void Release(bool commit = true)
{
lock (BlobStore._lock)
{
BlobStore.Delete(Path.Combine(directory.path, this.currentFileName));
if (commit)
{
Commit(true);
}
directory.busyPackages.Remove(this);
if (ms.Length < (directory.packageSize ?? BlobStore.Configuration_PackageSize))
{
directory.availablePackages.Add(this);
}
else
{
WriteToFile(false);
this.ms = null;
}
}
}
internal static string GetFileSystemName(LazyUri url, string cacheFolder, string extension, bool createLng, bool partition = true)
{
var hashed = false;
var isazure = cacheFolder != null && cacheFolder.StartsWith("azure:");
if (cacheFolder == null)
{
return(null);
}
if (!isazure && cacheFolder.Length > CacheFolderMaxLength)
{
throw new ArgumentException("The path of the file cache folder must not be longer than " + CacheFolderMaxLength + " characters.");
}
var str = url.AbsoluteUri;
var hashcode = Math.Abs((long)str.GetHashCode());
var sb = ReseekableStringBuilder.AcquirePooledStringBuilder();
if (url.Scheme != "http" && url.Scheme != "https")
{
throw new NotSupportedException("URI scheme is not supported.");
}
var https = url.Scheme == "https";
sb.Append((string)url.DnsSafeHost);
if (!url.IsDefaultPort)
{
sb.Append("∴");
sb.AppendFast((int)url.Port);
}
sb.Append(https ? "₰" : "ℓ");
var abspath = url.AbsolutePath;
sb.Append(abspath, 1, abspath.Length - 1);
sb.Append((string)url.Query);
sb.Append((string)url.Fragment);
if (sb.Length <= CacheFileNameMaxLength)
{
FixupFabulousUrl(sb);
foreach (var item in Path.GetInvalidFileNameChars())
{
if (sb.IndexOf(item) != -1)
{
sb.Replace(item.ToString(), "℅" + ((int)item).ToString("X2"));
}
}
sb.Append(extension);
}
var folder = isazure ? null : partition?Path.Combine(cacheFolder, (hashcode % 1000).ToString("000")) : cacheFolder;
if (sb.Length > CacheFileNameMaxLength)
{
#if NET35
sb.Length = 0;
#else
sb.Clear();
#endif
sb.Append(url.DnsSafeHost.TrimSize(60, 0, false));
if (!url.IsDefaultPort)
{
sb.Append("∴");
sb.AppendFast((int)url.Port);
}
sb.Append(https ? "₰" : "ℓ");
sb.Append("↔");
using (var hashAlgorithm =
#if NATIVE_HTTP
System.Security.Cryptography.SHA256.Create()
#else
System.Security.Cryptography.Reimpl.SHA256.Create()
#endif
)
{
byte[] inputBytes = System.Text.Encoding.UTF8.GetBytes(str);
byte[] hash = hashAlgorithm.ComputeHash(inputBytes);
for (int i = 0; i < hash.Length; i++)
{
sb.Append(hash[i].ToString("x2"));
}
}
// IPv6 addresses
FixupFabulousUrl(sb);
if (!isazure)
{
sb.Append(extension);
}
hashed = true;
}
if (isazure)
{
sb.Length -= 4; // remove .dat
sb.Replace("₰", "/s/");
sb.Replace("ℓ", "/h/");
sb.Replace("↑", "");
sb.Replace('\u222F', '/'); // ∯
return(ReseekableStringBuilder.GetValueAndRelease(sb));
}
var path = Path.Combine(folder, ReseekableStringBuilder.GetValueAndRelease(sb));
if (createLng)
{
Directory.CreateDirectory(folder);
if (hashed)
{
var p = Path.ChangeExtension(path, ".lng");
if (!BlobStore.Exists(p))
{
BlobStore.WriteAllText(p, (string)url.AbsoluteUri, Encoding.UTF8);
}
}
}
return(path);
}
internal static void SaveCache(string cachePath, WebCache webCache)
{
if (cachePath == null || webCache == null)
{
return;
}
using (var stream = BlobStore.OpenWrite(cachePath))
{
using (var gz = new GZipStream2(stream, CompressionMode.Compress))
using (var bw = new BinaryWriter(gz, Encoding.UTF8))
{
Sanity.AssertFastWriteByte(gz.BaseStream);
bw.Write((byte)52);
bw.WriteNullableString(webCache.ContentType);
bw.Write(webCache.DateRetrieved.Ticks);
bw.Write(webCache.ErrorCode);
bw.WriteNullableString(webCache.ExceptionMessage);
bw.WriteNullableString(webCache.ExceptionType);
bw.Write(webCache.Headers != null ? webCache.Headers.Count : 0);
if (webCache.Headers != null)
{
foreach (var item in webCache.Headers)
{
bw.Write(item.Key);
bw.Write(item.Value);
}
}
bw.Write(webCache.Cookies != null ? webCache.Cookies.Count : 0);
if (webCache.Cookies != null)
{
foreach (var item in webCache.Cookies)
{
bw.Write(item.Key);
bw.Write(item.Value);
}
}
bw.Write((byte)webCache.DataType);
bw.WriteNullableString(webCache.RedirectUrl != null ? webCache.RedirectUrl.AbsoluteUri : null);
bw.WriteNullableString(webCache.Url != null ? webCache.Url.AbsoluteUri : null);
bw.WriteNullableString(webCache.Result);
bw.WriteNullableString(webCache.JsExecutionResults);
bw.WriteNullableString(webCache.PageUrl?.AbsoluteUri);
}
}
if (lastFlush == null)
{
lastFlush = Stopwatch.StartNew();
}
else if (lastFlush.ElapsedMilliseconds > Configuration_FlushIntervalMs)
{
BlobStore.FlushDirectory(Path.GetDirectoryName(cachePath));
#if NET35
lastFlush.Stop();
lastFlush.Start();
#else
lastFlush.Restart();
#endif
}
}
internal static WebCache TryReadCacheFile(string path, bool onlyIfFailed = false, bool fromFileSystem = false)
{
#if STANDALONE
HttpUtils.EnsureInitialized();
#else
Utils.EnsureInitialized();
#endif
Stream stream;
if (fromFileSystem)
{
if (!File.Exists(path))
{
return(null);
}
try
{
stream = File.Open(path, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.Read);
}
catch (FileNotFoundException)
{
return(null);
}
}
else
{
if (!BlobStore.Exists(path))
{
return(null);
}
try
{
stream = BlobStore.OpenRead(path);
}
catch (FileNotFoundException)
{
return(null);
}
}
Sanity.AssertFastReadByte(stream);
using (stream)
{
var q = stream.ReadByte();
if (q == 0xEF)
{
stream.ReadByte();
stream.ReadByte();
using (var sr = new StreamReader(stream, Encoding.UTF8))
{
var qq = JsonConvert.DeserializeObject <WebCache>(sr.ReadToEnd());
stream.Dispose();
SaveCache(path, qq);
return(qq);
}
}
if (q != 0x1F)
{
throw new ArgumentException("Invalid cache file.");
}
stream.Seek(0, SeekOrigin.Begin);
var gz = new GZipStream2(stream, CompressionMode.Decompress);
q = gz.ReadByte();
if (q < 50 || q > 80)
{
throw new ArgumentException("Invalid cache file.");
}
using (var br = new BinaryReader(gz, Encoding.UTF8))
{
Sanity.AssertFastReadByte(br.BaseStream);
var cache = new WebCache();
cache.ContentType = br.ReadNullableString();
cache.DateRetrieved = new DateTime(br.ReadInt64(), DateTimeKind.Utc);
cache.ErrorCode = br.ReadInt32();
cache.ExceptionMessage = br.ReadNullableString();
cache.ExceptionType = br.ReadNullableString();
if (onlyIfFailed && cache.ExceptionType == null)
{
return(null);
}
var headerCount = br.ReadInt32();
cache.Headers = new Dictionary <string, string>(headerCount);
for (int i = 0; i < headerCount; i++)
{
var name = br.ReadString();
var value = br.ReadString();
cache.Headers[name] = value;
}
var cookieCount = br.ReadInt32();
cache.Cookies = new Dictionary <string, string>(cookieCount);
for (int i = 0; i < cookieCount; i++)
{
var name = br.ReadString();
var value = br.ReadString();
cache.Cookies[name] = value;
}
cache.DataType = (WebCacheDataType)br.ReadByte();
cache.RedirectUrl = br.ReadNullableString()?.AsLazyUri();
var p = br.ReadNullableString();
cache.Url = p != null ? new LazyUri(p) : null;
cache.Result = br.ReadNullableString();
if (q >= 51)
{
cache.JsExecutionResults = br.ReadNullableString();
if (q >= 52)
{
var pp = br.ReadNullableString();
cache.PageUrl = pp != null ? new LazyUri(pp) : null;
}
}
return(cache);
}
}
}
public static void MigrateFolder(string folder)
{
if (folder == null)
{
throw new ArgumentNullException();
}
folder = Path.GetFullPath(folder);
#if NET35
foreach (var file in Directory.GetFiles(folder))
#else
foreach (var file in Directory.EnumerateFiles(folder))
#endif
{
var ext = Path.GetExtension(file);
if (ext == ".shaman-blobs")
{
continue;
}
if (ext == ".shaman-blob-index")
{
continue;
}
if (BlobStore.Exists(file))
{
continue;
}
Console.WriteLine(file);
using (var fs = File.Open(file, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.Read))
{
using (var blob = OpenWrite(file))
{
#if NET35
InternalCopyTo(fs, blob, 81920);
#else
fs.CopyTo(blob);
#endif
blob.Commit();
}
}
}
var dir = GetDirectory(folder);
dir.Flush(true);
Console.WriteLine("Migration done, deleting files...");
foreach (var file in Directory
#if NET35
.GetFiles(
#else
.EnumerateFiles(
#endif
folder))
{
var ext = Path.GetExtension(file);
if (ext == ".shaman-blobs")
{
continue;
}
if (ext == ".shaman-blob-index")
{
continue;
}
File.Delete(file);
}
Console.WriteLine("Files deleted.");
}