internal static string GetPrefetchedFilePath(LazyUri url, bool checkExistence) { if (Caching.AzureApi != null) { Sanity.NotImplemented(); } var path = Caching.GetFileCachePath(url); if (checkExistence) { if (!BlobStore.Exists(path)) { return(null); } if (BlobStore.GetLength(path) == 0) { if (BlobStore.Exists(Path.ChangeExtension(path, ".err"))) { return(null); } } } return(path); }
internal static WebCache TryReadCacheFile(string path, bool onlyIfFailed = false, bool fromFileSystem = false) { #if STANDALONE HttpUtils.EnsureInitialized(); #else Utils.EnsureInitialized(); #endif Stream stream; if (fromFileSystem) { if (!File.Exists(path)) { return(null); } try { stream = File.Open(path, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.Read); } catch (FileNotFoundException) { return(null); } } else { if (!BlobStore.Exists(path)) { return(null); } try { stream = BlobStore.OpenRead(path); } catch (FileNotFoundException) { return(null); } } Sanity.AssertFastReadByte(stream); using (stream) { var q = stream.ReadByte(); if (q == 0xEF) { stream.ReadByte(); stream.ReadByte(); using (var sr = new StreamReader(stream, Encoding.UTF8)) { var qq = JsonConvert.DeserializeObject <WebCache>(sr.ReadToEnd()); stream.Dispose(); SaveCache(path, qq); return(qq); } } if (q != 0x1F) { throw new ArgumentException("Invalid cache file."); } stream.Seek(0, SeekOrigin.Begin); var gz = new GZipStream2(stream, CompressionMode.Decompress); q = gz.ReadByte(); if (q < 50 || q > 80) { throw new ArgumentException("Invalid cache file."); } using (var br = new BinaryReader(gz, Encoding.UTF8)) { Sanity.AssertFastReadByte(br.BaseStream); var cache = new WebCache(); cache.ContentType = br.ReadNullableString(); cache.DateRetrieved = new DateTime(br.ReadInt64(), DateTimeKind.Utc); cache.ErrorCode = br.ReadInt32(); cache.ExceptionMessage = br.ReadNullableString(); cache.ExceptionType = br.ReadNullableString(); if (onlyIfFailed && cache.ExceptionType == null) { return(null); } var headerCount = br.ReadInt32(); cache.Headers = new Dictionary <string, string>(headerCount); for (int i = 0; i < headerCount; i++) { var name = br.ReadString(); var value = br.ReadString(); cache.Headers[name] = value; } var cookieCount = br.ReadInt32(); cache.Cookies = new Dictionary <string, string>(cookieCount); for (int i = 0; i < cookieCount; i++) { var name = br.ReadString(); var value = br.ReadString(); cache.Cookies[name] = value; } cache.DataType = (WebCacheDataType)br.ReadByte(); cache.RedirectUrl = br.ReadNullableString()?.AsLazyUri(); var p = br.ReadNullableString(); cache.Url = p != null ? new LazyUri(p) : null; cache.Result = br.ReadNullableString(); if (q >= 51) { cache.JsExecutionResults = br.ReadNullableString(); if (q >= 52) { var pp = br.ReadNullableString(); cache.PageUrl = pp != null ? new LazyUri(pp) : null; } } return(cache); } } }
internal static string GetFileSystemName(LazyUri url, string cacheFolder, string extension, bool createLng, bool partition = true) { var hashed = false; var isazure = cacheFolder != null && cacheFolder.StartsWith("azure:"); if (cacheFolder == null) { return(null); } if (!isazure && cacheFolder.Length > CacheFolderMaxLength) { throw new ArgumentException("The path of the file cache folder must not be longer than " + CacheFolderMaxLength + " characters."); } var str = url.AbsoluteUri; var hashcode = Math.Abs((long)str.GetHashCode()); var sb = ReseekableStringBuilder.AcquirePooledStringBuilder(); if (url.Scheme != "http" && url.Scheme != "https") { throw new NotSupportedException("URI scheme is not supported."); } var https = url.Scheme == "https"; sb.Append((string)url.DnsSafeHost); if (!url.IsDefaultPort) { sb.Append("∴"); sb.AppendFast((int)url.Port); } sb.Append(https ? "₰" : "ℓ"); var abspath = url.AbsolutePath; sb.Append(abspath, 1, abspath.Length - 1); sb.Append((string)url.Query); sb.Append((string)url.Fragment); if (sb.Length <= CacheFileNameMaxLength) { FixupFabulousUrl(sb); foreach (var item in Path.GetInvalidFileNameChars()) { if (sb.IndexOf(item) != -1) { sb.Replace(item.ToString(), "℅" + ((int)item).ToString("X2")); } } sb.Append(extension); } var folder = isazure ? null : partition?Path.Combine(cacheFolder, (hashcode % 1000).ToString("000")) : cacheFolder; if (sb.Length > CacheFileNameMaxLength) { #if NET35 sb.Length = 0; #else sb.Clear(); #endif sb.Append(url.DnsSafeHost.TrimSize(60, 0, false)); if (!url.IsDefaultPort) { sb.Append("∴"); sb.AppendFast((int)url.Port); } sb.Append(https ? "₰" : "ℓ"); sb.Append("↔"); using (var hashAlgorithm = #if NATIVE_HTTP System.Security.Cryptography.SHA256.Create() #else System.Security.Cryptography.Reimpl.SHA256.Create() #endif ) { byte[] inputBytes = System.Text.Encoding.UTF8.GetBytes(str); byte[] hash = hashAlgorithm.ComputeHash(inputBytes); for (int i = 0; i < hash.Length; i++) { sb.Append(hash[i].ToString("x2")); } } // IPv6 addresses FixupFabulousUrl(sb); if (!isazure) { sb.Append(extension); } hashed = true; } if (isazure) { sb.Length -= 4; // remove .dat sb.Replace("₰", "/s/"); sb.Replace("ℓ", "/h/"); sb.Replace("↑", ""); sb.Replace('\u222F', '/'); // ∯ return(ReseekableStringBuilder.GetValueAndRelease(sb)); } var path = Path.Combine(folder, ReseekableStringBuilder.GetValueAndRelease(sb)); if (createLng) { Directory.CreateDirectory(folder); if (hashed) { var p = Path.ChangeExtension(path, ".lng"); if (!BlobStore.Exists(p)) { BlobStore.WriteAllText(p, (string)url.AbsoluteUri, Encoding.UTF8); } } } return(path); }
public static void MigrateFolder(string folder) { if (folder == null) { throw new ArgumentNullException(); } folder = Path.GetFullPath(folder); #if NET35 foreach (var file in Directory.GetFiles(folder)) #else foreach (var file in Directory.EnumerateFiles(folder)) #endif { var ext = Path.GetExtension(file); if (ext == ".shaman-blobs") { continue; } if (ext == ".shaman-blob-index") { continue; } if (BlobStore.Exists(file)) { continue; } Console.WriteLine(file); using (var fs = File.Open(file, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.Read)) { using (var blob = OpenWrite(file)) { #if NET35 InternalCopyTo(fs, blob, 81920); #else fs.CopyTo(blob); #endif blob.Commit(); } } } var dir = GetDirectory(folder); dir.Flush(true); Console.WriteLine("Migration done, deleting files..."); foreach (var file in Directory #if NET35 .GetFiles( #else .EnumerateFiles( #endif folder)) { var ext = Path.GetExtension(file); if (ext == ".shaman-blobs") { continue; } if (ext == ".shaman-blob-index") { continue; } File.Delete(file); } Console.WriteLine("Files deleted."); }