public bool DownloadFileWithProgress() { // Download the larger payload files. Call ProgressEvent periodically. try { const int nChunkSize = FileUtil.kDefaultBufferSize; int iTotalBytesRead = 0; using (var oFS = new FileStream(DestPath, FileMode.Create, FileAccess.Write)) { WebRequest wRemote = WebRequest.Create(SrcURL); WebResponse myWebResponse = wRemote.GetResponse(); var bBuffer = new byte[nChunkSize + 1]; using (Stream sChunks = myWebResponse.GetResponseStream()) { for (; ;) { if (ProgressEvent != null) { ProgressEvent(iTotalBytesRead, myWebResponse.ContentLength); } if (iTotalBytesRead >= myWebResponse.ContentLength) // done. { break; } int iBytesRead = sChunks.Read(bBuffer, 0, nChunkSize); if (iBytesRead == 0) { // This is a failure ! throw new IOException("Not enough data was provided to complete the file."); } oFS.Write(bBuffer, 0, iBytesRead); iTotalBytesRead += iBytesRead; } } } return(true); } catch (Exception ex) { if (FailedEvent != null) { FailedEvent(ex); } LoggerUtil.DebugException("DownloadFileWithProgress", ex); return(false); } }
private async Task UpdateIntegrity(string dstPath) { // Generate integrity if its missing. It should NOT be ! // Calculate and display an integrity value to be added. We should add this to CdnAll if (TagName == CdnTagName.a) // integrity does nothing for this type? { return; } if (this.CdnPath1 == null) { return; } string hash = await GetFileHash(dstPath, kIntegrityAlgDef); integrity = string.Concat(kIntegrityAlgDef, kIntegritySep, hash); // Print it out to log so i can update CdnAll manually!! LoggerUtil.DebugException(dstPath + " integrity='" + integrity + "'", null); }
public static async Task UpdateDirectory(string dirPath, string searchPattern = null) { // make minified versions of all the files in this directory if they don't already exist. var dir = new DirectoryInfo(dirPath); var files1 = ((searchPattern == null) ? dir.GetFiles() : dir.GetFiles(searchPattern)).Where(x => extensions.Contains(x.Extension)); foreach (FileInfo info in files1) { string name = info.Name; if (IsMinName(name)) { continue; } // does the non min file have a valid minified version? must be ~newer string dstMinPath; FileInfo infoMin = files1.FirstOrDefault(x => GetNonMinName(x.Name) == name && x.Name != name); if (infoMin != null) { // Min file exists. if ((infoMin.CreationTimeUtc - info.CreationTimeUtc).Minutes >= -3) // minified must be newer or close enough. { continue; } dstMinPath = infoMin.FullName; } else { dstMinPath = Path.Combine(Path.GetDirectoryName(info.FullName), Path.GetFileNameWithoutExtension(name) + ".min" + Path.GetExtension(name)); // kMin } LoggerUtil.DebugEntry($"Update minified file '{dstMinPath}'"); await CreateMinified(info.FullName, dstMinPath); } }
public async Task <CdnRet> SyncFile(string dstPath) { // Get a file from CDN if i don't already have it and check its integrity (if supplied) // can throw. byte[] hashCode1 = null; HashUtil hasher = null; var fi = new FileInfo(dstPath); if (integrity == null) { // integrity doesn't exist so just check that the file exists locally. if (fi.Exists && fi.Length > 0) { await UpdateIntegrity(dstPath); // suggest that we manually update integrity. return(CdnRet.Valid); // it exists. good enough since we don't have integrity attribute. } // It doesn't exist locally. pull a local copy from CDN. No big deal. } else { // test local file integrity hash e.g. "sha256-", "sha384-" int i = integrity.IndexOf(kIntegritySep); if (i <= 0) // badly formed integrity ?? Fail ? { // This is bad !! i cant really fix this. fix it manually. LoggerUtil.DebugException($"Bad CDN integrity format '{integrity}'", null); return(CdnRet.Error); } hashCode1 = Convert.FromBase64String(integrity.Substring(i + 1)); hasher = new HashUtil(HashUtil.FindHasherByName(integrity)); if (fi.Exists) { // Does current file match integrity? byte[] hashCode2 = await hasher.GetHashFileAsync(dstPath); Debug.Assert(hashCode2 != null); // string debugHash2 = Convert.ToBase64String(hashCode2); if (ByteUtil.CompareBytes(hashCode1, hashCode2) == 0) // integrity match is good. Skip this. { return(CdnRet.Valid); } // local file DOES NOT MATCH integrity!! // This really should never happen! Pull another file from the CDN and hope it matches. } } if (this.CdnPath1 == null) { // this file has no CDN. So i can't do anything! if (Minifier.IsMinName(dstPath) && await Minifier.CreateMinified(Minifier.GetNonMinName(dstPath), dstPath)) { // Local only lacked a minified version. so i made one. return(CdnRet.Updated); } LoggerUtil.DebugException($"No File ({integrity}) and No CDN path for '{name}'", null); return(CdnRet.Error); } // Pull/Get the file from CDN. LoggerUtil.DebugEntry($"Get '{this.CdnPath1}'"); var dl = new HttpDownloader(this.CdnPath1, dstPath); // CDN can get "OperationCanceledException: The operation was canceled." await dl.DownloadFileAsync(true); // Assume directory is created on demand. if (integrity == null) { // destination file should exist now. Does it? var fi2 = new FileInfo(dstPath); if (!fi2.Exists || fi2.Length <= 0) { LoggerUtil.DebugException("CDN file size 0 for " + dstPath, null); return(CdnRet.Error); } await UpdateIntegrity(dstPath); } else { // Now (re)test integrity for the file i just got! hasher.Init(); byte[] hashCode2 = await hasher.GetHashFileAsync(dstPath); if (ByteUtil.CompareBytes(hashCode1, hashCode2) != 0) // MUST match. { // This is BAD. It should never happen! string debugHash2 = Convert.ToBase64String(hashCode2); LoggerUtil.DebugException($"CDN integrity hash does not match for '{dstPath}'. should be integrity='{string.Concat(kIntegrityAlgDef, kIntegritySep, debugHash2)}'", null); return(CdnRet.Error); } } return(CdnRet.Updated); // got it. }
public static async Task <bool> CreateMinified(string srcPath, string dstMinPath) { // create a minified version of a file. CSS, JS or HTML. try { var contents = await FileUtil.ReadAllLinesAsync(srcPath); // Filter the contents to make the minified file. using (var wr = File.CreateText(dstMinPath)) { bool commentOpen = false; foreach (string line in contents) { // remove extra whitespace at start and end of lines. string line2 = line.Trim(); if (commentOpen) { // Look for close of comment. int j = line2.IndexOf(kCommentClose); if (j < 0) { continue; } line2 = line2.Substring(j + kCommentClose.Length).Trim(); commentOpen = false; } // remove blank lines. if (string.IsNullOrWhiteSpace(line2)) { continue; } // remove whole line comments. if (line2.StartsWith("//")) { continue; } do_retest: // remove /* Block Comment */ int i = line2.IndexOf("/*"); if (i >= 0) { string lineStart = line2.Substring(0, i).Trim(); int j = line2.IndexOf(kCommentClose, i); if (j < 0) { // to next line. commentOpen = true; line2 = lineStart; } else { commentOpen = false; line2 = lineStart + line2.Substring(j + kCommentClose.Length).Trim(); goto do_retest; } } // ?? remove end line // comments if (string.IsNullOrWhiteSpace(line2)) { continue; } await wr.WriteLineAsync(line2); } } return(true); } catch (Exception ex) { LoggerUtil.DebugException("CreateMinified", ex); return(false); } }