public bool DownloadFileWithProgress()
{
// Download the larger payload files. Call ProgressEvent periodically.
try
{
const int nChunkSize = FileUtil.kDefaultBufferSize;
int iTotalBytesRead = 0;
using (var oFS = new FileStream(DestPath, FileMode.Create, FileAccess.Write))
{
WebRequest wRemote = WebRequest.Create(SrcURL);
WebResponse myWebResponse = wRemote.GetResponse();
var bBuffer = new byte[nChunkSize + 1];
using (Stream sChunks = myWebResponse.GetResponseStream())
{
for (; ;)
{
if (ProgressEvent != null)
{
ProgressEvent(iTotalBytesRead, myWebResponse.ContentLength);
}
if (iTotalBytesRead >= myWebResponse.ContentLength) // done.
{
break;
}
int iBytesRead = sChunks.Read(bBuffer, 0, nChunkSize);
if (iBytesRead == 0)
{
// This is a failure !
throw new IOException("Not enough data was provided to complete the file.");
}
oFS.Write(bBuffer, 0, iBytesRead);
iTotalBytesRead += iBytesRead;
}
}
}
return(true);
}
catch (Exception ex)
{
if (FailedEvent != null)
{
FailedEvent(ex);
}
LoggerUtil.DebugException("DownloadFileWithProgress", ex);
return(false);
}
}
private async Task UpdateIntegrity(string dstPath)
{
// Generate integrity if its missing. It should NOT be !
// Calculate and display an integrity value to be added. We should add this to CdnAll
if (TagName == CdnTagName.a) // integrity does nothing for this type?
{
return;
}
if (this.CdnPath1 == null)
{
return;
}
string hash = await GetFileHash(dstPath, kIntegrityAlgDef);
integrity = string.Concat(kIntegrityAlgDef, kIntegritySep, hash);
// Print it out to log so i can update CdnAll manually!!
LoggerUtil.DebugException(dstPath + " integrity='" + integrity + "'", null);
}
public static async Task UpdateDirectory(string dirPath, string searchPattern = null)
{
// make minified versions of all the files in this directory if they don't already exist.
var dir = new DirectoryInfo(dirPath);
var files1 = ((searchPattern == null) ? dir.GetFiles() : dir.GetFiles(searchPattern)).Where(x => extensions.Contains(x.Extension));
foreach (FileInfo info in files1)
{
string name = info.Name;
if (IsMinName(name))
{
continue;
}
// does the non min file have a valid minified version? must be ~newer
string dstMinPath;
FileInfo infoMin = files1.FirstOrDefault(x => GetNonMinName(x.Name) == name && x.Name != name);
if (infoMin != null)
{
// Min file exists.
if ((infoMin.CreationTimeUtc - info.CreationTimeUtc).Minutes >= -3) // minified must be newer or close enough.
{
continue;
}
dstMinPath = infoMin.FullName;
}
else
{
dstMinPath = Path.Combine(Path.GetDirectoryName(info.FullName), Path.GetFileNameWithoutExtension(name) + ".min" + Path.GetExtension(name)); // kMin
}
LoggerUtil.DebugEntry($"Update minified file '{dstMinPath}'");
await CreateMinified(info.FullName, dstMinPath);
}
}
public async Task <CdnRet> SyncFile(string dstPath)
{
// Get a file from CDN if i don't already have it and check its integrity (if supplied)
// can throw.
byte[] hashCode1 = null;
HashUtil hasher = null;
var fi = new FileInfo(dstPath);
if (integrity == null)
{
// integrity doesn't exist so just check that the file exists locally.
if (fi.Exists && fi.Length > 0)
{
await UpdateIntegrity(dstPath); // suggest that we manually update integrity.
return(CdnRet.Valid); // it exists. good enough since we don't have integrity attribute.
}
// It doesn't exist locally. pull a local copy from CDN. No big deal.
}
else
{
// test local file integrity hash e.g. "sha256-", "sha384-"
int i = integrity.IndexOf(kIntegritySep);
if (i <= 0) // badly formed integrity ?? Fail ?
{
// This is bad !! i cant really fix this. fix it manually.
LoggerUtil.DebugException($"Bad CDN integrity format '{integrity}'", null);
return(CdnRet.Error);
}
hashCode1 = Convert.FromBase64String(integrity.Substring(i + 1));
hasher = new HashUtil(HashUtil.FindHasherByName(integrity));
if (fi.Exists)
{
// Does current file match integrity?
byte[] hashCode2 = await hasher.GetHashFileAsync(dstPath);
Debug.Assert(hashCode2 != null);
// string debugHash2 = Convert.ToBase64String(hashCode2);
if (ByteUtil.CompareBytes(hashCode1, hashCode2) == 0) // integrity match is good. Skip this.
{
return(CdnRet.Valid);
}
// local file DOES NOT MATCH integrity!!
// This really should never happen! Pull another file from the CDN and hope it matches.
}
}
if (this.CdnPath1 == null)
{
// this file has no CDN. So i can't do anything!
if (Minifier.IsMinName(dstPath) && await Minifier.CreateMinified(Minifier.GetNonMinName(dstPath), dstPath))
{
// Local only lacked a minified version. so i made one.
return(CdnRet.Updated);
}
LoggerUtil.DebugException($"No File ({integrity}) and No CDN path for '{name}'", null);
return(CdnRet.Error);
}
// Pull/Get the file from CDN.
LoggerUtil.DebugEntry($"Get '{this.CdnPath1}'");
var dl = new HttpDownloader(this.CdnPath1, dstPath);
// CDN can get "OperationCanceledException: The operation was canceled."
await dl.DownloadFileAsync(true); // Assume directory is created on demand.
if (integrity == null)
{
// destination file should exist now. Does it?
var fi2 = new FileInfo(dstPath);
if (!fi2.Exists || fi2.Length <= 0)
{
LoggerUtil.DebugException("CDN file size 0 for " + dstPath, null);
return(CdnRet.Error);
}
await UpdateIntegrity(dstPath);
}
else
{
// Now (re)test integrity for the file i just got!
hasher.Init();
byte[] hashCode2 = await hasher.GetHashFileAsync(dstPath);
if (ByteUtil.CompareBytes(hashCode1, hashCode2) != 0) // MUST match.
{
// This is BAD. It should never happen!
string debugHash2 = Convert.ToBase64String(hashCode2);
LoggerUtil.DebugException($"CDN integrity hash does not match for '{dstPath}'. should be integrity='{string.Concat(kIntegrityAlgDef, kIntegritySep, debugHash2)}'", null);
return(CdnRet.Error);
}
}
return(CdnRet.Updated); // got it.
}
public static async Task <bool> CreateMinified(string srcPath, string dstMinPath)
{
// create a minified version of a file. CSS, JS or HTML.
try
{
var contents = await FileUtil.ReadAllLinesAsync(srcPath);
// Filter the contents to make the minified file.
using (var wr = File.CreateText(dstMinPath))
{
bool commentOpen = false;
foreach (string line in contents)
{
// remove extra whitespace at start and end of lines.
string line2 = line.Trim();
if (commentOpen)
{
// Look for close of comment.
int j = line2.IndexOf(kCommentClose);
if (j < 0)
{
continue;
}
line2 = line2.Substring(j + kCommentClose.Length).Trim();
commentOpen = false;
}
// remove blank lines.
if (string.IsNullOrWhiteSpace(line2))
{
continue;
}
// remove whole line comments.
if (line2.StartsWith("//"))
{
continue;
}
do_retest:
// remove /* Block Comment */
int i = line2.IndexOf("/*");
if (i >= 0)
{
string lineStart = line2.Substring(0, i).Trim();
int j = line2.IndexOf(kCommentClose, i);
if (j < 0)
{
// to next line.
commentOpen = true;
line2 = lineStart;
}
else
{
commentOpen = false;
line2 = lineStart + line2.Substring(j + kCommentClose.Length).Trim();
goto do_retest;
}
}
// ?? remove end line // comments
if (string.IsNullOrWhiteSpace(line2))
{
continue;
}
await wr.WriteLineAsync(line2);
}
}
return(true);
}
catch (Exception ex)
{
LoggerUtil.DebugException("CreateMinified", ex);
return(false);
}
}
