// Only checks the PDF and version string from the PDF header
//
// Bad PDFs that I received from book scanning service were MBs of 0x00s - this quickly detects these
public static string Check(FileInfo fi, StreamWriter listFile)
{
string path = fi.FullName;
const int headerBufferSize = 32;
string PDFHeaderStart = "%PDF-";
const int PDFVersionLength = 3; // e.g. "1.5"
byte[] headerBuffer = new byte[headerBufferSize];
try
{
FileStream file = new FileStream(path, FileMode.Open, FileAccess.Read); // FileAccess.Read opens read-only files
file.Read(headerBuffer, 0, headerBufferSize);
file.Close();
}
catch (System.Exception ex)
{
return("** Exception opening/reading file: " + ex.Message);;
}
string header = Encoding.ASCII.GetString(headerBuffer);
if (PDFHeaderStart == header.Substring(0, PDFHeaderStart.Length))
{
return(header.Substring(PDFHeaderStart.Length, PDFVersionLength));
}
else
{
FileAnalysis.CountByteValues(fi, listFile);
return("** Invalid PDF Header");
}
}
public void CorrectFileAnalysisData_GivenInitialData()
{
var expectedPath = "/test/file/path.test.extension";
var expectedExtension = "test.extension";
var expectedLines = 10;
var result = new FileAnalysis(expectedPath, expectedLines);
Assert.AreEqual(expectedPath, result.Path);
Assert.AreEqual(expectedLines, result.Lines);
Assert.AreEqual(expectedExtension, result.Extension);
}
public async static Task Run([BlobTrigger("cost/report/{name}", Connection = "StorageConnection")] Stream myBlob, string name,
[SendGrid(ApiKey = "SendGridKeyAppSettingName")] IAsyncCollector <SendGridMessage> messageCollector, ILogger log, ExecutionContext context)
{
try
{
var config = new ConfigurationBuilder()
.AddJsonFile("local.settings.json", optional: true, reloadOnChange: true)
.AddEnvironmentVariables()
.Build();
FileAnalysis fa = new FileAnalysis(log);
var contents = await fa.ReadFile(myBlob, name);
log.LogInformation($"# of records {contents.Count}");
var filtered = contents.ToList().GroupBy(g => new
{
g.ServiceName,
g.ResourceGroup,
g.UsageDateTime.Year,
g.UsageDateTime.Month
}).OrderByDescending(o => o.First().UsageDateTime).Select(s => new
{
UsageDate = $"{s.Key.Year}/{String.Format("{0:00}", s.Key.Month)}",
s.Key.ResourceGroup,
s.Key.ServiceName,
Sum = s.Sum(s => Math.Round(s.PreTaxCost, 2)),
s.First().Currency
});
var output = filtered.ToList().ToHtmlTable();
var message = new SendGridMessage();
message.AddTo(config["Sender"]);
message.AddContent("text/html", output);
message.SetFrom(new EmailAddress("*****@*****.**"));
message.SetSubject("Azure - Cost Analysis");
await messageCollector.AddAsync(message);
}
catch (Exception ex)
{
log.LogError(ex, ex.Message);
}
}
private void RunCorrector(FileAnalysis[] selectedFiles)
{
// If nothing is selected exit
if (selectedFiles == null || selectedFiles.Length <= 0)
return;
// Disable all the interactable UI
ShowProgressAndDisableUI("Correcting library files", "Correcting file locations");
_corrector.RunAsync(selectedFiles).ContinueWith(task =>
{
try
{
if (task.IsCanceled)
{
toolStripStatuslbl.Text = "Correction Cancelled";
}
else if (task.IsFaulted)
{
toolStripStatuslbl.Text = "Correction Error";
}
else if (task.IsCompleted)
{
toolStripStatuslbl.Text = "Correction Completed";
ResetMainGrid();
}
}
catch (Exception ex)
{
// TODO: show errors, how?
}
finally
{
HideProgressAndEnableUI();
}
}, scheduler: _uiScheduler, continuationOptions: TaskContinuationOptions.AttachedToParent, cancellationToken: CancellationToken.None);
}
/// <summary>
/// Creates a class report.
/// </summary>
/// <param name="reportRenderer">The report renderer.</param>
/// <param name="codeFile"></param>
/// <param name="fileAnalysis">The file analyses that correspond to the class.</param>
/// <param name="sourceFile">The source file.</param>
public virtual void CreateFileReport(IReportRenderer reportRenderer, CodeFile codeFile, FileAnalysis fileAnalysis)
{
if (reportRenderer == null)
{
throw new ArgumentNullException(nameof(reportRenderer));
}
if (fileAnalysis == null)
{
throw new ArgumentNullException(nameof(fileAnalysis));
}
string additionalTitle = this.ReportContext.ReportConfiguration.Title != null ? $"{this.ReportContext.ReportConfiguration.Title} - " : null;
reportRenderer.BeginFileReport(this.ReportContext.ReportConfiguration.TargetDirectory, codeFile.Path, additionalTitle);
if (this.ReportContext.ReportConfiguration.Title != null)
{
reportRenderer.HeaderWithBackLink($"{ReportResources.Summary} - {this.ReportContext.ReportConfiguration.Title}");
}
else
{
reportRenderer.HeaderWithBackLink(ReportResources.Summary);
}
reportRenderer.BeginKeyValueTable();
reportRenderer.KeyValueRow(ReportResources.Files3, codeFile.Path);
reportRenderer.KeyValueRow(ReportResources.CoveredLines, Convert.ToString(codeFile.CoveredLines, CultureInfo.CurrentUICulture));
reportRenderer.KeyValueRow(ReportResources.UncoveredLines, Convert.ToString(codeFile.CoverableLines - codeFile.CoveredLines, CultureInfo.CurrentUICulture));
reportRenderer.KeyValueRow(ReportResources.CoverableLines, Convert.ToString(codeFile.CoverableLines, CultureInfo.CurrentUICulture));
reportRenderer.KeyValueRow(ReportResources.TotalLines, Convert.ToString(codeFile.TotalLines, CultureInfo.CurrentUICulture));
if (this.ReportContext.ReportConfiguration.Tag != null)
{
reportRenderer.KeyValueRow(ReportResources.Tag, this.ReportContext.ReportConfiguration.Tag);
}
reportRenderer.FinishTable();
if (codeFile.MethodMetrics.Any())
{
reportRenderer.Header(ReportResources.Metrics);
reportRenderer.MetricsTable(codeFile.MethodMetrics);
}
reportRenderer.Header(ReportResources.Files);
reportRenderer.File(codeFile.Path);
if (!string.IsNullOrEmpty(fileAnalysis.Error))
{
reportRenderer.Paragraph(fileAnalysis.Error);
}
else
{
reportRenderer.BeginLineAnalysisTable(new[] { string.Empty, "#", ReportResources.Line, string.Empty, ReportResources.Coverage });
foreach (var line in fileAnalysis.Lines)
{
reportRenderer.LineAnalysis(0, line);
}
reportRenderer.FinishTable();
}
reportRenderer.AddFooter();
{
var testMethods = codeFile.TestMethods.OrderBy(l => l.ShortName);
var codeElementsByFileIndex = new Dictionary <int, IEnumerable <CodeElement> >();
int fileIndex = 0;
codeElementsByFileIndex.Add(fileIndex++, codeFile.CodeElements.OrderBy(c => c.FirstLine));
reportRenderer.TestMethods(testMethods, new[] { fileAnalysis }, codeElementsByFileIndex);
}
reportRenderer.SaveClassReport(this.ReportContext.ReportConfiguration.TargetDirectory, null, null);
}
private FileAnalysis ReadData(FileAnalysis fa)
{
try
{
using (var tran = this.dbInstance.GetTransaction())
{
if (fa.StoredHash != null)
{
murmurs.RemoveMurMur(tran, fa.Name);
hashes.RemoveHash(tran, fa.StoredHash);
}
murmurs.InsertMurMur(tran, fa.Name, fa.CurrentHash);
tran.Commit();
}
using (var fs = new FileStream(fa.Name, FileMode.Open, FileAccess.Read, FileShare.Read, 4096 * 1024, true))
using (var ms = new MemoryStream((int)fs.Length))
{
var imgW = 0;
var imgH = 0;
using (var img = Image.FromStream(fs, false, false))
{
imgW = img.Width;
imgH = img.Height;
if (img.RawFormat.Guid == System.Drawing.Imaging.ImageFormat.Bmp.Guid)
{
fs.CopyTo(ms);
}
else
{
img.Save(ms, System.Drawing.Imaging.ImageFormat.Bmp);
}
}
var bytes = ms.GetBuffer();
return new FileAnalysis
{
Name = fa.Name,
LastModified = fa.LastModified,
CurrentHash = fa.CurrentHash,
StoredHash = fa.StoredHash,
Width = imgW,
Height = imgH,
Data = bytes
};
}
}
catch (Exception ex)
{
this.log.Warn(string.Format("Data reading failed for {0}", fa.Name), ex);
using (var tran = this.dbInstance.GetTransaction())
{
murmurs.RemoveMurMur(tran, fa.Name);
tran.Commit();
}
if (OnSkipProgress != null)
OnSkipProgress(fa.Name);
}
return null;
}
private unsafe void CalculatePHash(FileAnalysis fa)
{
try
{
ulong hash = 0;
fixed (byte* p = fa.Data)
{
ph_dct_imagehash2((IntPtr)p, fa.Width, fa.Height, ref hash);
}
using (var tran = this.dbInstance.GetTransaction())
{
hashes.InsertHash(tran, fa.CurrentHash, hash);
tran.Commit();
}
if (OnProgress != null)
OnProgress(fa.Name);
}
catch (AccessViolationException avex)
{
this.log.Warn(string.Format("Access violation for {0}", fa.Name), avex);
using (var tran = this.dbInstance.GetTransaction())
{
murmurs.RemoveMurMur(tran, fa.Name);
tran.Commit();
}
if (OnSkipProgress != null)
OnSkipProgress(fa.Name);
}
catch (SEHException sehex)
{
this.log.Warn(string.Format("SEH error for {0}", fa.Name), sehex);
using (var tran = this.dbInstance.GetTransaction())
{
murmurs.RemoveMurMur(tran, fa.Name);
tran.Commit();
}
if (OnSkipProgress != null)
OnSkipProgress(fa.Name);
}
}
private FileAnalysis CalculateHashes(FileAnalysis fa)
{
var storedMurmur = string.Empty;
using (var tran = this.dbInstance.GetTransaction())
{
storedMurmur = murmurs.GetMurMur(tran, fa.Name);
}
if (fa.LastModified < DateTime.Now.AddDays(-1) && !string.IsNullOrEmpty(storedMurmur))
{
return new FileAnalysis
{
Name = fa.Name,
LastModified = fa.LastModified,
CurrentHash = storedMurmur,
StoredHash = storedMurmur
};
}
using (var fs = new FileStream(fa.Name, FileMode.Open, FileAccess.Read, FileShare.Read, 4096 * 1024, true))
{
var currentMurmur = BitConverter.ToString(murmur128.ComputeHash(fs)).Replace("-", "");
return new FileAnalysis
{
Name = fa.Name,
LastModified = fa.LastModified,
CurrentHash = currentMurmur,
StoredHash = storedMurmur
};
}
}
