private async Task <DumpArtifact> AddDumpArtifactToDbAsync(DumplingDb dumplingDb, string dumpId, string localPath, string hash, CancellationToken cancelToken)
{
using (var opTracker = new TrackedOperation("AddDumpArtifactToDbAsync"))
{
//if the specified dumpId is not valid throw an exception
if (await dumplingDb.Dumps.FindAsync(cancelToken, dumpId) == null)
{
throw new HttpResponseException(Request.CreateErrorResponse(HttpStatusCode.BadRequest, "The specified dumpling id is invalid."));
}
var dumpArtifact = new DumpArtifact()
{
DumpId = dumpId,
LocalPath = localPath,
DebugCritical = true,
ExecutableImage = false,
Hash = hash
};
dumplingDb.DumpArtifacts.AddOrUpdate(dumpArtifact);
await dumplingDb.SaveChangesAsync(cancelToken);
return(dumpArtifact);
}
}
public async Task <Dump> GetDumplingManifest(string dumplingId)
{
using (var op1 = new TrackedOperation("GetDumplingManifest"))
{
using (DumplingDb dumplingDb = new DumplingDb())
{
Dump dump = null;
using (var op2 = new TrackedOperation("FindDumpAsync"))
{
dump = await dumplingDb.Dumps.FindAsync(dumplingId);
}
if (dump != null)
{
using (var op3 = new TrackedOperation("LoadDumpArtifactsAsync"))
{
await dumplingDb.Entry(dump).Collection(d => d.DumpArtifacts).LoadAsync();
}
}
return(dump);
}
}
}
private async Task StoreArtifactContentAsync(HttpContent content, string hash, string dumpId, string localPath, CancellationToken cancelToken)
{
//if the specified hash is not formatted properly throw an exception
if (!ValidateHashFormat(hash))
{
throw new HttpResponseException(Request.CreateErrorResponse(HttpStatusCode.BadRequest, "The specified hash is improperly formatted"));
}
using (DumplingDb dumplingDb = new DumplingDb())
{
var artifact = await AddArtifactToDbAsync(dumplingDb, hash, localPath, cancelToken);
//if the artifact didn't already exist and we added it to the db upload the file
if (artifact != null)
{
using (var uploaded = await UploadContentValidateHashAsync(content, hash, cancelToken))
{
artifact.CompressedSize = uploaded.Length;
uploaded.Position = 0;
artifact.Url = await DumplingStorageClient.StoreArtifactAsync(uploaded, hash, artifact.FileName + ".gz", cancelToken);
await dumplingDb.SaveChangesAsync(cancelToken);
}
}
//if a dumpId was specified add the dumpartifact entry
if (dumpId != null)
{
await AddDumpArtifactToDbAsync(dumplingDb, dumpId, localPath, hash, cancelToken);
}
}
}
public async Task <HttpResponseMessage> DownloadArtifact(string hash, CancellationToken cancelToken)
{
using (var dumplingDb = new DumplingDb())
{
var artifact = await dumplingDb.Artifacts.FindAsync(hash);
return(await GetArtifactRedirectAsync(artifact, cancelToken));
}
}
public async Task <HttpResponseMessage> DownloadIndexedArtifact(string index, CancellationToken cancelToken)
{
Artifact artifact = null;
using (var dumplingDb = new DumplingDb())
{
var artifactIndex = await dumplingDb.ArtifactIndexes.FindAsync(index);
artifact = artifactIndex?.Artifact;
}
return(await GetArtifactRedirectAsync(artifact, cancelToken));
}
public async Task <HttpResponseMessage> DownloadArchivedDump(string dumplingid, CancellationToken cancelToken)
{
using (var dumplingDb = new DumplingDb())
{
var dump = await dumplingDb.Dumps.FindAsync(cancelToken, dumplingid);
if (dump == null)
{
return(Request.CreateResponse(HttpStatusCode.NotFound));
}
var archiveTasks = new List <Task>();
dumplingDb.Entry(dump).Collection(d => d.Properties).Load();
var fileName = dump.DisplayName + ".zip";
var tempFile = CreateTempFile();
try
{
using (var zipArchive = new ZipArchive(tempFile, ZipArchiveMode.Create, true))
using (var archiveLock = new SemaphoreSlim(1, 1))
{
//find all the artifacts associated with the dump
foreach (var dumpArtifact in dump.DumpArtifacts.Where(da => da.Hash != null))
{
await dumplingDb.Entry(dumpArtifact).Reference(d => d.Artifact).LoadAsync(cancelToken);
archiveTasks.Add(DownloadArtifactToArchiveAsync(dumpArtifact, zipArchive, archiveLock, cancelToken));
}
await Task.WhenAll(archiveTasks.ToArray());
await tempFile.FlushAsync();
}
await tempFile.FlushAsync();
tempFile.Position = 0;
HttpResponseMessage result = new HttpResponseMessage(HttpStatusCode.OK);
result.Content = new StreamContent(tempFile);
result.Content.Headers.ContentType = new MediaTypeHeaderValue("application/zip");
result.Content.Headers.ContentDisposition = new ContentDispositionHeaderValue("attachment");
result.Content.Headers.ContentDisposition.FileName = fileName;
return(result);
}
catch
{
tempFile.Dispose();
throw;
}
}
}
public async Task <ActionResult> Index(int?days)
{
var queryDict = new Dictionary <string, string>();
foreach (var key in this.Request.QueryString.AllKeys)
{
queryDict[key] = this.Request.QueryString.Get(key);
}
days = days.HasValue ? days : 10;
var minTime = DateTime.UtcNow.AddDays(days.Value * -1);
var maxTime = DateTime.UtcNow.AddDays(1);
List <Dump> loadedDumps = null;
using (var dumplingDb = new DumplingDb())
{
loadedDumps = dumplingDb.FindDumps(minTime, maxTime, queryDict).ToList();
foreach (var dump in loadedDumps)
{
dumplingDb.Entry(dump).Reference(d => d.Failure).Load();
dumplingDb.Entry(dump).Collection(d => d.Properties).Load();
}
}
var dumpsByFailure = loadedDumps.GroupBy(d => d.FailureHash);
var failures = new List <string>();
foreach (var failure in dumpsByFailure.OrderByDescending(g => g.Count()))
{
failures.Add(failure.Key ?? "UNTRIAGED");
var dumps = failure.OrderByDescending(d => d.DumpTime).ToArray();
ViewData[failure.Key ?? "UNTRIAGED"] = dumps;
foreach (var dump in dumps)
{
ViewData["dumpid." + dump.DumpId] = JsonConvert.SerializeObject(dump.GetPropertyBag());
}
}
ViewBag.Title = "dumpling";
ViewData["Failures"] = failures.ToArray();
return(View("Index"));
}
private async Task <Artifact> AddArtifactToDbAsync(DumplingDb dumplingDb, string hash, string localPath, CancellationToken cancelToken)
{
using (var opTracker = new TrackedOperation("AddArtifactToDbAsync"))
{
var artifact = new Artifact()
{
Hash = hash,
FileName = Path.GetFileName(localPath),
UploadTime = DateTime.UtcNow,
Format = ArtifactFormat.Unknown,
Uuid = null
};
return(await dumplingDb.TryAddAsync(artifact, cancelToken) ? artifact : null);
}
}
public async Task <string> CreateDump([FromUri] string hash, [FromUri] string user, [FromUri] string displayName, CancellationToken cancelToken)
{
//if the specified hash is not formatted properly throw an exception
if (!ValidateHashFormat(hash))
{
throw new HttpResponseException(Request.CreateErrorResponse(HttpStatusCode.BadRequest, "The specified hash is improperly formatted"));
}
using (DumplingDb dumplingDb = new DumplingDb())
{
var dumpling = await dumplingDb.Dumps.FindAsync(cancelToken, hash);
if (dumpling != null)
{
return(hash);
}
dumpling = new Dump()
{
DumpId = hash, User = user, DisplayName = displayName, DumpTime = DateTime.UtcNow, Os = OS.Unknown
};
dumplingDb.Dumps.Add(dumpling);
try
{
await dumplingDb.SaveChangesAsync();
}
catch (DbEntityValidationException)
{
dumpling = await dumplingDb.Dumps.FindAsync(cancelToken, hash);
//if the specified dump was not found throw an exception
if (dumpling != null)
{
return(hash);
}
throw;
}
return(hash);
}
}
public async Task <HttpResponseMessage> UpdateDumpProperties(string dumplingid, [FromBody] JToken properties, CancellationToken cancelToken)
{
using (DumplingDb dumplingDb = new DumplingDb())
{
var dumpling = await dumplingDb.Dumps.FindAsync(cancelToken, dumplingid);
//if the specified dump was not found throw an exception
if (dumpling == null)
{
throw new HttpResponseException(Request.CreateErrorResponse(HttpStatusCode.BadRequest, "The given dumplingId is invalid"));
}
var propDict = JsonConvert.DeserializeObject <Dictionary <string, string> >(properties.ToString());
string failureHash = null;
//if the properties contain FAILURE_HASH get the failure
if (propDict.TryGetValue("FAILURE_HASH", out failureHash))
{
//if the failure is not in the db yet try to add it
if (await dumplingDb.Failures.FindAsync(cancelToken, failureHash) == null)
{
try
{
dumplingDb.Failures.Add(new Failure()
{
FailureHash = failureHash
});
await dumplingDb.SaveChangesAsync();
}
//swallow the validation exception if the failure was inserted by another request since we checked
catch (DbEntityValidationException e)
{
}
}
dumpling.FailureHash = failureHash;
}
//update any properties which were pre-existing with the new value
foreach (var existingProp in dumpling.Properties)
{
string val = null;
if (propDict.TryGetValue(existingProp.Name, out val))
{
existingProp.Value = val;
propDict.Remove(existingProp.Name);
}
}
//add any properties which were not previously existsing
//(the existing keys have been removed in previous loop)
foreach (var newProp in propDict)
{
dumpling.Properties.Add(new Property()
{
Name = newProp.Key, Value = newProp.Value
});
}
await dumplingDb.SaveChangesAsync();
return(Request.CreateResponse(HttpStatusCode.OK));
}
}
public async Task ProcessAsync()
{
var stopwatch = Stopwatch.StartNew();
var telemProperties = new Dictionary <string, string>()
{
{ "Hash", ExpectedHash },
{ "FileName", FileName },
{ "DumpId", DumpId },
{ "OperationToken", _optoken },
};
Telemetry.Client.TrackEvent(this.GetType().Name + "_Started", telemProperties);
try
{
using (_dumplingDb = new DumplingDb())
{
var artifact = await CreateArtifactAsync();
if (artifact != null)
{
using (Stream decompressed = await DecompAndHashAsync())
{
if (Hash == ExpectedHash)
{
Size = decompressed.Length;
ProcessDecompressedFile(decompressed);
await UpdateArtifactAsync(artifact);
}
else
{
telemProperties["ProcessingState"] = "failed_validation";
await DeleteArtifactAsync(artifact);
}
}
}
telemProperties["ProcessingState"] = "success";
}
}
catch (Exception e)
{
telemProperties["ProcessingState"] = "failed_exception";
telemProperties["Exception"] = e.ToString();
Telemetry.Client.TrackException(e, telemProperties);
}
File.Delete(_path);
stopwatch.Stop();
Telemetry.Client.TrackEvent(this.GetType().Name + "_Completed", telemProperties, new Dictionary <string, double>()
{
{ "ProcessingTime", Convert.ToDouble(stopwatch.ElapsedMilliseconds) }
});
Telemetry.Client.Flush();
// Allow some time for flushing before shutdown.
await Task.Delay(1000);
}
