public static ICollection <string> CreateEvidence(this DataSourceFetchItem item)
{
Requires.NonNull(item, nameof(item));
var e = new List <string>();
var meta = item.DataSourceFetchItemProperties?.Metadata;
if (meta != null)
{
var urns = meta.FindOrDefault(BlobStorageServices.MetaKeyNames.Urns);
if (urns != null)
{
foreach (var urn in CSV.ParseLine(urns))
{
e.Add(CreateFromUrn(urn));
}
}
if (meta.ContainsKey(BlobStorageServices.MetaKeyNames.ETag))
{
e.Add(CreateFromContentETag(meta[BlobStorageServices.MetaKeyNames.ETag]));
}
if (meta.ContainsKey(BlobStorageServices.MetaKeyNames.ContentMD5))
{
e.Add(CreateFromContentMD5(meta[BlobStorageServices.MetaKeyNames.ContentMD5]));
}
}
e.Add(CreateFromNameSizeModified(item.Name, item.Size, item.DataSourceFetchItemProperties.LastModifiedAtUtc));
e.Add(CreateFromNameSizeModified(item.DataSourceFetchItemProperties.Metadata.FindOrDefault(BlobStorageServices.MetaKeyNames.SourceFullName), item.Size, item.DataSourceFetchItemProperties.LastModifiedAtUtc));
return(e);
}
private void PopulateEvidence(DataSourceFetchItem item)
{
lock (FetchItemByEvidence)
{
foreach (var e in EvidenceFactory.CreateEvidence(item))
{
FetchItemByEvidence[e] = item;
}
}
}
private async Task FetchTheItemAsync(DataSourceFetche fetch, FileDetails details, DataSourceFetchItem.DataSourceFetchItemTypes dataSourceFetchItemType, DataSourceFetchItem parentFetchItem, Func <FileDetails, Task <string> > fetchAsync)
{
string tfn = null;
var item = new DataSourceFetchItem
{
DataSourceFetch = fetch,
DataSourceFetchItemType = dataSourceFetchItemType,
ParentDataSourceFetchItem = parentFetchItem,
Size = details.Size,
Name = details.Name,
};
item.DataSourceFetchItemProperties.LastModifiedAtUtc = details.LastModifiedAtUtc;
item.DataSourceFetchItemProperties.ContentMD5 = details.ContentMD5;
item.DataSourceFetchItemProperties.ETag = details.ETag;
try
{
Trace.WriteLine($"Checking {details.FullName} size={details.Size} LastWriteTimeUtc={details.LastModifiedAtUtc}");
var sameDataSourceReplicatedDataSourceFetchItem = FindEvidenceItems(details.CreateEvidence()).FirstOrDefault();
if (sameDataSourceReplicatedDataSourceFetchItem != null)
{
item.DataSourceFetchItemType = DataSourceFetchItem.DataSourceFetchItemTypes.Duplicate;
item.SameDataSourceReplicatedDataSourceFetchItem = sameDataSourceReplicatedDataSourceFetchItem;
return;
}
// Logger.LogInformation("Downloading", file.FullName, file.Length, tfn);
tfn = await fetchAsync(details);
using (var st = File.OpenRead(tfn))
{
item.Size = st.Length;
using (var muxer = new StreamMuxer(st, true))
{
var p = new BlobStorageServices.FileProperties
{
LastModifiedAtUtc = details.LastModifiedAtUtc
};
p.Metadata[BlobStorageServices.MetaKeyNames.SourcePath] = details.Folder;
p.Metadata[BlobStorageServices.MetaKeyNames.SourceFullName] = details.FullName;
var urns = new List <string>();
Parallel.ForEach(
new[]
{
Hash.CommonHashAlgorithmNames.Md5,
Hash.CommonHashAlgorithmNames.Sha1,
Hash.CommonHashAlgorithmNames.Sha512,
},
hashAlgName =>
{
var urn = Hash.Compute(muxer.OpenRead(), hashAlgName).Urn;
if (urn == null)
{
return; //yes... in some cases this somehow happens...
}
urns.Add(urn);
});
if (urns.Count > 0)
{
p.Metadata[BlobStorageServices.MetaKeyNames.Urns] = CSV.FormatLine(urns, false);
sameDataSourceReplicatedDataSourceFetchItem = FindEvidenceItems(urns).FirstOrDefault();
if (sameDataSourceReplicatedDataSourceFetchItem != null)
{
item.DataSourceFetchItemType = DataSourceFetchItem.DataSourceFetchItemTypes.Duplicate;
item.SameDataSourceReplicatedDataSourceFetchItem = sameDataSourceReplicatedDataSourceFetchItem;
return;
}
}
var res = await BlobStorageServices.StoreStreamAsync(
Runner.BlobConfig,
BlobStorageServices.ContainerNames.Secure,
$"{BlobRootPath}{details.Folder.Substring(1)}{details.Name}",
muxer.OpenRead(),
p,
amt => Trace.WriteLine($"Uploading {amt}/{muxer.Length}")
);
item.DataSourceFetchItemProperties = new DataSourceFetchItemProperties();
item.DataSourceFetchItemProperties.Set(p);
item.Url = res.Uri.ToString();
PopulateEvidence(item);
}
}
}
catch (Exception ex)
{
item.DataSourceFetchItemType = DataSourceFetchItem.DataSourceFetchItemTypes.Errored;
item.DataSourceFetchItemProperties.Error = new ExceptionError(ex);
Trace.WriteLine(ex);
}
finally
{
if (item != null)
{
await GdbLocker.GoAsync(async() => {
Gdb.DataSourceFetchItems.Add(item);
await Gdb.SaveChangesAsync();
});
}
}
var ext = Path.GetExtension(details.Name).ToLower();
if (ext == ".pgp" || details.Name.ToLower().Contains(".pgp."))
{
var name = details.Name;
if (name.ToLower().EndsWith(".pgp"))
{
name = name.Left(name.Length - 4);
}
else if (name.ToLower().EndsWith(".pgp.asc"))
{
name = name.Left(name.Length - 8);
}
else if (name.ToLower().Contains(".pgp."))
{
name = new Regex(@"\.pgp\.", RegexOptions.IgnoreCase).Replace(name, ".");
}
await FetchTheItemAsync(
fetch,
new FileDetails(details, name),
DataSourceFetchItem.DataSourceFetchItemTypes.Decrypted,
item,
async _ =>
{
var utfp = Path.GetTempFileName();
using (var st = File.OpenRead(tfn))
{
await Runner.DecryptAsync(st, utfp);
}
return(utfp);
}
);
}
else if (
MimeType.Application.Zip.DoesExtensionMatch(details.Name) &&
DS.DataSourceSettings.DecompressItems &&
dataSourceFetchItemType != DataSourceFetchItem.DataSourceFetchItemTypes.UnpackedRecompressedSingleton)
{
var relUnzipFolder = Path.GetFileNameWithoutExtension(details.Name);
var unzipFolder = Path.Combine(Path.GetDirectoryName(tfn), relUnzipFolder);
using (var st = File.OpenRead(tfn))
{
using (var za = new ZipArchive(st, ZipArchiveMode.Read))
{
if (za.Entries.Count < 2)
{
return;
}
}
}
ZipFile.ExtractToDirectory(tfn, unzipFolder);
await TaskWhenAllOneAtATime(
Directory.GetFiles(unzipFolder, "*.*", SearchOption.AllDirectories).ConvertAll(
unzipped =>
{
string rezipped = unzipped;
bool isRezipped = false;
if (!MimeType.Application.Zip.DoesExtensionMatch(unzipped))
{
rezipped = unzipped + MimeType.Application.Zip.PrimaryFileExtension;
using (var st = File.Create(rezipped))
{
using (var za = new ZipArchive(st, ZipArchiveMode.Create))
{
za.CreateEntryFromFile(unzipped, Path.GetFileName(unzipped));
}
isRezipped = true;
}
}
return(FetchTheItemAsync(
fetch,
new FileDetails(new FileInfo(rezipped), Path.Combine(details.Folder, relUnzipFolder)),
isRezipped ? DataSourceFetchItem.DataSourceFetchItemTypes.UnpackedRecompressedSingleton : DataSourceFetchItem.DataSourceFetchItemTypes.Unpacked,
item,
_ => Task.FromResult(rezipped)
));
}));
Stuff.Noop();
}
}
