public virtual PersistenceStageResult Persist(SinkStageArtifact sa, Dictionary <string, object> extraMeta = null)
{
if (sa != null && sa.ByteStream != null && sa.ByteStream.CanRead && sa.ByteStream.CanSeek)
{
Id = Guid.NewGuid();
Timestamp = DateTime.UnixEpoch;
ArtifactName = FinalArtifactName(_genConfig, sa);
return(new PersistenceStageResult());
}
return(null);
}
// Returned format:
// [userPrefixIfSet-][specificationChars][recordCount]-[first4configGuid]-[timestamp]-[first4artifactGuid].[outputTypeExtenstion]
// |----------- static for all artifacts in a pipeline -----------| |-------------- dynamic per-artifact -----------------|
private string FinalArtifactName(GeneratorConfig gCfg, SinkStageArtifact prevArt)
{
return
// [userPrefixIfSet-]
(((string.IsNullOrEmpty(gCfg.UserPrefix) ?
"" : $"{gCfg.UserPrefix}-") +
// [specificationChars][recordCount]
$"{gCfg.Specification.ToString().ToUpper().Substring(0, 3)}{gCfg.Count}" +
// -[first4configGuid]
$"-{gCfg.Id.ToString("N").Substring(0, 4)}" +
// -[timestamp]-
$"-{Timestamp:MMddyyTHHmmZ}-" +
// [first4artifactGuid]
$"{prevArt.Id.ToString("N").Substring(0, 4)}").ToUpper() +
// .[outputTypeExtenstion]
$".{prevArt.Type}");
}
public override PersistenceStageResult Persist(SinkStageArtifact artifact, Dictionary <string, object> extraMeta = null)
{
if (base.Persist(artifact, extraMeta) != null)
{
var fullArtifactPath = Path.GetFullPath(Path.Combine(_config.PersistDirectory, ArtifactName));
using (var fileStream = new FileStream(fullArtifactPath, FileMode.OpenOrCreate))
{
artifact.ByteStream.CopyTo(fileStream);
Status = PersistenceStatus.Success;
Console.WriteLine($"{this}: {artifact.Id} saved to {fullArtifactPath}");
return(new PersistenceStageResult()
{
Id = Id,
PipelineId = _genConfig.Id,
Timestamp = Timestamp,
Success = true,
Meta = new Dictionary <string, object>()
{
{ "id", artifact.Id.ToString() },
{ "specification", _genConfig.Specification.ToString().ToLower() },
{ "recordCount", _genConfig.Count.ToString() },
{ "sizeBytes", new FileInfo(fullArtifactPath).Length },
{ "path", fullArtifactPath }
}
});
}
}
Status = PersistenceStatus.Errored;
return(new PersistenceStageResult()
{
Id = Id,
Timestamp = Timestamp,
PipelineId = _genConfig.Id,
Success = false,
Meta = null
});
}
public override PersistenceStageResult Persist(SinkStageArtifact artifact, Dictionary <string, object> extraMeta = null)
{
if (base.Persist(artifact, extraMeta) != null)
{
// replace the current artifact with a zipped version for upload
if (_config.Storage.Blob.Compress)
{
var tempZipName = Path.GetTempFileName();
using (var zipFileWriter = new FileStream(tempZipName, FileMode.OpenOrCreate))
using (var zipArchive = new ZipArchive(zipFileWriter, ZipArchiveMode.Create))
{
Status = PersistenceStatus.Compressing;
var artifactEntry = zipArchive.CreateEntry(ArtifactName, CompressionLevel.Optimal);
using (var zipArtStream = artifactEntry.Open())
{
artifact.ByteStream.CopyTo(zipArtStream);
}
}
ArtifactName += ".zip";
// don't modify the original artifact object, as persisters further down
// the change need it to remain as-emitted from the sink stage
artifact = new SinkStageArtifact(this)
{
Id = artifact.Id,
Name = tempZipName,
ByteStream = new FileStream(tempZipName, FileMode.Open, FileAccess.Read, FileShare.Read)
};
}
var fSize = new FileInfo(artifact.Name).Length;
// this matches the format (B64-encoded MD5) that Azure Blobs natively checksum with
var md5Base64 = Convert.ToBase64String(
new MD5CryptoServiceProvider().ComputeHash(artifact.ByteStream));
artifact.ByteStream.Seek(0, SeekOrigin.Begin);
var uploadSw = new Stopwatch();
var blobUrl = $"https://{_act}.blob.core.windows.net/{_cnt}/{ArtifactName}";
if (_config.DevSettings != null && _config.DevSettings.DisableUpload)
{
Console.WriteLine("DevSettings.DisableUpload=True, skipping upload.");
}
else
{
CloudBlockBlob blob = _blobContainer.GetBlockBlobReference(ArtifactName);
Console.WriteLine($"{this} uploading:\n\tPipelineId: {_genConfig.Id}" +
$"\n\tSize: {fSize}\n\tBlob: {blobUrl}");
Status = PersistenceStatus.Uploading;
uploadSw.Start();
blob.UploadFromStream(artifact.ByteStream);
blob.Metadata["specification"] = _genConfig.Specification.ToString();
blob.Metadata["recordcount"] = _genConfig.Count.ToString();
blob.Metadata["sizebytes"] = fSize.ToString();
blob.Metadata["pipelineid"] = _genConfig.Id.ToString();
blob.Metadata["artifactid"] = artifact.Id.ToString();
blob.Metadata["contentmd5"] = md5Base64;
blob.Metadata["timestamp"] = Timestamp.ToString("O");
if (!string.IsNullOrEmpty(_genConfig.UserPrefix))
{
blob.Metadata["userprefix"] = _genConfig.UserPrefix;
}
if (extraMeta != null)
{
if (extraMeta.ContainsKey("pipelineInitiator") && extraMeta["pipelineInitiator"] != null)
{
blob.Metadata["remote"] = ((PipelineRequestTracker)extraMeta["pipelineInitiator"]).RemoteAddr;
}
if (extraMeta.ContainsKey("tranformStageElapsed"))
{
blob.Metadata["elapsed"] = extraMeta["tranformStageElapsed"].ToString();
}
}
if (_genConfig.Filters.Count > 0)
{
blob.Metadata["defectlist"] = string.Join(',', _genConfig.Filters);
}
blob.SetMetadata();
uploadSw.Stop();
Console.WriteLine($"{this}: {artifact.Id} uploaded in {uploadSw.Elapsed}");
Status = PersistenceStatus.CleaningUp;
}
// cleanup the extra zip temp file, if extant
if (_config.Storage.Blob.Compress)
{
artifact.Cleanup();
}
Status = PersistenceStatus.Success;
return(new PersistenceStageResult()
{
Id = Id,
PipelineId = _genConfig.Id,
Timestamp = Timestamp,
Success = true,
Meta = new Dictionary <string, object>()
{
{ "container", _cnt },
{ "blob", new Dictionary <string, object>()
{
{ "id", artifact.Id.ToString() },
{ "specification", _genConfig.Specification.ToString().ToLower() },
{ "recordCount", _genConfig.Count.ToString() },
{ "sizeBytes", fSize },
{ "contentMD5", md5Base64 },
{ "uploadDuration", uploadSw.Elapsed },
{ "url", blobUrl }
} }
}
});
}
Status = PersistenceStatus.Errored;
return(new PersistenceStageResult()
{
Id = Id,
Timestamp = Timestamp,
PipelineId = _genConfig.Id,
Success = false,
Meta = null
});
}