public async Task <(int count, List <string> files)> DeliveryFileReader(WriterConfiguration conf)
{
var settings = new XmlReaderSettings {
Async = true
};
var files = new List <string>();
var count = 0;
using (var stream = File.OpenRead(Path.Combine(conf.ImportPath, conf.DeliveryFile))) {
using (var reader = XmlReader.Create(stream, settings)) {
while (await reader.ReadAsync().ConfigureAwait(false))
{
switch (reader.NodeType)
{
case XmlNodeType.Element:
switch (reader.Name)
{
case "files":
count = int.Parse(reader.GetAttribute("total.files"));
break;
case "file":
files.Add(Path.Combine(conf.ImportPath, reader.GetAttribute("name")));
break;
}
break;
}
}
}
return(count, files);
}
}
public PipelineCreator(
IParserDefinition definition,
WriterConfiguration config,
VariantType variant
)
{
_definition = definition;
_configuration = config;
_variant = variant;
_singleWriterExecution = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 1
};
_options = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount,
BoundedCapacity = 1000000,
MaxMessagesPerTask = 10000
};
_linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
_plDictionary = new Dictionary <string, HierarchyNode>();
_hierarchyRoot = new HPEHierarchyNode();
_csvOutputWriter = new CsvOutputWriter(config);
_jsonOutputWriter = new JsonOutputWriter(config);
}
protected async Task Import(WriterConfiguration config, IParserDefinition parserDefinition,
VariantType variant)
{
var sw = new Stopwatch();
sw.Start();
const FileTypes fileTypes = FileTypes.Detail
| FileTypes.Link
| FileTypes.Marketing
| FileTypes.Option
| FileTypes.Option
| FileTypes.Product
| FileTypes.Specification
| FileTypes.Supplier
| FileTypes.Json;
FileHelpers.DeleteExistingFiles(fileTypes, config);
Console.WriteLine("Deleting files");
var(pipeline, targetTask) = new PipelineCreator(parserDefinition, config, variant).CreatePipeline();
await pipeline.SendAsync(config).ConfigureAwait(false);
pipeline.Complete();
await targetTask.ContinueWith(_ => {
sw.Stop();
Console.WriteLine($"Done in {sw.Elapsed.ToString()}");
}).ConfigureAwait(false);
Console.ReadLine();
}
public EmittedStream(
string streamId, WriterConfiguration writerConfiguration, ProjectionVersion projectionVersion,
PositionTagger positionTagger, CheckpointTag fromCheckpointPosition, IPublisher publisher,
IODispatcher ioDispatcher,
IEmittedStreamContainer readyHandler, bool noCheckpoints = false)
{
if (string.IsNullOrEmpty(streamId))
{
throw new ArgumentNullException("streamId");
}
if (writerConfiguration == null)
{
throw new ArgumentNullException("writerConfiguration");
}
if (positionTagger == null)
{
throw new ArgumentNullException("positionTagger");
}
if (fromCheckpointPosition == null)
{
throw new ArgumentNullException("fromCheckpointPosition");
}
if (publisher == null)
{
throw new ArgumentNullException("publisher");
}
if (ioDispatcher == null)
{
throw new ArgumentNullException("ioDispatcher");
}
if (readyHandler == null)
{
throw new ArgumentNullException("readyHandler");
}
_streamId = streamId;
_metadataStreamId = SystemStreams.MetastreamOf(streamId);
_writerConfiguration = writerConfiguration;
_projectionVersion = projectionVersion;
_writeAs = writerConfiguration.WriteAs;
_positionTagger = positionTagger;
_zeroPosition = positionTagger.MakeZeroCheckpointTag();
_fromCheckpointPosition = fromCheckpointPosition;
_lastQueuedEventPosition = null;
_publisher = publisher;
_ioDispatcher = ioDispatcher;
_readyHandler = readyHandler;
_maxWriteBatchLength = writerConfiguration.MaxWriteBatchLength;
_logger = writerConfiguration.Logger;
_noCheckpoints = noCheckpoints;
}
protected void VerifyPaths(WriterConfiguration config)
{
var csvOutputDir = Path.Combine(config.OutputPath, config.CsvDirectory);
if (!Directory.Exists(csvOutputDir))
{
Directory.CreateDirectory(csvOutputDir);
}
var jsonOutputDir = Path.Combine(config.OutputPath, config.JsonDirectory);
if (!Directory.Exists(jsonOutputDir))
{
Directory.CreateDirectory(jsonOutputDir);
}
}
public EmittedStream(
string streamId, WriterConfiguration writerConfiguration, ProjectionVersion projectionVersion,
PositionTagger positionTagger, CheckpointTag @from, IODispatcher ioDispatcher,
IEmittedStreamContainer readyHandler, bool noCheckpoints = false)
{
if (streamId == null)
{
throw new ArgumentNullException("streamId");
}
if (writerConfiguration == null)
{
throw new ArgumentNullException("writerConfiguration");
}
if (positionTagger == null)
{
throw new ArgumentNullException("positionTagger");
}
if (@from == null)
{
throw new ArgumentNullException("from");
}
if (ioDispatcher == null)
{
throw new ArgumentNullException("ioDispatcher");
}
if (readyHandler == null)
{
throw new ArgumentNullException("readyHandler");
}
if (streamId == "")
{
throw new ArgumentException("streamId");
}
_streamId = streamId;
_writerConfiguration = writerConfiguration;
_projectionVersion = projectionVersion;
_writeAs = writerConfiguration.WriteAs;
_positionTagger = positionTagger;
_zeroPosition = positionTagger.MakeZeroCheckpointTag();
_from = @from;
_lastQueuedEventPosition = null;
_ioDispatcher = ioDispatcher;
_readyHandler = readyHandler;
_maxWriteBatchLength = writerConfiguration.MaxWriteBatchLength;
_logger = writerConfiguration.Logger;
_noCheckpoints = noCheckpoints;
}
public async Task OnExecuteAsync()
{
var name = Variant.ToString();
var deliveryFile = FileHelpers.FindDeliveryFile(ImportDirectory);
var config = new WriterConfiguration {
ImportPath = ImportDirectory,
DeliveryFile = deliveryFile,
OutputPath = OutputDirectory,
CsvZipFileName = $"{name}.zip",
JsonZipFileName = $"{name}All.zip"
};
var runner = Variant == VariantType.HPE
? new HpEnterpriseImportParser(config)
: (IRunner) new HpIncImportParser(config);
await runner.Execute().ConfigureAwait(false);
}
public static void DeleteExistingFiles(FileTypes fileTypes, WriterConfiguration config)
{
DeleteFile(FileTypes.Detail, fileTypes,
Path.Combine(config.OutputPath, config.CsvDirectory, config.DetailFileName));
DeleteFile(FileTypes.Link, fileTypes,
Path.Combine(config.OutputPath, config.CsvDirectory, config.LinkFileName));
DeleteFile(FileTypes.Marketing, fileTypes,
Path.Combine(config.OutputPath, config.CsvDirectory, config.MarketingFileName));
DeleteFile(FileTypes.Option, fileTypes,
Path.Combine(config.OutputPath, config.CsvDirectory, config.OptionFileName));
DeleteFile(FileTypes.Product, fileTypes,
Path.Combine(config.OutputPath, config.CsvDirectory, config.ProductFileName));
DeleteFile(FileTypes.Specification, fileTypes,
Path.Combine(config.OutputPath, config.CsvDirectory, config.SpecificationFileName));
DeleteFile(FileTypes.Pure_Hierarchy, fileTypes,
Path.Combine(config.OutputPath, config.CsvDirectory, config.PureHierarchyFileName));
DeleteFile(FileTypes.Supplier, fileTypes,
Path.Combine(config.OutputPath, config.CsvDirectory, config.SupplierFileName));
DeleteFile(FileTypes.Json, fileTypes,
Path.Combine(config.OutputPath, config.CsvDirectory, config.JsonFileName));
}
private static void OutputHierarchies(WriterConfiguration config,
Dictionary <string, HierarchyNode> plDictionary,
HPEHierarchyNode hierarchyRoot)
{
var pureHierarchyPath = Path.Combine(config.OutputPath, config.CsvDirectory, config.PureHierarchyFileName);
Console.WriteLine($"Write hierarchies: {pureHierarchyPath}");
var productLineHierarchyRoot = new HierarchyNode {
Level = 1, CategoryName = "HPE PL Root", CategoryID = "Pl"
};
productLineHierarchyRoot.Children.AddRange(plDictionary.Values);
FileHelpers.DeleteFile(FileTypes.Pure_Hierarchy, FileTypes.Pure_Hierarchy, pureHierarchyPath);
using (var stream =
new StreamWriter(File.OpenWrite(pureHierarchyPath))) {
using (var fileWriter = new CsvHierarchyOutputWriter(stream)) {
WriteHierarchy(productLineHierarchyRoot, fileWriter);
}
using (var fileWriter = new CsvHierarchyOutputWriter2(stream)) {
WriteHierarchy2(hierarchyRoot, fileWriter);
}
}
}
public HpIncImportParser(WriterConfiguration configuration)
{
_configuration = configuration;
}
public OrcCompressedBufferFactory(WriterConfiguration configuration)
{
CompressionBlockSize = configuration.BufferSize;
CompressionKind = configuration.Compress.ToCompressionKind();
CompressionStrategy = configuration.CompressionStrategy;
}
public HpEnterpriseImportParser(WriterConfiguration configuration)
{
_configuration = configuration;
}
public CsvHierarchyOutputWriter2(WriterConfiguration configuration)
{
_hierarchyGenerator = new CsvHierarchyGenerator2();
_hierarchyWriter =
new StreamWriter(File.OpenWrite(Path.Combine(configuration.OutputPath, "pure_hierarchy.txt")));
}