public DataWriter GetDataWriter() { DataWriter writer = DataWriterOption.GetWriter(); if (writer is FileDataWriter fileDataWriter) { fileDataWriter.SetParentDirectory(ProcessedDataOutput.ToString()); } return(writer); }
/// <summary> /// Entrypoint of this program which parses the following arguments from the CLI: /// <list type="bullet"> /// <item><code>--raw-dir</code>: The path to output directory to put the raw downloaded files in</item> /// <item><code>--processed-dir</code>: The path to the directory to put the processed files in</item> /// <item><code>--fetch-url</code>: The url to download to use as raw data</item> /// </list> /// </summary> /// <param name="args">The arguments passed from the command line</param> public static void Main(string[] args) { List <string> arguments = new List <string>(); arguments.AddRange(args); // Print the help section when requested string[] helpLong = FindOption(arguments, "--help", 0); string[] helpShort = FindOption(arguments, "-h", 0); if (helpLong != null || helpShort != null) { PrintHelp(); return; } // First parse the interval, if available string[] intervalOption = FindOption(arguments, "--interval"); IntervalRange[] intervalRanges = null; if (intervalOption != null) { intervalRanges = IntervalRange.GetIntervals(intervalOption[0], Console.Error); if (intervalRanges == null) { Environment.Exit(1); } } DataWriterOption selectedOption = null; DataWriterOption[] availableOptions = { new DbDataWriterOption(), new FileDataWriterOption() }; foreach (DataWriterOption option in availableOptions) { if (option.isAvailable(arguments)) { if (!option.validArguments()) { Environment.Exit(1); } selectedOption = option; break; } } // Parse the arguments Options options = new Options { RawDataOutput = GetDirectory(arguments, "--raw-dir"), ProcessedDataOutput = GetDirectory(arguments, "--processed-dir"), ReaderOption = GetEnumOption <ReaderOption>(arguments, "--reader"), DataWriterOption = selectedOption, RemoveRawFiles = FindOption(arguments, "--remove-raw", 0) != null }; // Parse the Fetch url string[] urlOption = FindOption(arguments, "--fetch-url"); if (urlOption != null) { Uri fetchUrl; if (!Uri.TryCreate(urlOption[0], UriKind.Absolute, out fetchUrl) || (fetchUrl.Scheme != Uri.UriSchemeHttp && fetchUrl.Scheme != Uri.UriSchemeHttps)) { Console.Error.WriteLine($"Invalid fetch url {urlOption[0]}"); Environment.Exit(1); } options.FetchUri = fetchUrl; } BaseAction action = new DownloadAndProcessAction(options); if (intervalRanges == null) { action.Execute(); } else { IntervalScheduler intervalScheduler = new IntervalScheduler(action, intervalRanges); intervalScheduler.RunScheduler(CancellationToken.None); } }