public DataWriter GetDataWriter()
{
DataWriter writer = DataWriterOption.GetWriter();
if (writer is FileDataWriter fileDataWriter)
{
fileDataWriter.SetParentDirectory(ProcessedDataOutput.ToString());
}
return(writer);
}
/// <summary>
/// Entrypoint of this program which parses the following arguments from the CLI:
/// <list type="bullet">
/// <item><code>--raw-dir</code>: The path to output directory to put the raw downloaded files in</item>
/// <item><code>--processed-dir</code>: The path to the directory to put the processed files in</item>
/// <item><code>--fetch-url</code>: The url to download to use as raw data</item>
/// </list>
/// </summary>
/// <param name="args">The arguments passed from the command line</param>
public static void Main(string[] args)
{
List <string> arguments = new List <string>();
arguments.AddRange(args);
// Print the help section when requested
string[] helpLong = FindOption(arguments, "--help", 0);
string[] helpShort = FindOption(arguments, "-h", 0);
if (helpLong != null || helpShort != null)
{
PrintHelp();
return;
}
// First parse the interval, if available
string[] intervalOption = FindOption(arguments, "--interval");
IntervalRange[] intervalRanges = null;
if (intervalOption != null)
{
intervalRanges = IntervalRange.GetIntervals(intervalOption[0], Console.Error);
if (intervalRanges == null)
{
Environment.Exit(1);
}
}
DataWriterOption selectedOption = null;
DataWriterOption[] availableOptions =
{
new DbDataWriterOption(), new FileDataWriterOption()
};
foreach (DataWriterOption option in availableOptions)
{
if (option.isAvailable(arguments))
{
if (!option.validArguments())
{
Environment.Exit(1);
}
selectedOption = option;
break;
}
}
// Parse the arguments
Options options = new Options
{
RawDataOutput = GetDirectory(arguments, "--raw-dir"),
ProcessedDataOutput = GetDirectory(arguments, "--processed-dir"),
ReaderOption = GetEnumOption <ReaderOption>(arguments, "--reader"),
DataWriterOption = selectedOption,
RemoveRawFiles = FindOption(arguments, "--remove-raw", 0) != null
};
// Parse the Fetch url
string[] urlOption = FindOption(arguments, "--fetch-url");
if (urlOption != null)
{
Uri fetchUrl;
if (!Uri.TryCreate(urlOption[0], UriKind.Absolute, out fetchUrl) ||
(fetchUrl.Scheme != Uri.UriSchemeHttp && fetchUrl.Scheme != Uri.UriSchemeHttps))
{
Console.Error.WriteLine($"Invalid fetch url {urlOption[0]}");
Environment.Exit(1);
}
options.FetchUri = fetchUrl;
}
BaseAction action = new DownloadAndProcessAction(options);
if (intervalRanges == null)
{
action.Execute();
}
else
{
IntervalScheduler intervalScheduler = new IntervalScheduler(action, intervalRanges);
intervalScheduler.RunScheduler(CancellationToken.None);
}
}