public override Result Execute(InputDataProvider data, MergerApplicationOptions options)
{
try
{
var configuration = GetConfiguration(options);
var files = data.GetData()
.Where(x => x.IsValid())
.Select(x => CreateWorkItem(x, configuration))
.Cast <UrlWorkUnit>()
.Select(x => x.Data)
.ToArray();
var outputFilePath = Path.Combine(
HTMLOperationContextScope.Context.OutputFolder,
HTMLOperationContextScope.Context.Service.FileNameProvider
.GetNextFileName(Application.DEFAULT_NAME_FOR_MERGED_SET_OF_FILES, options.OutputFormat.ToFileExtension()));
SvgMergeHelper.GetHelper(options)
.Merge(files, outputFilePath);
return(Result.FromFiles(outputFilePath));
}
catch (Exception e)
{
return(Result.Error(e));
}
}
protected virtual Result ConvertFiles(InputDataProvider data, ConversionApplicationOptions options)
{
try
{
foreach (var dataSource in data.GetData().Where(x => x.IsValid()))
{
var outputFilePath = Path.Combine(
HTMLOperationContextScope.Context.OutputFolder,
HTMLOperationContextScope.Context.Service.FileNameProvider
.GetNextFileName(dataSource.Name, options.OutputFormat.ToFileExtension()));
using (var configuration = GetConfiguration(options))
using (var wu = CreateWorkItem(dataSource, configuration))
{
Render(wu, options, outputFilePath);
}
}
return(Result.FromDirectory(HTMLOperationContextScope.Context.OutputFolder));
}
catch (Exception e)
{
return(Result.Error(e));
}
}
private static async Task HandleMenuAsync()
{
var configuration = GetConfiguration(settingsFileName);
var options = GetDbContextOptions(configuration);
var dbContext = new ScrapperContext(options);
IDataHandler <Person> handler = new PersonDataHandler <Person>(dbContext);
InputDataProvider inputData = await InputDataProviderFactory.GetInputDataProviderForPerson();
switch (Console.ReadLine())
{
case "1":
break;
case "2":
string PhoneTemplate = configuration[nameof(PhoneTemplate)];
inputData = await InputDataProviderFactory.GetInputDataProviderForPersonWithSpecialDomen(PhoneTemplate);
break;
case "3":
string SiteTemplate = configuration[nameof(SiteTemplate)];
handler = new TelegramDataHandler <Person>(dbContext);
inputData = await InputDataProviderFactory.GetInputDataProviderForPersonWithSpecialDomen(SiteTemplate);
break;
case "0":
return;
}
Console.WriteLine();
Console.WriteLine("Start scrapping process...\n");
await RunScrappingAsync(handler, inputData);
}
protected virtual Result MergeFiles(InputDataProvider data, ConversionApplicationOptions options)
{
using (var configuration = GetConfiguration(options))
{
var units = Enumerable.Empty <RenderingWorkUnit>();
try
{
units = data.GetData()
.Where(x => x.IsValid())
.Select(x => CreateWorkItem(x, configuration))
.ToArray();
var outputFilePath = Path.Combine(
HTMLOperationContextScope.Context.OutputFolder,
HTMLOperationContextScope.Context.Service.FileNameProvider
.GetNextFileName(ConversionApplication.DEFAULT_NAME_FOR_MERGED_SET_OF_FILES,
options.OutputFormat.ToFileExtension()));
Render(units.Select(x => x).ToArray(), options, outputFilePath);
return(Result.FromFiles(outputFilePath));
}
catch (Exception e)
{
return(Result.Error(e));
}
finally
{
units.ForEach(x => x?.Dispose());
}
}
}
private static async Task <InputDataProvider> GetInputDataProvider()
{
var inputData = new InputDataProvider(new FileReader());
await inputData.FillData();
return(inputData);
}
public void GetData_Should_Be_Able_To_Read_Json_String_And_Provide_data(string jsonString)
{
var inputDataProviderProvider = new InputDataProvider();
var data = inputDataProviderProvider.GetData(jsonString);
Assert.NotNull(data);
Assert.IsType <InputModel>(data);
}
public override Result Execute(InputDataProvider data, ConversionApplicationOptions options)
{
if (options.Merge && data.GetData().Count(x => x.IsValid()) > 1)
{
return(MergeFiles(data, options));
}
return(ConvertFiles(data, options));
}
public virtual Result Execute(InputDataProvider data, T options)
{
var handler = TryGetHandler(options);
if (handler == MatchResult.True)
{
return(((MatchResult <ApplicationExecutionHandler>)handler).Data.Execute(data, options));
}
return(Result.Error(handler.ErrorCode));
}
public void GetDataList_Should_Be_Able_To_Read_Json_String_And_Provide_data(string jsonString)
{
var inputDataProviderProvider = new InputDataProvider();
var dataCollection = inputDataProviderProvider.GetDataList(jsonString);
Assert.NotNull(dataCollection);
foreach (var item in dataCollection)
{
Assert.NotNull(item);
Assert.IsType <InputModel>(item);
}
}
private static async Task RunScrappingAsync(IDataHandler <Person> handler, InputDataProvider inputData)
{
var logRepository = LogManager.GetRepository(Assembly.GetEntryAssembly());
XmlConfigurator.Configure(logRepository, new FileInfo(configLoggingFileName));
ILog log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
IScrapper <Person> scrapper = new Scrapper <Person>();
IScrapersManager scrapperManager = new ScrapersManager(inputData, handler, scrapper, log);
scrapperManager.Notify += ScrapperManagerNotify;
await scrapperManager.ScrapDataAsync();
}
public void GetInputSet_Test()
{
// Arrange
string separator = ConfigurationManager.AppSettings["Separator"];
string lines = string.Format(
@"5{0} indigobreadcrumb1{0} currentGoogleBreadCrumb1{0} newGoogleBreadCrumb1
7{0} indigobreadcrumb2{0} currentGoogleBreadCrumb2{0} newGoogleBreadCrumb2"
, separator);
var inputDataProvider = new InputDataProvider(() => new StringReader(lines),
CommonLibrary.SetupLogger().Object);
// Act
var result = inputDataProvider.GetInputSet();
// Assert
Assert.AreEqual(2, result.Count);
var record1 = result[0];
Assert.AreEqual(5, record1.IndigoCategoryId);
Assert.AreEqual("indigobreadcrumb1", record1.IndigoBreadcrumb);
Assert.AreEqual("currentGoogleBreadCrumb1", record1.CurrentGoogleBreadcrumb);
Assert.AreEqual("newGoogleBreadCrumb1", record1.NewGoogleBreadcrumb);
var record2 = result[1];
Assert.AreEqual(7, record2.IndigoCategoryId);
Assert.AreEqual("indigobreadcrumb2", record2.IndigoBreadcrumb);
Assert.AreEqual("currentGoogleBreadCrumb2", record2.CurrentGoogleBreadcrumb);
Assert.AreEqual("newGoogleBreadCrumb2", record2.NewGoogleBreadcrumb);
}
public override Result Execute(InputDataProvider data, ApplicationOptions options)
{
return(Execute(data, (T)options));
}
public abstract Result Execute(InputDataProvider data, TOptions options);
public void TestMethod1()
{
var provider = new InputDataProvider("mosdb.xlsx");
provider.GetInputData();
}
