public ExpenseClaimService(IDataExtractor emailEmailDataExtractor, IExpenseClaimFactory expenseClaimFactory,
ILogger <IExpenseClaimService> logger)
{
_emailDataExtractor = emailEmailDataExtractor;
_expenseClaimFactory = expenseClaimFactory;
_logger = logger;
}
private static IDataExtractor <T> ExtractProperty <T>(IDataExtractor <T> extractor,
PropertyInfo property) where T : class, new()
{
if (property.PropertyType == typeof(string))
{
return(extractor.WithProperty(ExpressionGenerator <T> .GetStringProperty(property), property.ToColumn()));
}
if (property.PropertyType == typeof(int))
{
return(extractor.WithProperty(ExpressionGenerator <T> .GetIntProperty(property), property.ToColumn()));
}
if (property.PropertyType == typeof(int?))
{
return(extractor.WithProperty(ExpressionGenerator <T> .GetNullableIntProperty(property), property.ToColumn()));
}
if (property.PropertyType == typeof(DateTime))
{
return(extractor.WithProperty(ExpressionGenerator <T> .GetDateTimeProperty(property), property.ToColumn()));
}
if (property.PropertyType == typeof(DateTime?))
{
return(extractor.WithProperty(ExpressionGenerator <T> .GetNullableDateTimeProperty(property), property.ToColumn()));
}
if (property.PropertyType == typeof(bool))
{
return(extractor.WithProperty(ExpressionGenerator <T> .GetBooleanProperty(property), property.ToColumn()));
}
if (property.PropertyType == typeof(bool?))
{
return(extractor.WithProperty(ExpressionGenerator <T> .GetNullableBooleanProperty(property), property.ToColumn()));
}
throw new ArgumentException($"Unknown type {property.PropertyType}");
}
public DataPipeline(IDataExtractor dataExtractor, IDataLoader dataLoader)
{
_rawZone = new RawZone(dataExtractor);
_curatedZone = new CuratedZone(dataLoader);
RunState = DataPipelineRunState.Init;
}
public IDataExtractor <TRawData> CreateFromName(string name)
{
string jsonConfig = GetByName(name);
ConfigSection config = StructuredDataConfig.ParseJsonString(jsonConfig);
IDataExtractor <TRawData> dataExtractor = getDataExtractor(config);
return(dataExtractor);
}
public AdminApprovalHistory(IStopWordFilter stopWordFilter, IDataExtractor dataExtractor)
{
IDbConnectionManager odbcConnectionManager = new OdbcConnectionManager(ConfigurationManager.AppSettings["ConnectionString"]);
IWordsDataSource wordDataSource = new OdbcWordsDataSource(odbcConnectionManager);
Classifier = new BayesianClassifier(wordDataSource);
StopWordFilter = stopWordFilter;
DataExtractor = dataExtractor;
}
public static string Serialize(this IDataExtractor extractor)
{
using (var writer = new StringWriter())
{
var serializer = new XmlSerializer(extractor.GetType());
serializer.Serialize(writer, extractor);
return(writer.ToString());
}
}
public EventPersister(IPersistenceConfiguration configuration,
IDataExtractor dataExtractor,
IErrorDefinitionWriter errorDefinitionWriter,
ITimelineWriter timelineWriter)
{
_configuration = configuration;
_dataExtractor = dataExtractor;
_errorDefinitionWriter = errorDefinitionWriter;
_timelineWriter = timelineWriter;
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="storageManager">Storage Manager</param>
/// <param name="queue">Queue</param>
/// <param name="mapper">Automapper</param>
/// <param name="dataExtractor">Data Extractor</param>
public ScanProcessor(IStorageManager storageManager,
IBackgroundQueue queue,
IMapper mapper,
IDataExtractor dataExtractor)
{
_storageManager = storageManager;
_queue = queue;
_mapper = mapper;
_dataExtractor = dataExtractor;
}
public MRZReaderHandler(
ILogger <MRZReaderHandler> logger,
IDocumentRepository documentRepository,
IOptions <CloudOcrSettings> settings,
IDataExtractor dataExtractor)
{
_logger = logger;
_documentRepository = documentRepository;
PopulateCloudOcrSettingsSettings(settings);
_dataExtractor = dataExtractor;
}
private IEnumerable <string> GetXml(IDataExtractor extractor, int batchSize)
{
_maxCount = int.MaxValue;
while (!extractor.AtEnd)
{
var writer = new XmlDataWriter();
extractor.Write(batchSize, writer);
yield return(writer.ToString());
}
_maxCount = extractor.NumProcessed;
}
//[Inject]
//public TraceSource Trace { get; set; }
public BaseApiRequest(IApi Api, IDataExtractor DataExtractor)
{
api = Api;
dataExtractor = DataExtractor;
requestParams = new Dictionary<string, Dictionary<string, string>>();
objectTypeForMethods = new Dictionary<string, Type>();
batchSizes = new Dictionary<string, int>();
itemsMaxCounts = new Dictionary<string, int>();
requestTypes = new Dictionary<string, ApiRequestType>();
}
/// <summary>
/// Ctor
/// </summary>
/// <param name="extractor">Data extractor</param>
/// <param name="cleanser">Data cleanser</param>
/// <param name="transformer">Data transformation</param>
/// <param name="loader">Data loader</param>
public ETLProcessor(
IDataExtractor <T> extractor,
IDataCleanser <T> cleanser,
IDataTransformer <T, U> transformer,
IDataLoader <U> loader)
{
_extractor = extractor;
_cleanser = cleanser;
_transformer = transformer;
_loader = loader;
}
public VkApiRequest(IApi Api, IDataExtractor DataExtractor)
: base(Api, DataExtractor)
{
objectTypeForMethods.Add("groups.getById", typeof(VkGroup));
objectTypeForMethods.Add("groups.getMembers", typeof(long));
objectTypeForMethods.Add("groups.get", typeof(long));
objectTypeForMethods.Add("users.getSubscriptions", typeof(VkUserSubscriptions));
objectTypeForMethods.Add("users.get", typeof(VkUser));
objectTypeForMethods.Add("wall.get", typeof(VkPost));
objectTypeForMethods.Add("wall.getReposts", typeof(VkPost));
objectTypeForMethods.Add("friends.get", typeof(long));
objectTypeForMethods.Add("wall.getComments", typeof(VkComment));
requestTypes.Add("groups.getById", ApiRequestType.ListObjectsInfo);
requestTypes.Add("groups.getMembers", ApiRequestType.ListForObject);
requestTypes.Add("groups.get", ApiRequestType.ListForObject);
requestTypes.Add("users.getSubscriptions", ApiRequestType.ObjectInfo);
requestTypes.Add("users.get", ApiRequestType.ListObjectsInfo);
requestTypes.Add("wall.get", ApiRequestType.ListForObject);
requestTypes.Add("wall.getReposts", ApiRequestType.ListForObject);
requestTypes.Add("friends.get", ApiRequestType.ListForObject);
requestTypes.Add("wall.getComments", ApiRequestType.ListForObject);
requestParams.Add("groups.getById", new Dictionary<string, string>() {
{ "fields", "members_count" }
});
//requestParams.Add("groups.get",
// new ApiRequestParam(new Dictionary<string, string>() {
// { "filter", "moder" }
// })
//);
requestParams.Add("users.get", new Dictionary<string, string>() {
{ "fields", "education,contacts,nickname, screen_name, sex, bdate, city, country, timezone, photo_50, photo_100, photo_200, photo_max, has_mobile, online" }
});
requestParams.Add("wall.get", new Dictionary<string, string>() {
{ "filter", "all" }
});
requestParams.Add("wall.getComments", new Dictionary<string, string>() {
{ "need_likes", "1" }
});
itemsMaxCounts.Add("groups.getMembers", 1000);
itemsMaxCounts.Add("groups.get", 1000);
itemsMaxCounts.Add("wall.get", 100);
itemsMaxCounts.Add("wall.getReposts", 1000);
itemsMaxCounts.Add("users.getSubscriptions", 200);
itemsMaxCounts.Add("friends.get", Int32.MaxValue);
itemsMaxCounts.Add("wall.getComments", Int32.MaxValue);
batchSizes.Add("users.get", 300);
batchSizes.Add("groups.getById", 300);
}
public static IDataExtractor <T> WithAllProperties <T>(this IDataExtractor <T> extractor)
{
var properties = typeof(T).GetProperties();
foreach (var propertyInfo in properties)
{
var parameter = Expression.Parameter(typeof(T), "x");
var property = Expression.Property(parameter, propertyInfo);
var lambda = Expression.Lambda <Func <T, object> >(property, parameter);
extractor = extractor.WithProperty(lambda);
}
return(extractor);
}
private BodyTreeArray GetStructuredArray(ServiceParameter parameter, object structuredValueCollection,
IDataExtractor dataExtractor)
{
var array = new BodyTreeArray {
Parameter = parameter
};
foreach (object item in (IEnumerable)structuredValueCollection)
{
var treeValue = GetParameterValue <BodyTreeObject>(parameter, item, dataExtractor);
array.Add(treeValue);
}
return(array);
}
private bool TryGetDataExtractor(object value, out IDataExtractor dataExtractor)
{
dataExtractor = null;
if (value == null || TypeExtractorsMap.Count == 0)
{
return(false);
}
var type = value.GetType();
if (TypeExtractorsMap.ContainsKey(type))
{
dataExtractor = TypeExtractorsMap[type];
return(true);
}
return(false);
}
private void btnImportData_Click(object sender, EventArgs e)
{
try
{
IDataExtractor result = null;
using (var dialog = new Dialog.ImportSelectDialog())
{
dialog.Filter = "File / Folders to Import|*.*|Data File|*.csv;*.xslx";
dialog.Multiselect = true;
if (dialog.ShowDialog() == DialogResult.OK)
{
if (dialog.FileName == Utils.GetAppFilePath(AppFileType.ImportExtractor))
{
result = DataExtractorFactory.Deserialize(File.ReadAllText(dialog.FileName));
}
else if (dialog.FilterIndex == 1 || Directory.Exists(dialog.FileName))
{
result = DataExtractorFactory.Get(dialog.FileNames, ImportType.Files);
}
else
{
throw new NotSupportedException();
}
}
}
if (result != null)
{
var mapping = new ImportMapping();
mapping.Extractor = result;
var connSelect = new ConnectionSelection();
connSelect.MultiSelect = false;
connSelect.GoNextAction = () => _wizard.GoToStep(mapping);
_wizard.GoToStep(connSelect);
}
}
catch (Exception ex)
{
Utils.HandleError(ex);
}
}
public PlanGenerator(
IPlanService planService,
IDriverService driverService,
IMapperService mapperService,
IConfigurationProvider <OptimizerSettings> settingsConfigurationProvider,
Drayage.EnhancedOptimization.Services.IPlanGenerator planGenerator,
IDataExtractor dataExtractor,
IEnumerable <IPlanGeneratorInitializer> planGeneratorInitializers,
IReportingService reportingService)
{
_planService = planService;
_driverService = driverService;
_mapperService = mapperService;
_settingsConfigurationProvider = settingsConfigurationProvider;
_planGenerator = planGenerator;
this.dataExtractor = dataExtractor;
_planGeneratorInitializers = planGeneratorInitializers;
_reportingService = reportingService;
}
public static IDataExtractor <TRow> WithAllProperties <TRow>(this IDataExtractor <TRow> extractor)
{
var p = Expression.Parameter(typeof(IDataExtractor <TRow>), "p"); // final lambda parameter
Expression ansBody = p; // start with p => p
var withPropGenericMI = typeof(IDataExtractor <TRow>).GetMethod("WithProperty"); // lookup WithProperty<> generic method
var properties = typeof(TRow).GetProperties();
foreach (var property in properties)
{
var withPropMI = withPropGenericMI.MakeGenericMethod(property.PropertyType); // instantiate generic WithProperty<> to property type
var pxp = Expression.Parameter(typeof(TRow), "x"); // property accessor lambda parameter
var pxb = Expression.PropertyOrField(pxp, property.Name); // property accessor expression x.property
Expression propExpr = Expression.Lambda(pxb, new[] { pxp }); // x => x.property
ansBody = Expression.Call(ansBody, withPropMI, propExpr); // build up p => p.WithProperty(x => x.property)...
}
return((IDataExtractor <TRow>)(Expression.Lambda(ansBody, new[] { p }).Compile().DynamicInvoke(extractor)));
}
private T GetParameterValue <T>(ServiceParameter parameter, object value,
IDataExtractor dataExtractor) where T : BodyTreeNode, new()
{
var innerParameters = parameter.InnerParameters;
var container = innerParameters.Select(p => new ParameterValue {
ParameterInfo = p
}).ToList();
dataExtractor.FillDataValues(container, value);
var result = new T {
Parameter = parameter
};
foreach (ParameterValue parameterValue in container)
{
var parameterInfo = innerParameters.Single(p => p.UId == parameterValue.ParameterInfo.UId);
var nodeValue = GetParameterValue(parameterInfo, parameterValue.Value);
result.Add(nodeValue);
}
return(result);
}
public void ExtractDataWith(IDataExtractor dataExtractor, CancellationToken token = default)
{
OnOperationEvent(dataExtractor.Name, TaskStatus.Created);
using var bulkInsert = _documentStore.BulkInsert(_databaseName, token);
OnOperationEvent(dataExtractor.Name, TaskStatus.Running);
try
{
foreach (var dataItem in dataExtractor.ExtractData(_runtime, message => OnOperationEvent(dataExtractor.Name, TaskStatus.Running, message)))
{
var newId = bulkInsert.Store(dataItem);
OnOperationEvent(dataExtractor.Name, TaskStatus.Running, $"Imported data item with Id = {newId} (data item type = {dataExtractor.TypeName})");
}
}
catch (System.Exception e)
{
OnOperationEvent(dataExtractor.Name, TaskStatus.Faulted, $"Failed data extraction task. Message: {e.Message}, Stack trace: {e.StackTrace}. Retrying one-by-one");
try
{
foreach (var dataItem in dataExtractor.ExtractData(_runtime, message => OnOperationEvent(dataExtractor.Name, TaskStatus.Running, message)))
{
using (var session = _documentStore.OpenSession(_databaseName))
{
session.Store(dataItem);
session.SaveChanges();
OnOperationEvent(dataExtractor.Name, TaskStatus.Running, $"Imported data item(data item type = {dataExtractor.TypeName})");
}
}
}
catch (System.Exception inner)
{
OnOperationEvent(dataExtractor.Name, TaskStatus.Faulted, $"Failed data extraction task. Message: {inner.Message}, Stack trace: {inner.StackTrace}");
throw;
}
}
OnOperationEvent(dataExtractor.Name, TaskStatus.RanToCompletion);
}
/// <summary>
/// Gets list of specified type <typeparamref name="T"/>.
/// </summary>
/// <typeparam name="T">Type to instantiate.</typeparam>
/// <param name="query">SQL query.</param>
/// <param name="extractor">Data extractor instance used to instantiate specified type.</param>
/// <returns>List of instances of type <typeparamref name="T"/>.</returns>
protected IEnumerable <T> GetList <T>(string query, IDataExtractor <T> extractor)
{
return(GetList <T>(query, null, extractor));
}
/// <summary>
/// Execute reader and create list of provided type using IDataExtractor interface.
/// </summary>
/// <typeparam name="T">Type of object to create.</typeparam>
/// <typeparam name="P">Extractor arguments type.</typeparam>
/// <param name="query">Sql Query.</param>
/// <param name="args">Sql Parameters.</param>
/// <param name="extractor">IDataExtractor used to map object instance from reader.</param>
/// <param name="extractorArgs">Arguments to pass to extract method.</param>
/// <returns>List of provided object type.</returns>
public IEnumerable <T> Select <T, P>(string query, IEnumerable <QueryParameter> args, IDataExtractor <T, P> extractor, P extractorArgs)
{
return(SelectWith(query, args, (reader) =>
{
return extractor.Extract(reader, extractorArgs);
}));
}
/// <summary>
/// Execute reader and create list of provided type using IRowMapper interface.
/// </summary>
/// <typeparam name="T">Type of object to create.</typeparam>
/// <param name="query">Sql Query.</param>
/// <param name="extractor">IDataExtractor used to map object instance from reader.</param>
/// <param name="isStoredProcedure">indicating if query type is stored procedure.</param>
/// <returns>List of provided object type.</returns>
public IEnumerable <T> Select <T>(string query, IDataExtractor <T> extractor)
{
return(Select <T>(query, null, extractor));
}
public void ProcessImport(IDataExtractor extractor, string xslt, int batchSize)
{
_extractor = extractor;
Exception error = null;
_completeDate = DateTime.MinValue;
_globallyProcessed = 0;
try
{
extractor.Reset();
using (_cts = new CancellationTokenSource())
{
var opts = new ParallelOptions();
opts.CancellationToken = _cts.Token;
Parallel.ForEach(GetXml(extractor, batchSize), opts, xml =>
{
int locallyProcessed = 0;
var doc = new XmlDocument();
doc.LoadXml(ArasXsltExtensions.Transform(xslt, xml, _conn.ExternalConnection));
IList<XmlElement> levelElements = new XmlElement[] { doc.DocumentElement };
while (!levelElements.Any(e => e.LocalName == "AML" || e.LocalName == "SQL"
|| e.LocalName == "sql" || e.LocalName == "Item"))
levelElements = levelElements.Elements(e => true).ToList();
var isError = false;
for (var i = 0; i < levelElements.Count; i++)
{
Action<int, int> progressReporter = (curr, count) =>
{
var progress = (int)(((double)(count * i + curr) / (count * levelElements.Count)) * batchSize);
if (progress > locallyProcessed) Interlocked.Add(ref _globallyProcessed, progress - locallyProcessed);
locallyProcessed = progress;
SignalProgress(batchSize);
};
XmlNode result;
switch (levelElements[i].LocalName)
{
case "AML":
result = _conn.CallAction("ApplyAML", levelElements[i], progressReporter);
break;
case "SQL":
case "sql":
result = _conn.CallAction("ApplySQL", levelElements[i], progressReporter);
break;
case "Item":
result = _conn.CallAction("ApplyItem", levelElements[i], progressReporter);
break;
default:
result = null;
break;
}
if (_conn.GetError(result) != null)
{
_errors.Add(new ErrorEntry() { Error = result.OuterXml, Query = xml });
isError = true;
break;
}
}
if (isError) Interlocked.Increment(ref _errorBlocks);
Interlocked.Add(ref _globallyProcessed, batchSize - locallyProcessed);
SignalProgress(batchSize);
opts.CancellationToken.ThrowIfCancellationRequested();
});
}
}
catch (OperationCanceledException) { }
catch (Exception ex)
{
error = ex;
}
finally
{
_cts = null;
}
_completeDate = DateTime.Now;
OnActionComplete(new ActionCompleteEventArgs() { Exception = error });
}
private IEnumerable<string> GetXml(IDataExtractor extractor, int batchSize)
{
_maxCount = int.MaxValue;
while (!extractor.AtEnd)
{
var writer = new XmlDataWriter();
extractor.Write(batchSize, writer);
yield return writer.ToString();
}
_maxCount = extractor.NumProcessed;
}
public MaterialDataExtractorTests()
{
materialDataExtractor = new MaterialDataExtractor();
}
public StructuredValue(IDataExtractor dataExtractor, object value)
{
DataExtractor = dataExtractor;
Value = value;
}
public HtmlDataSource(IDataExtractor <TRawData> extractor, IHtmlSource htmlSource)
{
this.extractor = extractor;
this.htmlSource = htmlSource;
}
public static ICollectionPropertyConfiguration <T> WithAllProperties <T>(
this IDataExtractor <T> extractor) where T : class, new() =>
typeof(T)
.GetProperties()
.Aggregate(extractor, ExtractProperty) as ICollectionPropertyConfiguration <T>;
/// <summary>
/// Execute reader and create list of provided type using IDataExtractor interface.
/// </summary>
/// <typeparam name="T">Type of object to create.</typeparam>
/// <typeparam name="P">Extractor arguments type.</typeparam>
/// <param name="query">Sql Query.</param>
/// <param name="args">Sql Parameters.</param>
/// <param name="extractor">IDataExtractor used to map object instance from reader.</param>
/// <param name="extractorArgs">Arguments to pass to extract method.</param>
/// <returns>List of provided object type.</returns>
public IEnumerable <T> Select <T, P>(string query, IEnumerable <SQLiteQueryParameter> args, IDataExtractor <T, P> extractor, P extractorArgs)
{
using (var connection = new SQLiteConnection(ConnectionString))
{
connection.Open();
foreach (var item in SelectInternal(query, args, extractor, extractorArgs, connection))
{
yield return(item);
}
}
}
private IEnumerable <T> SelectInternal <T, P>(string query, IEnumerable <SQLiteQueryParameter> args, IDataExtractor <T, P> extractor, P extractorArgs, SQLiteConnection connection)
{
using (SQLiteCommand command = connection.CreateCommand())
{
command.CommandText = query;
PrepareCommandParameters(command, args);
IEnumerable <T> result;
using (SQLiteDataReader reader = command.ExecuteReader())
{
result = extractor.Extract(reader, extractorArgs);
reader.Close();
}
return(result);
}
}
/// <summary>
/// Gets list of specified type <typeparamref name="T"/>.
/// </summary>
/// <typeparam name="T">Type to instantiate.</typeparam>
/// <param name="query">SQL query.</param>
/// <param name="args">SQL query arguments.</param>
/// <param name="extractor">Data extractor instance used to instantiate specified type.</param>
/// <returns>List of instances of type <typeparamref name="T"/>.</returns>
protected IEnumerable <T> GetList <T>(string query, IEnumerable <QueryParameter> args, IDataExtractor <T> extractor)
{
using (var connector = new Connector())
return(connector.Select(query, args, extractor));
}
/// <summary>
/// Execute reader and create list of provided type using IDataExtractor interface.
/// </summary>
/// <typeparam name="T">Type of object to create.</typeparam>
/// <typeparam name="P">Extractor arguments type.</typeparam>
/// <param name="query">Sql Query.</param>
/// <param name="args">Sql Parameters.</param>
/// <param name="extractor">IDataExtractor used to map object instance from reader.</param>
/// <param name="connection">Database connection instance to use.</param>
/// <param name="extractorArgs">Arguments to pass to extract method.</param>
/// <returns>List of provided object type.</returns>
public IEnumerable <T> Select <T, P>(string query, IEnumerable <SQLiteQueryParameter> args, IDataExtractor <T, P> extractor, P extractorArgs, SQLiteConnection connection)
{
foreach (var item in SelectInternal(query, args, extractor, extractorArgs, connection))
{
yield return(item);
}
}
/// <summary>
/// Execute reader and create list of provided type using IRowMapper interface.
/// </summary>
/// <typeparam name="T">Type of object to create.</typeparam>
/// <param name="query">Sql Query.</param>
/// <param name="args">Sql Parameters.</param>
/// <param name="extractor">IDataExtractor used to map object instance from reader.</param>
/// <param name="isStoredProcedure">indicating if query type is stored procedure.</param>
/// <returns>List of provided object type.</returns>
public IEnumerable <T> Select <T>(string query, IEnumerable <QueryParameter> args, IDataExtractor <T> extractor)
{
using (SqlConnection Connection = new SqlConnection(ConnectionString))
{
Connection.Open();
using (SqlCommand command = Connection.CreateCommand())
{
command.CommandText = query;
PrepareCommandParameters(command, args);
IEnumerable <T> result;
using (SqlDataReader reader = command.ExecuteReader())
{
result = extractor.Extract(reader);
reader.Close();
}
return(result);
}
}
}
