public override void Register(IExtractor obj)
{
Feeder reg = new Feeder();
BlockList_de feed = new BlockList_de();
reg.FeedList.Add(feed);
}
public void Init()
{
_fileOps = A.Fake <IFileOps>();
_extractor = A.Fake <IExtractor <FlexCelOrderLineDto> >();
_validator = new OrderLineSourcePathValidator(_fileOps);
_factory = new SourcePathFactory <FlexCelOrderLineDto>(_fileOps, _extractor);
}
/// <summary>
/// Instancia um novo objeto da classe Observer.
/// </summary>
public Observer(GestureManager manager, IExtractor extractor, IFilter filter)
{
GestureManager = manager;
this.filter = filter;
this.extractor = extractor;
observations = new List <Feature>();
}
public async Task <ChapterInfo> Read(LinkInfo Uri)
{
switch (Uri.Type)
{
case LinkType.OfflineZip:
_extractor = App.DiContainer.Resolve <IExtractor>("zipExtractor");
// _extractor = DIContainer.GetModule<ZipExtractor>();
break;
case LinkType.OfflineFolder:
_extractor = App.DiContainer.Resolve <IExtractor>("folderExtractor");
break;
case LinkType.Online:
_extractor = App.DiContainer.Resolve <IExtractor>("zipExtractor");
_downloader = App.DiContainer.Resolve <IDownloader>();
var file = await _downloader.Download(Uri.Path);
_info.Pages = CreatePageInfos(await _extractor.Extract(file.Path));
return(_info);
default:
throw new ArgumentOutOfRangeException();
}
var images = await _extractor.Extract(Uri.Path);
_info.Pages = CreatePageInfos(images);
return(_info);
}
public override void Register(IExtractor obj)
{
Feeder reg = new Feeder();
SSLIpBlackList feed = new SSLIpBlackList();
reg.FeedList.Add(feed);
}
public void Init()
{
var config = TestHelper.GetFakeConfig().Object;
_extractor = new MorphologyExtractor(config);
_quantizer = new MorphologyQuantizer(config);
}
public JapaneseDateParserConfiguration(JapaneseDateTimeParserConfiguration configuration)
{
config = configuration;
integerExtractor = new IntegerExtractor();
durationExtractor = new JapaneseDurationExtractorConfiguration();
numberParser = new BaseCJKNumberParser(new JapaneseNumberParserConfiguration());
}
/// <summary>
/// Create a new context, one per source file/assembly.
/// </summary>
/// <param name="e">The extractor.</param>
/// <param name="c">The Roslyn compilation.</param>
/// <param name="extractedEntity">Name of the source/dll file.</param>
/// <param name="scope">Defines which symbols are included in the trap file (e.g. AssemblyScope or SourceScope)</param>
public Context(IExtractor e, Compilation c, TrapWriter trapWriter, IExtractionScope scope)
{
Extractor = e;
Compilation = c;
Scope = scope;
TrapWriter = trapWriter;
}
public static IExtractor LoadExtractor(string ddlPath)
{
string assemblyPath = System.IO.Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) + "\\" + ddlPath;
Assembly assembly = Assembly.LoadFile(assemblyPath);
Type[] types = assembly.GetTypes();
Type pluginType = null;
foreach (Type t in assembly.GetTypes())
{
if (typeof(IExtractor).IsAssignableFrom(t))
{
pluginType = t;
break;
}
}
if (pluginType == null)
{
throw new Exception("Plugin in path '" + assemblyPath + "' not a valid extractor plugin");
}
IExtractor result = (IExtractor)Activator.CreateInstance(pluginType);
return(result);
}
public void AddCodec <TCarrier>(IFormat <TCarrier> format, IInjector injector, IExtractor extractor)
{
var formatString = format.ToString();
_injectors[formatString] = injector;
_extractors[formatString] = extractor;
}
public void Edit(IExtractor extractor)
{
Debug.Assert(extractor != null, "Extractor must be supplied to allow editing.");
var window = new AddEditExtractorWindow();
var data = new AddEditExtractor(window, true);
window.DataContext = data;
window.Owner = Application.Current.MainWindow;
data.Name = extractor.Name;
data.Field = extractor.Field;
data.Pattern = extractor.Pattern;
data.Mode = extractor.Mode;
bool?dialogResult = window.ShowDialog();
if (dialogResult != null && (bool)dialogResult)
{
extractor.Name = data.Name;
extractor.Pattern = data.Pattern;
extractor.Mode = data.Mode;
extractor.Field = data.Field;
}
}
private static bool TryExtract <C>(C carrier, IExtractor <C> extractor, out Trace trace)
{
ITraceContext traceContext = default(SpanState);
traceContext = extractor.Extract(carrier);
return(TryCreateTraceFromTraceContext(traceContext, out trace));
}
internal ExtraFieldExtractor(IExtractor <C> underlyingExtractor, Getter <C, K> getter,
IDictionary <string, K> names)
{
_underlyingExtractor = underlyingExtractor;
_getter = getter;
_names = names;
}
public async Task Process(byte[] processMessage, IPlatformExit platformExit)
{
KernelMessage kernelMessage = JsonConvert.DeserializeObject <KernelMessage>(Encoding.UTF8.GetString(processMessage));
ScrapySource scrapySource = await coreCache.RestoreAsync <ScrapySource>(PrefixConst.SOURCE_META + kernelMessage.JobId);
var sourceType = scrapySource.Source.Type;
IExtractor extractor = extractorManager.GetExtrator(sourceType);
await extractor.ExtractTarget(scrapySource.Source.Parameters.ToString(), scrapySource.SaveTo);
string transformJobIdsKey = PrefixConst.SOURCE_TRANSFOR_MAP + kernelMessage.JobId;
TaskingManager taskingManager = new TaskingManager();
List <string> jobIds = await coreCache.RestoreAsync <List <string> >(transformJobIdsKey);
if (jobIds != null)
{
foreach (var x in jobIds)
{
KernelMessage transforMsg = new KernelMessage()
{
JobId = x,
MessageId = kernelMessage.MessageId,
MessageName = kernelMessage.MessageName
};
taskingManager.AddTask(platformExit.OutRandom(transforMsg));
logger.Debug(JsonConvert.SerializeObject(transforMsg));
}
}
await taskingManager.WhenAll();
}
public override void Register(IExtractor obj)
{
Feeder reg = new Feeder();
Ipset feed = new Ipset();
reg.FeedList.Add(feed);
}
//protected override void FinishedNotify()
//{
//}
protected override void SpecificProcessing()
{
IExtractor <Image> extractor = ExtractorFactory.CreateExtractor("datetime");
foreach (var item in _filesInfo)
{
try
{
using (Image img = Image.FromFile(item))
{
string creationTime = extractor.Extract(img, 0x9003).Replace("\0", string.Empty).Replace('/', '_').Replace(':', '_');// GetImageDate(img);
File.Copy(item, _destinationPath.FullName + "\\" + Path.GetFileNameWithoutExtension(item) + "_"
+ (string.IsNullOrWhiteSpace(creationTime) ? (File.GetCreationTime(item).ToString().Replace(':', '_').Replace('/', '_')) : creationTime)
+ Path.GetExtension(item), true);
}
}
catch (OutOfMemoryException ex)
{
string msg = ex.Message;
}
catch (FileNotFoundException ex)
{
string msg = ex.Message;
}
catch (ArgumentException ex)
{
string msg = ex.Message;
}
//Thread.Sleep();
}
Console.WriteLine();
Console.WriteLine("RENAME DATETIME FINISHED!!!!");
Console.WriteLine();
}
public ExpenseService(IExtractor extractor, IValidator validators, IElementValidator missingTotalValidator, IMapper mapper)
{
_extractor = extractor;
_validators = validators;
_missingTotalValidator = missingTotalValidator;
_mapper = mapper;
}
public TracingMiddleware(RequestDelegate next, IOptions <TracingOptions> options)
{
_next = next;
_extractor = Propagations.B3String.Extractor <IHeaderDictionary>((carrier, key) => carrier[key]);
_serviceName = options.Value.ServiceName;
}
public static void Write(string lang, IExtractor extractor, string source, IEnumerable <ExtractResult> results,
[CallerFilePath] string callerFilePath = "", [CallerMemberName] string callerMemberName = "")
{
var modelStr = GetName(extractor);
Write(lang, modelStr, null, source, results.Select(o => new TestExtractorResult(o)), callerFilePath, callerMemberName);
}
protected void UpdateCoreInfos()
{
if (!TryCreateDirectory(_infoDirectory))
{
return;
}
if (!TryCreateAbsoluteUrl(_baseUrl, _infoUrl, out Uri uri))
{
ServiceRegistration.Get <ILogger>().Error("CoreHandler: Unable to create absolute core info url from settings, base url: '{0}', info url: '{1}'", _baseUrl, _infoUrl);
return;
}
try
{
byte[] data = _downloader.DownloadDataAsync(uri.AbsoluteUri).Result;
if (data == null || data.Length == 0)
{
ServiceRegistration.Get <ILogger>().Error("CoreInfoHandler: Failed to download core infos from '{0}', response was null or empty", uri.AbsoluteUri);
return;
}
using (Stream stream = new MemoryStream(data))
using (IExtractor extractor = ExtractorFactory.Create(uri.AbsoluteUri, stream))
extractor.ExtractAll(_infoDirectory);
}
catch (Exception ex)
{
ServiceRegistration.Get <ILogger>().Error("CoreInfoHandler: Exception updating core infos", ex);
}
}
public static IAppBuilder UseZipkinTracer(
this IAppBuilder appBuilder,
string serviceName,
IExtractor <IHeaderDictionary> traceExtractor,
Func <IOwinContext, string> getRpc = null,
Func <PathString, bool> routeFilter = null)
=> appBuilder.Use <ZipkinMiddleware>(serviceName, traceExtractor, getRpc, routeFilter);
private static void ExtractAssets(IExtractor extractor, string outDir, string[] files)
{
ExtractAssets(new List <IExtractor>()
{
extractor
}, outDir, files);
}
//private void AddJob(Job<SourceType, DestinationType> job)
//{
// this.Jobs.Add(job);
//}
private Job <SourceType, DestinationType> CreateJob(
IExtractor <SourceType> extractor,
ITransformer <SourceType, DestinationType> transformer,
ILoader <DestinationType> loader
)
{
return(new Job <SourceType, DestinationType>(extractor, transformer, loader));
}
public ChineseDateParserConfiguration(ChineseDateTimeParserConfiguration configuration)
{
config = configuration;
integerExtractor = new IntegerExtractor();
ordinalExtractor = new OrdinalExtractor();
durationExtractor = new ChineseDurationExtractorConfiguration();
numberParser = new BaseCJKNumberParser(new ChineseNumberParserConfiguration());
}
/// <summary>
/// <para>
/// Creates a Processor with only one loader. This will
/// create only one job.
/// </para>
/// </summary>
/// <param name="id">ID of this processor.</param>
/// <param name="extractor">Data extractor.</param>
/// <param name="transformer">Data transformer.</param>
/// <param name="loader">Data loader.</param>
public Processor(
string id,
IExtractor <SourceType> extractor,
ITransformer <SourceType, DestinationType> transformer,
ILoader <DestinationType> loader
) : this(id, extractor, transformer, new List <ILoader <DestinationType> >() { loader })
{
}
/// <summary>
/// Create a new context, one per source file/assembly.
/// </summary>
/// <param name="e">The extractor.</param>
/// <param name="c">The Roslyn compilation.</param>
/// <param name="extractedEntity">Name of the source/dll file.</param>
/// <param name="scope">Defines which symbols are included in the trap file (e.g. AssemblyScope or SourceScope)</param>
/// <param name="addAssemblyTrapPrefix">Whether to add assembly prefixes to TRAP labels.</param>
public Context(IExtractor e, Compilation c, TrapWriter trapWriter, IExtractionScope scope, bool addAssemblyTrapPrefix)
{
Extractor = e;
Compilation = c;
this.scope = scope;
TrapWriter = trapWriter;
ShouldAddAssemblyTrapPrefix = addAssemblyTrapPrefix;
}
public void Register <T>(IExtractor <T> extractor)
{
if (extractor == null)
{
throw new ArgumentNullException(nameof(extractor));
}
extractors.Add(typeof(T), extractor);
}
public DateParser(IFullDateTimeParserConfiguration configuration)
{
config = configuration;
integerExtractor = new IntegerExtractor();
ordinalExtractor = new OrdinalExtractor();
durationExtractor = new DurationExtractorChs();
numberParser = new ChineseNumberParser(new ChineseNumberParserConfiguration());
}
public void GetExtractorTest()
{
ExtractorManager extractorManager = new ExtractorManager(injectionProvider);
IExtractor extractor = extractorManager.GetExtrator("Http");
Assert.NotNull(extractor);
Assert.Equal(typeof(HttpExtractor), extractor.GetType());
}
public ZipkinMiddleware(
OwinMiddleware next,
string serviceName,
IExtractor <IHeaderDictionary> traceExtractor) : base(next)
{
this.serviceName = serviceName;
this.traceExtractor = traceExtractor;
}
public ProducerUnit(
IInputQueue<string> inputQueue,
IOutputQueue<string> outputQueue,
IExtractor<string, string> extractor)
{
InputQueue = inputQueue;
OutputQueue = outputQueue;
Extractor = extractor;
}
public Worker(IFileSystem fs, IExtractor ext,
ILayerParser lp, IRelatedDataSaver rds, ISuitableCommentsPicker scp)
{
_fs = fs;
_ext = ext;
_lp = lp;
_rds = rds;
_scp = scp;
}
public void Reset(IExtractor extractor, Dictionary<string, IImageCreator> imageCreators, string[] imageNames, int index)
{
this.extractor = extractor;
this.imageCreators = imageCreators;
this.imageNames = imageNames;
if (0<=index && index<imageNames.Length) {
currentIndex = index;
}
}
private bool ValidateDr(ref string info, object desc, IExtractor ex, int validateId)
{
Validator validator = new Validator();
if (!validator.Validate(desc, ex, (short)validateId))
{
info = validator.ErrorString;
return false;
}
return true;
}
private bool Validate(ref string info, object desc, IExtractor ex, short validatedID)
{
Validator validator = new Validator();
if (!validator.Validate(desc, ex, validatedID))
{
info = validator.ErrorString;
return false;
}
return true;
}
public void Remove(IExtractor extractor)
{
var service = ServiceLocator.Instance.Get<IExtractingService<IExtractor>>();
if (service != null)
{
var prompt = "Are you sure you want to remove the selected extractor?\r\n\r\n" +
$"Extractor Name = \"{extractor.Name}\"";
var result = MessageBox.Show(
prompt,
"Remove Extractor",
MessageBoxButton.YesNo,
MessageBoxImage.Question,
MessageBoxResult.No);
if (result == MessageBoxResult.Yes)
{
service.Extractors.Remove(extractor);
}
}
}
public void Edit(IExtractor extractor)
{
Debug.Assert(extractor != null, "Extractor must be supplied to allow editing.");
var window = new AddEditExtractorWindow();
var data = new AddEditExtractor(window, true);
window.DataContext = data;
window.Owner = Application.Current.MainWindow;
data.Name = extractor.Name;
data.Field = extractor.Field;
data.Pattern = extractor.Pattern;
data.Mode = extractor.Mode;
var dialogResult = window.ShowDialog();
if (dialogResult != null && (bool)dialogResult)
{
extractor.Name = data.Name;
extractor.Pattern = data.Pattern;
extractor.Mode = data.Mode;
extractor.Field = data.Field;
}
}
static BoilerPipe()
{
ArticleExtractor = new ArticleExtractorWrapper();
ArticleSentencesExtractor = new ArticleSentencesExtractorWrapper();
CanolaExtractor = new CanolaExtractorWrapper();
DefaultExtractor = new DefaultExtractorWrapper();
KeepEverythingExtractor = new KeepEverythingExtractorWrapper();
KeepEverythingWithMinKWordsExtractor = new KeepEverythingWithMinKWordsExtractorWrapper();
LargestContentExtractor = new LargestContentExtractorWrapper();
NumWordsRulesExtractor = new NumWordsRulesExtractorWrapper();
}
/// <summary>
/// Command line exit Code Summary:
/// 0 – Application completed its task with no errors
/// 1 – Configuration.xml error
/// 2 – Missing plugin dll
/// 3 - Missing refmapper xml
/// 4 - Datasource directory error
/// 5 - XML output directory error
/// 6 - Data Transfer failed
/// 7 - Log directory error
/// 8 - Failed to move sent file to the "completed" subdirectory
/// 9 - No data to send. There was no valid data to send, double check source files.
/// </summary>
public static void Main(string[] args)
{
// path of where app is running from
_path = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetEntryAssembly().Location);
// init _conf and check files
if (CheckFiles(_path))
{
// log to catch errors
Log log = new Log(_conf.LogDirectory);
// load plugin
_plugin = PluginLoader.LoadExtractor("plugins\\" + _conf.Plugin);
// DataTable to store all the extracted data from files
DataTable results = new DataTable();
string[] files = System.IO.Directory.GetFiles(_conf.DataSourceDirectory);
// scans folder for file
foreach (string file in files)
{
// extract data
DataTable dt = _plugin.GetData(_conf, file);
// valid datatable of all valid rows
DataTable validDt = dt.Clone();
// validate the data
foreach (DataRow dr in dt.Rows)
{
Validator validator = new Validator();
bool valid = true;
if (!validator.ValidateDataRow(dr))
valid = false;
if (!_plugin.Validate(dr, _path + "\\plugins\\" + _conf.RefMapper))
valid = false;
// if valid import to validDt
if (valid)
{
validDt.ImportRow(dr);
}
else
{
log.write("Error validating row in " + file + ": ");
// print datarow contents to log file
foreach (DataColumn dc in dr.Table.Columns)
{
log.write(dc.ColumnName + " : " + dr[dc].ToString());
}
}
}
// merged data to Result
results.Merge(validDt);
}
// timestamp for use in the xml
DateTime date = DateTime.Now;
string timestamp = String.Format("{0:yyyyMMddHHmmss}", date);
// if results table is empty then don't generate an empty xml
if (results.Rows.Count != 0)
{
// generate xml
XmlOutput output = new XmlOutput(null);
string xmlFileName = timestamp + ".xml";
// need to parse an emtpy genelist
IList<SiteConf.Gene.Object> geneList = new List<SiteConf.Gene.Object>();
output.Generate("", _conf.XmlOutputDirectory + "\\" + xmlFileName, results, null, null, _conf.OrgHashCode, geneList);
// call VariantExporterCmdLn to send data
if (SendData())
{
// check if completed directory exist
if (!System.IO.Directory.Exists(_conf.DataSourceDirectory + "\\completed"))
{
// create "completed" sub directory inside the xmlOutputDirectory
System.IO.Directory.CreateDirectory(_conf.DataSourceDirectory + "\\completed");
}
// if send successful move files to completed sub directory
foreach (string file in files)
{
// rename file by appending the date to the end of file
// from this "file.tsv" to this "file_20140716143423.tsv"
string renamedFile = System.IO.Path.GetFileNameWithoutExtension(file) + "(" +
DateTime.Now.ToString("yyyyMMddHHmmss") + ")" + System.IO.Path.GetExtension(file);
try
{
System.IO.File.Move(file, _conf.DataSourceDirectory + "\\completed\\" + renamedFile);
log.write("Data from file: " + file + " sent.");
}
catch
{
log.write("Data from file: " + file + " was sent, but due to an error the file could not be moved to the completed sub directory.");
}
}
}
}
else
{
log.write("No valid data to send. Check source that all mandatory fields are provided and correct.");
System.Environment.ExitCode = 9;
}
}
}
public Crawler(Parameters param)
{
this.param = param;
Extractor = new Extractor();
}
private static void DbShow(IExtractor shunDb)
{
using (StreamWriter sw = File.CreateText("blacklistDB.csv"))
{
foreach (KeyValuePair<string, List<KeyValuePair<string, List<string>>>> item in shunDb.BlackListDB)
{
string ip = item.Key;
//Console.WriteLine("IP = {0}", ip);
for (int i = 0; i < item.Value.Count; i++)
{
//Console.WriteLine("{0}", item.Value.ElementAt(i).Key);
foreach (var item2 in item.Value.ElementAt(i).Value)
{
//Console.WriteLine(item2);
sw.WriteLine("{0},{1},{2}",ip, item.Value.ElementAt(i).Key, item2);
}
}
}
}
}
public void feeds(IExtractor obj)
{
Dictionary<string, List<KeyValuePair<string, List<string>>>> temp = obj.ExtractIP();
BlackListFull.Add(temp);
}
public bool Validate(object desc, IExtractor ex, short validatedID)
{
this.m_ID = validatedID;
return (this.IsPropertyOk(desc, ex) && this.IsObjectOk(desc));
}
public override void Register(IExtractor obj)
{
Feeder reg = new Feeder();
BlockList_de feed = new BlockList_de();
reg.FeedList.Add(feed);
}
/// <summary>
/// Loads an upload to main form by filling in all the details in the form based
/// on the UploadID of that upload.
/// </summary>
/// <param name="uploadID"></param>
public void LoadUpload(int uploadID)
{
// set the uploadID
_uploadID = uploadID;
SiteConf.Upload.Object upload = ExporterCommon.DataLoader.GetUpload(uploadID);
displayUploadDetails(upload);
ReloadDgGene(upload);
plugin = PluginLoader.LoadExtractor(upload.Plugin);
// hide the save message
lblSavedMessage.Visible = false;
// clear the datagrid in the send data tab
dgVariant.DataSource = null;
RefreshTabs();
// display message if private key not found on startup
if (!CheckKeys(conf))
{
MessageBox.Show("The system can not locate the Private Key, a connection can not be established without the private key. You can add a private key from the Tools menu and selecting Import Private Key. If you don't have a private key please contact someone from HVP.",
"No private key set", MessageBoxButtons.OK, MessageBoxIcon.Warning);
}
}
public Crawler(XElement e, Parameters param)
{
this.param = param;
_urls = DictionaryFromXml(e);
Extractor = new Extractor();
}
public abstract void Register(IExtractor obj);
public ExtractorHelper(ProjectUpdaterProvider projectUpdaterProvider, IExtractor extractor)
{
ProjectUpdaterProvider = projectUpdaterProvider;
Extractor = extractor;
}
public override void Register(IExtractor obj)
{
Feeder reg = new Feeder();
Ipset feed = new Ipset();
reg.FeedList.Add(feed);
}
public override void Register(IExtractor obj)
{
Feeder reg = new Feeder();
SSLIpBlackList feed = new SSLIpBlackList();
reg.FeedList.Add(feed);
}
private bool IsPropertyOk(object desc, IExtractor ex)
{
this.m_desc = desc;
this.m_ex = ex;
return Array.TrueForAll<PropertyInfo>(desc.GetType().GetProperties(), new Predicate<PropertyInfo>(this.ValidateProperty));
}
