public static void Reduce(IFileSource<string, string> inputfolders, IFileDestination<string, string> outputfolder, IReducer reducer, int threads)
{
var threadpool = new ThreadPool(threads, "Reducer file threads");
Console.WriteLine("Preparing reducer threads.");
string inputfileid;
while (inputfolders.ReadNext(out inputfileid))
{
var ft = new ReduceFileThread();
ft.InputFolders = inputfolders;
ft.OutputFolder = outputfolder;
// ft.Writer = wrt;
ft.reducer = reducer;
ft.FileID = inputfileid;
threadpool.Queue(ft);
}
Console.WriteLine("Waiting for reducer threads to finish...");
var t1 = new Timing("Inner mapper");
threadpool.WaitAll();
t1.End();
Console.WriteLine("Reducer threads finished.");
GC.Collect();
}
public blogPost parse()
{
blogPost blogPost = new blogPost();
//quitamos la ruta
string title = fileRoute.Split('\\')[fileRoute.Split('\\').Length - 1];
//Obtener extension del archivo
switch (this.fileRoute.Split('.')[this.fileRoute.Split('.').Length - 1])
{
case "txt":
this.reducer = new TextReducer();
blogPost.title = title.Replace(".txt", "");
blogPost.body = this.reducer.reduceContent(this.fileRoute);
break;
case "pdf":
blogPost.title = title.Replace(".pdf", "");
this.reducer = new PDFReducer();
blogPost.body = this.reducer.reduceContent(this.fileRoute);
break;
case "html":
blogPost.title = title.Replace(".html", "");
this.reducer = new HTMLParser();
blogPost.body = this.reducer.reduceContent(this.fileRoute);
break;
}
return(blogPost);
}
/// <summary>
/// Constructeur initialisant le Noeud parent, la méthode à exécuter, le mapper et le reducer à utiliser
/// </summary>
/// <param name="node">Noeud parent</param>
/// <param name="function">méthode à exécuter</param>
/// <param name="mapper">Mapper</param>
/// <param name="reducer">Reducer</param>
public TaskExecutor(Node node, Func <DataInput, object> function, IMapper mapper, IReducer reducer)
{
Executor = node;
Mapper = mapper;
Reducer = reducer;
ProcessAction = function;
}
public Reducer(
GuessingGameInstance instance,
IReducer <Reduction, GuessingGameResult> next)
{
this.instance = instance;
this.next = next;
}
public Reducer(
Catching <TInput, TResult, TException> transducer,
IReducer <TReduction, TResult> next) : base(next)
{
success = transducer.Success.Apply(next);
exceptional = transducer.Exceptional.Apply(next);
}
/// <summary>
/// Méthode d'écoute de fin d'exécution des BackgroundWorkers
/// </summary>
/// <param name="sender">BackgroundWorker emetteur</param>
/// <param name="e">résultat de fin de traitement</param>
protected void WorkerEndProcess(object sender, RunWorkerCompletedEventArgs e)
{
// Manage if e.Error != null
Tuple <int, DataInput, int> data = (Tuple <int, DataInput, int>)e.Result;
DataInput resp = data.Item2;
UpdateWorkerTaskStatus(data.Item1, data.Item2.NodeTaskId, NodeState.Finish);
UpdateResult(resp.Data, data.Item2.NodeTaskId, data.Item1);
if (TaskIsCompleted(resp.NodeTaskId))
{
Logger.Write($"Traitement terminé pour {data.Item2.Method}.");
Logger.Write(@"Résultat envoyé à l'orchestrateur.");
IReducer reducer = WorkerFactory.GetWorker(resp.Method).Reducer;
object result = reducer.Reduce(GetResultFromTaskId(resp.NodeTaskId));
resp.Data = result;
SendData(Orch, resp);
}
else
{
double progression = GetWorkersProgression(data.Item2.NodeTaskId, data.Item3);
DataInput status = new DataInput()
{
ClientGuid = data.Item2.ClientGuid,
NodeGuid = NodeGuid,
MsgType = MessageType.Call,
Method = TaskStatusMethod,
TaskId = data.Item2.TaskId,
NodeTaskId = data.Item2.NodeTaskId,
Data = new Tuple <NodeState, object>(NodeState.Work, progression)
};
SendData(Orch, status);
}
}
public void Dispatch <T>(IAction action) where T : IState <T>
{
if (!subscribers.ContainsKey(action.GetName()))
{
return;
}
// Before take action
foreach (Subscriber subscriber in subscribers[action.GetName()])
{
subscriber.Invoke(action, true);
}
for (int i = 0; i < reducers.Count; i++)
{
try
{
IReducer <T> reducer = (IReducer <T>)reducers[i];
this.store[reducer.GetId()] = reducer.Reduce(action, Select <T>(reducer.GetId()));
}
catch { }
}
// After take action
foreach (Subscriber subscriber in subscribers[action.GetName()])
{
subscriber.Invoke(action, false);
}
}
public JoiningReducer(
SplittingReducer <TReduction> splitter,
IReducer <TReduction, TResult> next)
{
Splitter = splitter;
Next = next;
}
private ulong Rho(ulong n, ulong xInit, ulong c, IReducer <ulong> reducer)
{
if ((n & 1) == 0)
{
return(2);
}
var x = reducer.ToResidue(xInit);
var y = x.Copy();
var ys = x.Copy();
var r = 1;
var m = batchSize;
var cPrime = reducer.ToResidue(c);
var one = reducer.ToResidue(1);
var diff = one.Copy();
var q = one.Copy();
var g = (ulong)1;
do
{
x.Set(y);
for (int i = 0; i < r; i++)
{
AdvanceF(y, cPrime);
}
var k = 0;
while (k < r && g == 1)
{
ys.Set(y);
var limit = Math.Min(m, r - k);
q.Set(one);
for (int i = 0; i < limit; i++)
{
AdvanceF(y, cPrime);
q.Multiply(diff.Set(x).Subtract(y));
}
g = IntegerMath.GreatestCommonDivisor(q.Value, n);
k += limit;
}
r <<= 1;
}while (g == 1);
if (g.CompareTo(n) == 0)
{
// Backtrack.
do
{
AdvanceF(ys, cPrime);
g = IntegerMath.GreatestCommonDivisor(diff.Set(x).Subtract(ys).Value, n);
}while (g == 1);
}
if (g.CompareTo(n) == 0)
{
return(0);
}
return(g);
}
public void Setup()
{
// Enhance the Todo store to only return todo Items that should be visibile in the View.
rootReducer = new ReducerComposer <Todo>().AddSubTreeReducer(new TodosReducer())
.AddSubTreeReducer(new VisibilityFilterReducer());
store = new Store <Todo>(rootReducer, new Todo(Filter.ShowAll, new List <TodoItem>()))
.EnhanceWith(typeof(FilterTodoItems));
}
public ReductionContext(IReducer reducer)
{
Reducer = reducer;
BooleanConverter = new BooleanConverter();
StringConverter = new StringConverter();
NumbersConverter = new NumbersConverter();
}
public Store(IReducer <TState> rootReducer, IActionResolver actionResolver, IStateStorage storage, INavigationTracker <TState> navigationTracker, IDevToolsInterop devToolsInterop)
{
_rootReducer = rootReducer;
_actionResolver = actionResolver;
_storage = storage;
_navigationTracker = navigationTracker;
_devToolsInterop = devToolsInterop;
}
public Store(TState initialState, IReducer <TState> reducer, IEnumerable <IEffect <TState> > effects)
{
_stateReplaySubject = new ReplaySubject <TState>(1);
_reducer = reducer;
_effects = effects;
_allDispatchListeners = new List <Action <IAction> >();
State = initialState;
}
private static async Task ProcessSubResultAsync(IClusterClient client, IReducer proxy, string partialData)
{
var mapper = client.GetGrain <IMapper>(Guid.NewGuid());
AccelerationDataPerSec subResult = await mapper.MapAccelerationData(partialData);
Console.WriteLine("\n\n{0}\n\n", subResult.ToString());
await proxy.ReduceAccelerationData(subResult);
}
public AllUpProcessor(IMutator mutator, IPnLConsumer pnLConsumer, ISourcer sourcer, IReducer reducer, IScenarioGenerator scenarioGenerator, IEventReceiver eventReceiver)
{
this.mutator = mutator;
this.pnLConsumer = pnLConsumer;
this.sourcer = sourcer;
this.reducer = reducer;
this.scenarioGenerator = scenarioGenerator;
this.eventReceiver = eventReceiver;
}
/// <summary>
/// Initializes a new instance of <see cref="StoreBuilder{TState}"/> class.
/// </summary>
/// <param name="reducer">
/// A reducing object that returns the next state tree.
/// </param>
public StoreBuilder(IReducer <TState> reducer)
{
if (reducer == null)
{
throw new ArgumentNullException(nameof(reducer));
}
_reducer = reducer;
}
public Store(IReducer <TApplicationState> reducer, TApplicationState initialState = default(TApplicationState))
{
dispatcher = new Subject <object>();
storeStateStream = GetStateObservable(reducer, initialState);
storeStateStream.Connect();
//for initialState, might be set in reducer
Dispatch <object>(null);
}
public UserStateReducer(
IReducer <AsyncOperationState <User, Unit> > registerReducer,
IReducer <AsyncOperationState <User, AccessProvider> > loginReducer,
IReducer <AsyncOperationState <User, object> > editReducer,
IReducer <User> userReducer)
{
RegisterReducer = registerReducer ?? throw new ArgumentNullException(nameof(RegisterReducer));
LoginReducer = loginReducer ?? throw new ArgumentNullException(nameof(LoginReducer));
EditReducer = editReducer ?? throw new ArgumentNullException(nameof(editReducer));
UserReducer = userReducer ?? throw new ArgumentNullException(nameof(UserReducer));
}
public StateActionWriterNode(
INode parent,
string stateKey,
StoreHolder storeHolder,
IReducer reducer
) : base(parent)
{
this.stateKey = stateKey;
this.storeHolder = storeHolder;
this.reducer = reducer;
}
public ReduceContext(TypeTable typeTable,
ITerminals terminals,
ActionExpr action,
INoAction noAction,
int id)
{
this.typeTable = typeTable;
this.terminals = terminals;
this.action = action;
this.noAction = noAction;
Id = id;
}
/// <summary>
/// Réplique cet objet
/// </summary>
/// <returns>CLone de cette instance</returns>
public object Clone()
{
IMapper newMapper = null;
IReducer newReducer = null;
if (Mapper != null && Reducer != null)
{
newMapper = (IMapper)Mapper.Clone();
newReducer = (IReducer)Reducer.Clone();
}
return(new TaskExecutor(Executor, ProcessAction, newMapper, newReducer));
}
public SplittingReducer(
IList <TransducerSwitch <TInput, TResult> > transducers,
IReducer <TReduction, TResult> next)
{
Reducers = transducers
.Select(tSwitch =>
new ReducerOption(
tSwitch.Test,
tSwitch.Transducer.Apply(new JoiningReducer(this, next))))
.ToList();
Next = next;
}
public StringFacade()
{
this.TheString = string.Empty;
this.PopulateAction = new ActionModel()
{
Type = "Populate"
};
this.stringReducer = new StringReducer("");
this.stringEffect = new StringEffect();
this.stringSelector = new StringQuery();
this.store = new StringStore(this.stringReducer, this.stringEffect);
}
public Store(
IActionDispatcher dispatcher,
IReducer <T> reducer,
IStateStreamSubject <T> stateStreamSubject,
IEnumerable <IEffect <T> > effects,
IExceptionHandler exceptionHandler)
{
_dispatcher = dispatcher;
_reducer = reducer;
_stateStreamSubject = stateStreamSubject;
_effects = effects;
_exceptionHandler = exceptionHandler;
}
public Generator(
IReducer reducer,
CSharpGeneratorService cSharpGeneratorService,
CsvGeneratorService csvGeneratorService,
JsonGeneratorService jsonGeneratorService,
TsvGeneratorService tsvGeneratorService)
{
_reducer = reducer;
_cSharpGeneratorService = cSharpGeneratorService;
_csvGeneratorService = csvGeneratorService;
_jsonGeneratorService = jsonGeneratorService;
_tsvGeneratorService = tsvGeneratorService;
}
public PlaceDataReducer(
IReducer <AsyncOperationState <Place, int> > currentPlaceReducer,
IReducer <AsyncOperationState <IList <Place>, Criteria> > placeListReducer,
IReducer <AsyncOperationState <Report, Unit> > reportReducer,
IReducer <AsyncOperationState <WrongPlaceReport, Unit> > wrongPlaceReportReducer,
IReducer <AsyncOperationState <IList <Place>, long> > usersPlacesListReducer
)
{
this.currentPlaceReducer = currentPlaceReducer ?? throw new ArgumentNullException(nameof(currentPlaceReducer));
this.placeListReducer = placeListReducer ?? throw new ArgumentNullException(nameof(placeListReducer));
this.reportReducer = reportReducer ?? throw new ArgumentNullException(nameof(reportReducer));
this.wrongPlaceReportReducer = wrongPlaceReportReducer ?? throw new ArgumentNullException(nameof(wrongPlaceReportReducer));
this.usersPlacesListReducer = usersPlacesListReducer ?? throw new ArgumentNullException(nameof(usersPlacesListReducer));
}
/// <summary>
/// Initializes a new instance of <see cref="Store{TState}"/> class.
/// </summary>
/// <param name="reducer">
/// A reducing object that returns the next state tree.
/// </param>
/// <param name="initialState">
/// The initial state.
/// </param>
/// <param name="middlewares">
/// Functions that conform to the Redux middleware API.
/// </param>
public Store(IReducer <TState> reducer, TState initialState = default(TState), params MiddlewareDelegate <TState>[] middlewares)
{
_reducer = reducer ?? throw new ArgumentNullException(nameof(reducer));
_dispatch = ApplyMiddlewares(middlewares);
if (initialState != null)
{
State = initialState;
}
else
{
_dispatch(new ReduxInitialAction());
}
}
public ComplexTypeReducer <TState> SetPropertyReducer <TProp>(Expression <Func <TState, TProp> > selector,
IReducer <TProp> reducer)
{
var property = (PropertyInfo)((MemberExpression)selector.Body).Member;
var propertyReducer =
(IPropertyReducer <TState, TProp>)_propertyReducers.FirstOrDefault(p => p.Property == property);
if (propertyReducer != null)
{
propertyReducer.Reducer = reducer;
}
return(this);
}
public virtual void AddReducer <TAction>(IReducer <TState, TAction> reducer)
{
if (reducer == null)
{
throw new ArgumentNullException(nameof(reducer));
}
Type actionType = typeof(TAction);
if (!ReducersByActionType.TryGetValue(actionType, out List <object> reducers))
{
reducers = new List <object>();
ReducersByActionType[actionType] = reducers;
}
reducers.Add(reducer);
}
public ReducerSection(IActivator activator, ITypeRequest subReducer, Expression <StateSubSectionSelectionDelegate <TState, TSectionType> > sectionSelector)
{
m_sectionSelector = sectionSelector;
m_sectionReducer = activator.Get <IReducer <TSectionType> >(subReducer);
MemberExpression memberExpression = (MemberExpression)sectionSelector.Body;
PropertyInfo propertyInfo = (PropertyInfo)memberExpression.Member;
if (!propertyInfo.CanWrite || !propertyInfo.CanRead)
{
throw new NotAccessableSubSectionProperty($"The property {propertyInfo.Name} is being used as a Sub Section reducer however it is an invalid target. Any property used *MUST* be both to be able to be read and written to.");
}
m_getValue = (GetterDelegate)Delegate.CreateDelegate(typeof(GetterDelegate), propertyInfo.GetMethod);
m_setValue = (SetterDelegate)Delegate.CreateDelegate(typeof(SetterDelegate), propertyInfo.SetMethod);
}
public static TreeNode Reduce2(TreeNode node)
{
var reducers = new IReducer[]
{
new AddReducer(),
new MultReducer(),
new DivReducer(),
new IncReducer(),
new DecReducer(),
new EqualsReducer(),
new LessThanReducer(),
new NegationReducer(),
new SCombinatorReducer(),
new CCombinatorReducer(),
new BCombinatorReducer(),
new TrueReducer(),
new FalseReducer(),
new Power2Reducer(),
new IdentityReducer(),
new PairReducer(),
new FirstReducer(),
new TailReducer(),
new NilReducer(),
new IsNilReducer(),
new IfZeroReducer(),
new UserDefinedFunctionReducer(),
new ApReducer(),
};
var changed = true;
while (changed)
{
changed = false;
foreach (var reducer in reducers)
{
var result = reducer.Reduce(node);
if (result != node)
{
ReducesCount++;
changed = true;
node = result;
}
}
}
return(node);
}
public static void Map(IFileSource<string, string> inputfolders, IFileDestination<string, string> shuffleroutput, IMapper mapper, IReducer prereducer, int threads)
{
// var partitioner = new StandardKeyPartitioner(Count);
var threadpool = new ThreadPool(threads, "Mapper file threads");
var threadpool2 = new ThreadPool(threads, "Mapper threads");
Console.WriteLine("Preparing mappers...");
string inputfileid;
while (inputfolders.ReadNext(out inputfileid))
{
var ft = new MapFilesThread();
ft.InputFolders = inputfolders;
ft.OutputFolder = shuffleroutput;
ft.mapper = mapper;
ft.reducer = prereducer;
ft.FileIDs.Add(inputfileid);
ft.threadpool = threadpool;
threadpool.Queue(ft);
threadpool.Step();
}
Console.WriteLine("Waiting for mapper file threads to finish...");
var t1 = new Timing("Inner mapper");
threadpool.WaitAll();
t1.End();
Console.WriteLine("Mapper file threads finished.");
/*
Console.WriteLine("Waiting for mapper threads to finish...");
var t2 = new Timing("Inner mapper");
threadpool2.WaitAll();
t2.End();
Console.WriteLine("Mapper threads finished.");
*/
GC.Collect();
}
public FluentMapAndReduce ReduceUsing(IReducer reducer)
{
job.Reducer = reducer;
return this;
}
public FluentMapAndReduce ReduceInMemoryUsing(IReducer reducer)
{
job.Reducer = reducer;
job.ReducerFitsInMemory = true;
return this;
}
public static void Map(IFileSource<string, string> inputfolders, IFileDestination<string, string> shuffleroutput, IMapper mapper, IReducer prereducer)
{
Map(inputfolders, shuffleroutput, mapper, prereducer, 50);
}
public static void Reduce(IFileSource<string, string> inputfolders, IFileDestination<string, string> outputfolder, IReducer reducer)
{
Reduce(inputfolders, outputfolder, reducer, 30);
}
