private InternalCollection <R2, T> Manufacture <R2>(Func <int, Stage <T>, UnaryVertex <Weighted <R>, Weighted <R2>, T> > factory, Expression <Func <Weighted <R>, int> > inputPartitionedBy, Expression <Func <Weighted <R2>, int> > outputPartitionedBy, string name) where R2 : IEquatable <R2> { var output = Foundry.NewUnaryStage(this.Output, factory, inputPartitionedBy, outputPartitionedBy, name); return(new InternalCollection <R2, T>(output, this.Immutable)); }
public static Stage <ConsumerVertex, Epoch> MakeStage(int numberToConsume, int startProcess, int endProcess, int numberOfWorkers, bool exchange, Stream <Pair <int, int>, Epoch> stream) { var locations = new List <VertexLocation>(); for (int i = 0; i < endProcess - startProcess; i++) { for (int j = 0; j < numberOfWorkers; j++) { locations.Add(new VertexLocation(locations.Count, i + startProcess, j)); } } Placement placement = new Placement.Explicit(locations); Stage <ConsumerVertex, Epoch> stage = Foundry.NewStage(placement, stream.Context, (i, s) => new ConsumerVertex(i, s, numberToConsume), "Consumer"); if (exchange) { stage.NewInput(stream, (m, v) => v.OnRecv(m), x => x.Second); } else { stage.NewInput(stream, (m, v) => v.OnRecv(m), x => x.First); } return(stage); }
internal StreamingInputStage(DataSource <R> source, Placement placement, InternalComputation internalComputation, string inputName) { this.inputName = inputName; this.stage = Foundry.NewStage(new TimeContext <Epoch>(internalComputation.ContextManager.RootContext), (i, v) => new StreamingInputVertex <R>(i, v), this.inputName); this.output = stage.NewOutput(vertex => vertex.output); this.stage.Materialize(); this.localVertices = placement.Where(x => x.ProcessId == internalComputation.Controller.Configuration.ProcessID) .Select(x => this.stage.GetVertex(x.VertexId) as StreamingInputVertex <R>) .ToArray(); source.RegisterInputs(this.localVertices); this.completedCalled = false; this.hasActivatedProgressTracker = false; // results in pointstamp comparisons which assert w/o this. this.InternalComputation.Reachability.UpdateReachabilityPartialOrder(internalComputation); this.InternalComputation.Reachability.DoNotImpersonate(stage.StageId); var initialVersion = new Runtime.Progress.Pointstamp(stage.StageId, new int[] { 0 }); internalComputation.ProgressTracker.BroadcastProgressUpdate(initialVersion, placement.Count); }
internal static Stream <R, T> Broadcast <R, T>(this Stream <R, T> stream) where R : Cloneable <R> where T : Time <T> { var controller = stream.ForStage.InternalComputation.Controller; int threadCount = stream.ForStage.InternalComputation.DefaultPlacement.Count / controller.Configuration.Processes; if (threadCount * controller.Configuration.Processes != stream.ForStage.InternalComputation.DefaultPlacement.Count) { throw new Exception("Uneven thread count?"); } var processDests = stream.ForStage.InternalComputation.DefaultPlacement.Where(x => x.ThreadId == 0).Select(x => x.VertexId).ToArray(); var boutput = Foundry.NewUnaryStage(stream, (i, v) => new BroadcastSendVertex <R, T>(i, v, processDests), null, null, "BroadcastProcessSend"); var collectable = boutput; if (stream.ForStage.InternalComputation.DefaultPlacement.Where(x => x.ProcessId == controller.Configuration.ProcessID).Count() > 1) { var threadDests = stream.ForStage.InternalComputation.DefaultPlacement .Where(x => x.ProcessId == controller.Configuration.ProcessID) .Select(x => x.VertexId) .ToArray(); collectable = Foundry.NewUnaryStage(boutput, (i, v) => new BroadcastForwardVertex <R, T>(i, v, threadDests), x => x.First, null, "BroadcastVertexSend"); } // TODO : fix this to use a streaming expression return(collectable.UnaryExpression(null, xs => xs.Select(x => x.Second), "Select")); }
public override Type GetType(string componentName, out string source, out string ns) { source = null; ns = null; Type type = null; Foundry foundry = tagnames [componentName] as Foundry; if (foundry != null) { return(foundry.GetType(componentName, out source, out ns)); } if (assemblyFoundry != null) { try { type = assemblyFoundry.GetType(componentName, out source, out ns); return(type); } catch { } } string msg = String.Format("Type {0} not registered for prefix {1}", componentName, tagPrefix); throw new ApplicationException(msg); }
internal static Stream <Pair <TKey, TState>, TTime> LocalReduce <TReducer, TState, TValue, TOutput, TKey, TInput, TTime>( this Stream <TInput, TTime> stream, Func <TInput, TKey> key, Func <TInput, TValue> val, Func <TReducer> factory, string name, Expression <Func <TInput, int> > inPlacement, Expression <Func <Pair <TKey, TState>, int> > outPlacement) where TReducer : IReducer <TState, TValue, TOutput> where TTime : Time <TTime> { return(Foundry.NewUnaryStage(stream, (i, v) => new LocalKeyedReduceVertex <TReducer, TState, TValue, TOutput, TKey, TInput, TTime>(i, v, key, val, factory), inPlacement, outPlacement, name)); }
internal static Stream <Pair <K, X>, T> LocalTimeReduce <A, X, R, S, K, I, T>( this Stream <I, T> stream, Func <I, K> key, Func <I, R> val, Func <A> factory, string name, Expression <Func <I, int> > inPlacement, Expression <Func <Pair <K, X>, int> > outPlacement) where A : IReducer <X, R, S> where T : Time <T> { return(Foundry.NewUnaryStage(stream, (i, v) => new LocalTimeKeyedReduceVertex <A, X, R, S, K, I, T>(i, v, key, val, factory), inPlacement, outPlacement, name)); }
private ShardedCollection <R2, T> Manufacture <S, R2>(TypedCollection <S, T> other, Func <int, Stage <T>, BinaryVertex <Weighted <R>, Weighted <S>, Weighted <R2>, T> > factory, Expression <Func <Weighted <R>, int> > input1PartitionedBy, Expression <Func <Weighted <S>, int> > input2PartitionedBy, Expression <Func <Weighted <R2>, int> > outputPartitionedBy, string name) where S : IEquatable <S> where R2 : IEquatable <R2> { var output = Foundry.NewStage(this.Output, other.Output, factory, input1PartitionedBy, input2PartitionedBy, outputPartitionedBy, name); return(new ShardedCollection <R2, T>(output, this.Immutable && other.Immutable)); }
public static Stream <Weighted <R>, Epoch> NewStage(Stream <Weighted <R>, Epoch> source) { var stage = Foundry.NewStage(source.Context, (i, v) => new IntegratorShard <R>(i, v), "Integrator"); stage.NewInput(source, shard => shard.input, source.PartitionedBy); return(stage.NewOutput(shard => shard.output, source.PartitionedBy)); }
public static Stage <ConsumerVertex, Epoch> MakeStage(int numberToConsume, int numberOfPartitions, Stream <int, Epoch> stream) { Placement placement = new Placement.Explicit(Enumerable.Range(0, numberOfPartitions).Select(x => new VertexLocation(x, 1, x))); Stage <ConsumerVertex, Epoch> stage = Foundry.NewStage(placement, stream.Context, (i, s) => new ConsumerVertex(i, s, numberToConsume), "Consumer"); stage.NewInput(stream, (m, v) => v.OnRecv(m), x => x); return(stage); }
public static Stream <int, Epoch> MakeStage(int numberToSend, int numberOfPartitions, Stream <int, Epoch> input) { Placement placement = new Placement.Explicit(Enumerable.Range(0, numberOfPartitions).Select(x => new VertexLocation(x, 0, x))); Stage <ProducerVertex, Epoch> stage = Foundry.NewStage(placement, input.Context, (i, s) => new ProducerVertex(i, s, numberToSend), "Producer"); stage.NewInput(input, (v, m) => { }, null); Stream <int, Epoch> stream = stage.NewOutput(v => v.output); return(stream); }
public static Stream <R, T> PartitionBy <R, T>(this Stream <R, T> stream, Expression <Func <R, int> > partitionBy) where T : Time <T> { // if the data are already partitioned (or claim to be) just return the stream. if (partitionBy == null || Naiad.CodeGeneration.ExpressionComparer.Instance.Equals(stream.PartitionedBy, partitionBy)) { return(stream); } return(Foundry.NewStage(stream, (i, v) => new PartitionByShard <R, T>(i, v, null), partitionBy, partitionBy, "PartitionBy")); }
public void built_objects_should_be_overridable() { var result = Foundry.Cast <TestClass>(x => { x.TestValue = "changed"; x.Id = 100; }); Assert.AreEqual("changed", result.TestValue); Assert.AreEqual(100, result.Id); }
internal static Stream <Pair <K, S>, T> LocalTimeCombine <A, X, R, S, K, T>( this Stream <Pair <K, X>, T> stream, Func <A> factory, string name, Expression <Func <Pair <K, S>, int> > outPlacement) where A : IReducer <X, R, S> where T : Time <T> { Expression <Func <Pair <K, X>, int> > inPlacement = null; if (outPlacement != null) { inPlacement = x => x.First.GetHashCode(); } return(Foundry.NewUnaryStage(stream, (i, v) => new LocalTimeKeyedCombineVertex <A, X, R, S, K, T>(i, v, factory), inPlacement, outPlacement, name)); }
void InternalRegister (string foundryName, Foundry foundry) { object f = foundries [foundryName]; if (f is CompoundFoundry) { ((CompoundFoundry) f).Add (foundry); } else if (f == null || (f is AssemblyFoundry && foundry is AssemblyFoundry)) { // If more than 1 namespace/assembly specified, the last one is used. foundries [foundryName] = foundry; } else if (f != null) { CompoundFoundry compound = new CompoundFoundry (foundryName); compound.Add ((Foundry) f); compound.Add (foundry); foundries [foundryName] = compound; } }
AspComponent CreateComponent(Foundry foundry, string tagName, string prefix, string tag) { string source, ns; Type type; type = foundry.GetType(tag, out source, out ns); if (type == null) { return(null); } AspComponent ret = new AspComponent(type, ns, prefix, source, foundry.FromConfig); Dictionary <string, AspComponent> components = Components; components.Add(tagName, ret); return(ret); }
public static Stream <Pair <int, int>, Epoch> MakeStage(int numberToSend, int startProcess, int endProcess, int numberOfWorkers, Stream <Pair <int, int>, Epoch> input) { var locations = new List <VertexLocation>(); for (int i = 0; i < endProcess - startProcess; i++) { for (int j = 0; j < numberOfWorkers; j++) { locations.Add(new VertexLocation(locations.Count, i + startProcess, j)); } } Placement placement = new Placement.Explicit(locations); Stage <ProducerVertex, Epoch> stage = Foundry.NewStage(placement, input.Context, (i, s) => new ProducerVertex(i, s, numberToSend), "Producer"); stage.NewInput(input, (v, m) => { }, null); Stream <Pair <int, int>, Epoch> stream = stage.NewOutput(v => v.output); return(stream); }
public void Add(Foundry foundry) { if (foundry is AssemblyFoundry) { assemblyFoundry = (AssemblyFoundry)foundry; return; } TagNameFoundry tn = (TagNameFoundry)foundry; string tagName = tn.TagName; if (tagnames.Contains(tagName)) { if (tn.FromWebConfig) { return; } string msg = String.Format("{0}:{1} already registered.", tagPrefix, tagName); throw new ApplicationException(msg); } tagnames.Add(tagName, foundry); }
public static Stream <Pair <TVertex, TState>, TTime> GraphJoin <TVertex, TState, TTime>(this Stream <Pair <TVertex, TState>, TTime> values, Stream <Pair <TVertex, TVertex>, TTime> edges) where TTime : Time <TTime> { return(Foundry.NewBinaryStage(edges, values, (i, s) => new GraphJoinVertex <TVertex, TState, TTime>(i, s), x => x.First.GetHashCode(), y => y.First.GetHashCode(), null, "GraphJoin")); }
/// <summary> /// Aggregates key-value pairs, producing new outputs whenever an aggregate changes. /// </summary> /// <typeparam name="TKey">key type</typeparam> /// <typeparam name="TValue">value type</typeparam> /// <typeparam name="TTime">time type</typeparam> /// <param name="input">input key-value stream</param> /// <param name="aggregate">aggregation function</param> /// <returns>aggregated key-value pairs</returns> public static Stream <Pair <TKey, TValue>, TTime> StreamingAggregate <TKey, TValue, TTime>(this Stream <Pair <TKey, TValue>, TTime> input, Func <TValue, TValue, TValue> aggregate) where TTime : Time <TTime> where TValue : IEquatable <TValue> { return(Foundry.NewUnaryStage(input, (i, s) => new StreamingAggregateVertex <TKey, TValue, TTime>(i, s, aggregate), x => x.First.GetHashCode(), x => x.First.GetHashCode(), "StreamingAggregate")); }
public override System.Web.Mvc.ActionResult Yours(Foundry.Security.FoundryUser user) { var callInfo = new T4MVC_ActionResult(Area, Name, ActionNames.Yours); callInfo.RouteValueDictionary.Add("user", user); return callInfo; }
public override System.Web.Mvc.ActionResult Create(Foundry.Website.Models.Project.CreateViewModel model, Foundry.Security.FoundryUser user) { var callInfo = new T4MVC_ActionResult(Area, Name, ActionNames.Create); callInfo.RouteValueDictionary.Add("model", model); callInfo.RouteValueDictionary.Add("user", user); return callInfo; }
public static Stream <R, T> AssertPartitionedBy <R, T>(Stream <R, T> stream, Expression <Func <R, int> > partitionBy) where T : Time <T> { return(Foundry.NewStage(stream, (i, v) => new PartitionByShard <R, T>(i, v, partitionBy), null, partitionBy, "PartitionBy")); }
public override System.Web.Mvc.ActionResult Register(Foundry.Website.Models.Account.RegisterViewModel model) { var callInfo = new T4MVC_ActionResult(Area, Name, ActionNames.Register); callInfo.RouteValueDictionary.Add("model", model); return callInfo; }
// key-value pairs on the first input are retrieved via the second input. public static Stream <Pair <TKey, TValue>, Epoch> KeyValueLookup <TKey, TValue>(this Stream <Pair <TKey, TValue>, Epoch> kvpairs, Stream <TKey, Epoch> requests) { return(Foundry.NewBinaryStage(kvpairs, requests, (i, s) => new KeyValueLookupVertex <TKey, TValue>(i, s), x => x.First.GetHashCode(), y => y.GetHashCode(), null, "Lookup")); }
public List <Instruction> GetInstructions(CharacterSheet sheet) { List <Instruction> instructions = new List <Instruction>(); Instruction getOre = new Instruction(); getOre.destination = sheet.baseCity.OreShops[0].gameObject.GetComponent <NavigationWaypoint>(); getOre.building = sheet.baseCity.OreShops[0]; getOre.gather = new ItemType[] { ItemType.ORE }; getOre.give = new ItemType[] { }; getOre.fun1 = new instructionFunction((getOre.building).GetItem); instructions.Add(getOre); Instruction makeBar = new Instruction(); Foundry destination = null; foreach (Foundry foundry in sheet.baseCity.Foundries) { if (foundry.workers.Contains(sheet)) { destination = foundry; break; } } if (destination == null) { foreach (Foundry foundry in sheet.baseCity.Foundries) { if (foundry.CurrentPositions[Jobs.SMITH] > 0) { destination = foundry; foundry.workers.Add(sheet); foundry.CurrentPositions[Jobs.SMITH]--; break; } } } makeBar.destination = destination.gameObject.GetComponent <NavigationWaypoint>(); makeBar.building = destination; makeBar.gather = new ItemType[] { ItemType.BAR }; makeBar.give = new ItemType[] { ItemType.ORE }; makeBar.recipe = MasterRecipe.Instance.Bar; makeBar.fun1 = new instructionFunction((makeBar.building).MakeRecipe); instructions.Add(makeBar); Instruction storeBar = new Instruction(); storeBar.destination = destination.gameObject.GetComponent <NavigationWaypoint>(); storeBar.building = destination; storeBar.gather = new ItemType[] { }; storeBar.give = new ItemType[] { ItemType.BAR }; storeBar.fun1 = new instructionFunction((storeBar.building).StoreItem); storeBar.fun2 = new instructionFunction2((destination).ReleaseJob); instructions.Add(storeBar); return(instructions); }
/// <summary> /// Returns elements in the first stream but not the second stream. /// </summary> /// <typeparam name="TRecord">Record type</typeparam> /// <typeparam name="TTime">Time type</typeparam> /// <param name="stream1">first input stream</param> /// <param name="stream2">second input stream</param> /// <returns></returns> public static Stream <TRecord, TTime> Except <TRecord, TTime>(this Stream <TRecord, TTime> stream1, Stream <TRecord, TTime> stream2) where TTime : Time <TTime> { return(Foundry.NewStage(stream1, stream2, (i, s) => new ExceptVertex <TRecord, TTime>(i, s), x => x.GetHashCode(), x => x.GetHashCode(), x => x.GetHashCode(), "Except")); }
/// <summary> /// Joins two input streams. /// </summary> /// <typeparam name="TInput1">First input type</typeparam> /// <typeparam name="TInput2">Second input type</typeparam> /// <typeparam name="TKey">Key type</typeparam> /// <typeparam name="TResult">Result type</typeparam> /// <typeparam name="TTime">Time type</typeparam> /// <param name="stream1">first input stream</param> /// <param name="stream2">second input stream</param> /// <param name="key1">first key selector</param> /// <param name="key2">second key selector</param> /// <param name="reducer">result selector</param> /// <returns>each pair of matching records, subjected to the reducer function</returns> public static Stream <TResult, TTime> Join <TInput1, TInput2, TKey, TResult, TTime>(this Stream <TInput1, TTime> stream1, Stream <TInput2, TTime> stream2, Func <TInput1, TKey> key1, Func <TInput2, TKey> key2, Func <TInput1, TInput2, TResult> reducer) where TTime : Time <TTime> { //return stream1.BinaryExpression(stream2, x => key1(x).GetHashCode(), x => key2(x).GetHashCode(), (x1, x2) => x1.Join(x2, key1, key2, reducer), "Join"); return(Foundry.NewStage(stream1, stream2, (i, s) => new JoinVertex <TInput1, TInput2, TKey, TResult, TTime>(i, s, key1, key2, reducer), x => key1(x).GetHashCode(), x => key2(x).GetHashCode(), null, "Join")); }
internal static Stream <TOutput, TTime> LocalCombine <TReducer, TState, TInput, TOutput, TTime>(this Stream <TState, TTime> stream, Func <TReducer> factory, string name) where TReducer : IReducer <TState, TInput, TOutput> where TTime : Time <TTime> { return(Foundry.NewUnaryStage(stream, (i, v) => new LocalCombineVertex <TReducer, TState, TInput, TOutput, TTime>(i, v, factory), null, null, name)); }
/// <summary> /// Used to write records to files. /// </summary> /// <typeparam name="S">Record type</typeparam> /// <param name="input">Source of records</param> /// <param name="format">Format string for filename; {0} replaced with shard id</param> /// <param name="action">Operation to apply to each record and the output stream. Often (r,s) => s.Write(r);</param> public static void WriteToFiles <S>(this Stream <S, Epoch> input, string format, Action <S, System.IO.BinaryWriter> action) { Foundry.NewStage(input, (i, v) => new Writer <S>(i, v, action, format), null, "Writer"); }
public void Add (Foundry foundry) { if (foundry is AssemblyFoundry) { assemblyFoundry = (AssemblyFoundry) foundry; return; } TagNameFoundry tn = (TagNameFoundry) foundry; string tagName = tn.TagName; if (tagnames.Contains (tagName)) { if (tn.FromWebConfig) return; string msg = String.Format ("{0}:{1} already registered.", tagPrefix, tagName); throw new ApplicationException (msg); } tagnames.Add (tagName, foundry); }
void InternalRegister (string foundryName, Foundry foundry, bool fromConfig) { object f = foundries [foundryName]; Foundry newFoundry = null; if (f is CompoundFoundry) { ((CompoundFoundry) f).Add (foundry); return; } else if (f == null || f is ArrayList || (f is AssemblyFoundry && foundry is AssemblyFoundry)) { newFoundry = foundry; } else if (f != null) { CompoundFoundry compound = new CompoundFoundry (foundryName); compound.Add ((Foundry) f); compound.Add (foundry); newFoundry = foundry; newFoundry.FromConfig = fromConfig; } if (newFoundry == null) return; if (f == null) { foundries [foundryName] = newFoundry; return; } ArrayList af = f as ArrayList; if (af == null) { af = new ArrayList (2); af.Add (f); foundries [foundryName] = af; } if (newFoundry is AssemblyFoundry) { object o; for (int i = 0; i < af.Count; i++) { o = af [i]; if (o is AssemblyFoundry) { af.Insert (i, newFoundry); return; } } af.Add (newFoundry); } else af.Insert (0, newFoundry); }
public void it_should_fail_if_it_finds_more_than_one_castring_from_executing_test_assembly() { var result = Foundry.Cast <Multiple>(); }
void InternalRegister(string foundryName, Foundry foundry, bool fromConfig) { object f = foundries [foundryName]; Foundry newFoundry = null; if (f is CompoundFoundry) { ((CompoundFoundry)f).Add(foundry); return; } else if (f == null || f is ArrayList || (f is AssemblyFoundry && foundry is AssemblyFoundry)) { newFoundry = foundry; } else if (f != null) { CompoundFoundry compound = new CompoundFoundry(foundryName); compound.Add((Foundry)f); compound.Add(foundry); newFoundry = foundry; newFoundry.FromConfig = fromConfig; } if (newFoundry == null) { return; } if (f == null) { foundries [foundryName] = newFoundry; return; } ArrayList af = f as ArrayList; if (af == null) { af = new ArrayList(2); af.Add(f); foundries [foundryName] = af; } if (newFoundry is AssemblyFoundry) { object o; for (int i = 0; i < af.Count; i++) { o = af [i]; if (o is AssemblyFoundry) { af.Insert(i, newFoundry); return; } } af.Add(newFoundry); } else { af.Insert(0, newFoundry); } }
public void it_should_fail_if_it_does_not_find_a_casting_from_executing_test_assembly() { var result = Foundry.Cast <NonExistent>(); }
public AspComponent GetComponent(string tagName) { if (tagName == null || tagName.Length == 0) { return(null); } if (components != null) { AspComponent ret; if (components.TryGetValue(tagName, out ret)) { return(ret); } } string foundryName, tag; int colon = tagName.IndexOf(':'); if (colon > -1) { if (colon == 0) { throw new Exception("Empty TagPrefix is not valid."); } if (colon + 1 == tagName.Length) { return(null); } foundryName = tagName.Substring(0, colon); tag = tagName.Substring(colon + 1); } else { foundryName = String.Empty; tag = tagName; } object o = foundries [foundryName]; if (o == null) { return(null); } Foundry foundry = o as Foundry; if (foundry != null) { return(CreateComponent(foundry, tagName, foundryName, tag)); } ArrayList af = o as ArrayList; if (af == null) { return(null); } AspComponent component = null; Exception e = null; foreach (Foundry f in af) { try { component = CreateComponent(f, tagName, foundryName, tag); if (component != null) { return(component); } } catch (Exception ex) { e = ex; } } if (e != null) { throw e; } return(null); }
AspComponent CreateComponent (Foundry foundry, string tagName, string prefix, string tag) { string source, ns; Type type; type = foundry.GetType (tag, out source, out ns); if (type == null) return null; AspComponent ret = new AspComponent (type, ns, prefix, source, foundry.FromConfig); Dictionary <string, AspComponent> components = Components; components.Add (tagName, ret); return ret; }