Exemplo n.º 1
0
        private ShardedCollection <R2, T> Manufacture <R2>(Func <int, Stage <T>, UnaryVertex <Weighted <R>, Weighted <R2>, T> > factory, Expression <Func <Weighted <R>, int> > inputPartitionedBy, Expression <Func <Weighted <R2>, int> > outputPartitionedBy, string name)
            where R2 : IEquatable <R2>
        {
            var output = Foundry.NewStage(this.Output, factory, inputPartitionedBy, outputPartitionedBy, name);

            return(new ShardedCollection <R2, T>(output, this.Immutable));
        }
Exemplo n.º 2
0
Arquivo: Input.cs Projeto: omidm/naiad
        internal StreamingInputStage(DataSource <R> source, Placement placement, InternalComputation internalComputation, string inputName)
        {
            this.inputName = inputName;

            this.stage = Foundry.NewStage(new TimeContext <Epoch>(internalComputation.ContextManager.RootContext), (i, v) => new StreamingInputVertex <R>(i, v), this.inputName);

            this.output = stage.NewOutput(vertex => vertex.output);

            this.stage.Materialize();

            this.localVertices = placement.Where(x => x.ProcessId == internalComputation.Controller.Configuration.ProcessID)
                                 .Select(x => this.stage.GetVertex(x.VertexId) as StreamingInputVertex <R>)
                                 .ToArray();

            source.RegisterInputs(this.localVertices);

            this.completedCalled             = false;
            this.hasActivatedProgressTracker = false;

            // results in pointstamp comparisons which assert w/o this.
            this.InternalComputation.Reachability.UpdateReachabilityPartialOrder(internalComputation);
            this.InternalComputation.Reachability.DoNotImpersonate(stage.StageId);

            var initialVersion = new Runtime.Progress.Pointstamp(stage.StageId, new int[] { 0 });

            internalComputation.ProgressTracker.BroadcastProgressUpdate(initialVersion, placement.Count);
        }
Exemplo n.º 3
0
        public static Stage <ConsumerVertex, Epoch> MakeStage(int numberToConsume, int startProcess, int endProcess, int numberOfWorkers, bool exchange, Stream <Pair <int, int>, Epoch> stream)
        {
            var locations = new List <VertexLocation>();

            for (int i = 0; i < endProcess - startProcess; i++)
            {
                for (int j = 0; j < numberOfWorkers; j++)
                {
                    locations.Add(new VertexLocation(locations.Count, i + startProcess, j));
                }
            }

            Placement placement = new Placement.Explicit(locations);

            Stage <ConsumerVertex, Epoch> stage = Foundry.NewStage(placement, stream.Context, (i, s) => new ConsumerVertex(i, s, numberToConsume), "Consumer");


            if (exchange)
            {
                stage.NewInput(stream, (m, v) => v.OnRecv(m), x => x.Second);
            }
            else
            {
                stage.NewInput(stream, (m, v) => v.OnRecv(m), x => x.First);
            }

            return(stage);
        }
Exemplo n.º 4
0
        public static Stream <Weighted <R>, Epoch> NewStage(Stream <Weighted <R>, Epoch> source)
        {
            var stage = Foundry.NewStage(source.Context, (i, v) => new IntegratorShard <R>(i, v), "Integrator");

            stage.NewInput(source, shard => shard.input, source.PartitionedBy);

            return(stage.NewOutput(shard => shard.output, source.PartitionedBy));
        }
Exemplo n.º 5
0
        public static Stage <ConsumerVertex, Epoch> MakeStage(int numberToConsume, int numberOfPartitions, Stream <int, Epoch> stream)
        {
            Placement placement = new Placement.Explicit(Enumerable.Range(0, numberOfPartitions).Select(x => new VertexLocation(x, 1, x)));

            Stage <ConsumerVertex, Epoch> stage = Foundry.NewStage(placement, stream.Context, (i, s) => new ConsumerVertex(i, s, numberToConsume), "Consumer");

            stage.NewInput(stream, (m, v) => v.OnRecv(m), x => x);
            return(stage);
        }
Exemplo n.º 6
0
        public static Stream <R, T> PartitionBy <R, T>(this Stream <R, T> stream, Expression <Func <R, int> > partitionBy)
            where T : Time <T>
        {
            // if the data are already partitioned (or claim to be) just return the stream.
            if (partitionBy == null || Naiad.CodeGeneration.ExpressionComparer.Instance.Equals(stream.PartitionedBy, partitionBy))
            {
                return(stream);
            }

            return(Foundry.NewStage(stream, (i, v) => new PartitionByShard <R, T>(i, v, null), partitionBy, partitionBy, "PartitionBy"));
        }
Exemplo n.º 7
0
        public static Stream <int, Epoch> MakeStage(int numberToSend, int numberOfPartitions, Stream <int, Epoch> input)
        {
            Placement placement = new Placement.Explicit(Enumerable.Range(0, numberOfPartitions).Select(x => new VertexLocation(x, 0, x)));

            Stage <ProducerVertex, Epoch> stage = Foundry.NewStage(placement, input.Context, (i, s) => new ProducerVertex(i, s, numberToSend), "Producer");

            stage.NewInput(input, (v, m) => { }, null);
            Stream <int, Epoch> stream = stage.NewOutput(v => v.output);

            return(stream);
        }
Exemplo n.º 8
0
        public static Stream <Pair <int, int>, Epoch> MakeStage(int numberToSend, int startProcess, int endProcess, int numberOfWorkers, Stream <Pair <int, int>, Epoch> input)
        {
            var locations = new List <VertexLocation>();

            for (int i = 0; i < endProcess - startProcess; i++)
            {
                for (int j = 0; j < numberOfWorkers; j++)
                {
                    locations.Add(new VertexLocation(locations.Count, i + startProcess, j));
                }
            }

            Placement placement = new Placement.Explicit(locations);

            Stage <ProducerVertex, Epoch> stage = Foundry.NewStage(placement, input.Context, (i, s) => new ProducerVertex(i, s, numberToSend), "Producer");

            stage.NewInput(input, (v, m) => { }, null);
            Stream <Pair <int, int>, Epoch> stream = stage.NewOutput(v => v.output);

            return(stream);
        }
Exemplo n.º 9
0
 public static Stream <R, T> AssertPartitionedBy <R, T>(Stream <R, T> stream, Expression <Func <R, int> > partitionBy)
     where T : Time <T>
 {
     return(Foundry.NewStage(stream, (i, v) => new PartitionByShard <R, T>(i, v, partitionBy), null, partitionBy, "PartitionBy"));
 }
Exemplo n.º 10
0
 /// <summary>
 /// Used to write records to files.
 /// </summary>
 /// <typeparam name="S">Record type</typeparam>
 /// <param name="input">Source of records</param>
 /// <param name="format">Format string for filename; {0} replaced with shard id</param>
 /// <param name="action">Operation to apply to each record and the output stream. Often (r,s) => s.Write(r);</param>
 public static void WriteToFiles <S>(this Stream <S, Epoch> input, string format, Action <S, System.IO.BinaryWriter> action)
 {
     Foundry.NewStage(input, (i, v) => new Writer <S>(i, v, action, format), null, "Writer");
 }
Exemplo n.º 11
0
 /// <summary>
 /// Returns elements in the first stream but not the second stream.
 /// </summary>
 /// <typeparam name="TRecord">Record type</typeparam>
 /// <typeparam name="TTime">Time type</typeparam>
 /// <param name="stream1">first input stream</param>
 /// <param name="stream2">second input stream</param>
 /// <returns></returns>
 public static Stream <TRecord, TTime> Except <TRecord, TTime>(this Stream <TRecord, TTime> stream1, Stream <TRecord, TTime> stream2) where TTime : Time <TTime>
 {
     return(Foundry.NewStage(stream1, stream2, (i, s) => new ExceptVertex <TRecord, TTime>(i, s), x => x.GetHashCode(), x => x.GetHashCode(), x => x.GetHashCode(), "Except"));
 }
Exemplo n.º 12
0
 /// <summary>
 /// Joins two input streams.
 /// </summary>
 /// <typeparam name="TInput1">First input type</typeparam>
 /// <typeparam name="TInput2">Second input type</typeparam>
 /// <typeparam name="TKey">Key type</typeparam>
 /// <typeparam name="TResult">Result type</typeparam>
 /// <typeparam name="TTime">Time type</typeparam>
 /// <param name="stream1">first input stream</param>
 /// <param name="stream2">second input stream</param>
 /// <param name="key1">first key selector</param>
 /// <param name="key2">second key selector</param>
 /// <param name="reducer">result selector</param>
 /// <returns>each pair of matching records, subjected to the reducer function</returns>
 public static Stream <TResult, TTime> Join <TInput1, TInput2, TKey, TResult, TTime>(this Stream <TInput1, TTime> stream1, Stream <TInput2, TTime> stream2, Func <TInput1, TKey> key1, Func <TInput2, TKey> key2, Func <TInput1, TInput2, TResult> reducer)  where TTime : Time <TTime>
 {
     //return stream1.BinaryExpression(stream2, x => key1(x).GetHashCode(), x => key2(x).GetHashCode(), (x1, x2) => x1.Join(x2, key1, key2, reducer), "Join");
     return(Foundry.NewStage(stream1, stream2, (i, s) => new JoinVertex <TInput1, TInput2, TKey, TResult, TTime>(i, s, key1, key2, reducer), x => key1(x).GetHashCode(), x => key2(x).GetHashCode(), null, "Join"));
 }