コード例 #1
0
ファイル: QaSelect.cs プロジェクト: erisonliang/qizmt
            public override void Reduce(ByteSlice key, IEnumerator<ByteSlice> values, MapReduceOutput output)
            {
#if DEBUG
                    string KeyStringValue = "<not evaluated yet>";
                    if(0 != ((key.Length - 1) % 2))
                    {
                        KeyStringValue = "<not a string>";
                    }
                    else
                    {
                        //KeyStringValue = System.Text.Encoding.Unicode.GetString(ByteSlice.Prepare(key, 1, key.Length - 1).ToBytes());
                        {
                            System.Text.Encoding ue = new System.Text.UnicodeEncoding(false, false, true); // throwOnInvalidBytes=true
                            try
                            {
                                KeyStringValue = ue.GetString(ByteSlice.Prepare(key, 1, key.Length - 1).ToBytes());
                            }
                            catch
                            {
                                KeyStringValue = "<not a string>";
                            }
                        }
                    }
#endif

                if (null == sOptions)
                {
                    // Done early to detect SFUNC
                    sOptions = (DSpace_ExecArgs.Length > 8) ? DSpace_ExecArgs[8] : "-";

                    WhatFunctions = -1 != sOptions.IndexOf("SFUNC");
                    GroupBy = -1 != sOptions.IndexOf("GBY");
                }

                if (!WhatFunctions)
                {
                    // Normal: no functions (aggregates or scalars).
                    if (GroupBy)
                    {
                        values.Reset();
                        while (values.MoveNext())
                        {
                            output.Add(values.Current);
                            break;
                        }
                    }
                    else
                    {
                        values.Reset();
                        while (values.MoveNext())
                        {
                            output.Add(values.Current);
                        }
                    }
                }
                else //if(WhatFunctions)
                {
                    if (null == TableName)
                    {
                        InitFields(); // Note: only called if functions (aggregates or scalars)!

                        agtools = new DbFunctionTools();
                    }

                    bool GettingFieldTypeStrings = null == FieldTypeStrings;
                    if (GettingFieldTypeStrings)
                    {
                        FieldTypeStrings = new List<string>();
                    }

                    rows.Clear();
                    outvalues.Clear();
                    values.Reset();
                    while (values.MoveNext())
                    {
                        ByteSlice row = values.Current;
                        rows.Add(row);
                        outvalues.Add(agtools.AllocBuffer(row.Length > 256 ? Entry.Round2Power(row.Length) : 256));
                    }

                    //List<SelectClause> sclauses
                    bool NewSelectClauses = null == sclauses;
                    if (NewSelectClauses)
                    {
                        sclauses = new List<SelectClause>(Whats.Count);
                    }
                    for (int iww = 0; iww < Whats.Count; iww++)
                    {
                        int wi = Whats[iww];
                        string Source = awhat[iww];
                        string calls = null;
                        if (NewSelectClauses)
                        {
                            sclauses.Add(new SelectClause(agtools, cols));
                            calls = Source;
                        }
                        SelectClause sclause = sclauses[iww];
                        List<DbValue> results = sclause.ProcessSelectPart(calls, rows);
#if DEBUG
                            if(1 != results.Count && results.Count != outvalues.Count)
                            {
                                throw new Exception("DEBUG:  (WhatFunctions) && (1 != results.Count && results.Count != outvalues.Count)");
                            }
#endif

                        int outvaluesCount = outvalues.Count;
                        DbType rtype = DbType.PrepareNull(); // Just for init.
                        bool JustOneResult = 1 == results.Count;
                        bool DifferentSize = false;
                        for (int ir = 0; ir < outvaluesCount; ir++)
                        {
                            ByteSlice bs;
                            int ix = ir;
                            if (JustOneResult)
                            {
                                ix = 0;
                            }
                            bs = results[ix].Eval(out rtype);
                            bs.AppendTo(outvalues[ir]);
                            for (int vdiff = OutputColumnSizes[iww] - bs.Length; vdiff > 0; vdiff--)
                            {
                                DifferentSize = true;
                                outvalues[ir].Add(0);
                            }
                        }

                        if (GettingFieldTypeStrings)
                        {
                            if (DifferentSize)
                            {
                                rtype = DbType.Prepare(OutputColumnSizes[iww], rtype.ID);
                            }
                            FieldTypeStrings.Add(rtype.Name);
                        }

                    }

                    {
                        int outvaluesCount = outvalues.Count;
                        for (int iv = 0; iv < outvaluesCount; iv++)
                        {
                            output.Add(ByteSlice.Prepare(outvalues[iv]));
                            if (GroupBy)
                            {
                                break;
                            }
                        }
                    }

                    //if(null != agtools)
                    {
                        agtools.ResetBuffers(); // Last. Important!
                    }

                }

            }
コード例 #2
0
ファイル: QaDistinct.cs プロジェクト: erisonliang/qizmt
 public override void Reduce(ByteSlice key, IEnumerator<ByteSlice> values, MapReduceOutput output)
 {
     output.Add(key);
 }
コード例 #3
0
ファイル: QaJoinOn.cs プロジェクト: xwyangjshb/qizmt
            public override void Reduce(ByteSlice key, IEnumerator <ByteSlice> values, MapReduceOutput output)
            {
                if (JoinType.X == type)
                {
                    ftools = new DbFunctionTools();

                    string stype = DSpace_ExecArgs[1];

                    if (0 == string.Compare("INNER", stype, true))
                    {
                        type = JoinType.INNER_JOIN;
                    }
                    else if (0 == string.Compare("LEFT_OUTER", stype, true))
                    {
                        type = JoinType.LEFT_OUTER_JOIN;
                    }
                    else if (0 == string.Compare("RIGHT_OUTER", stype, true))
                    {
                        type = JoinType.RIGHT_OUTER_JOIN;
                    }
                    else
                    {
                        throw new NotSupportedException("DEBUG:  JOIN type not supported: " + stype);
                    }

                    t0 = new List <ByteSlice>();
                    t1 = new List <ByteSlice>();
                }

                t0.Clear();
                t1.Clear();

                while (values.MoveNext())
                {
                    ByteSlice row = values.Current;

                    int tableid = GetTableID(row);

                    if (0 == tableid)
                    {
                        t0.Add(ByteSlice.Prepare(row, 5, row.Length - 5));
                    }
                    else if (1 == tableid)
                    {
                        t1.Add(ByteSlice.Prepare(row, 5, row.Length - 5));
                    }
                    else
                    {
                        throw new Exception("Reduce: Unexpected TableID: " + tableid);
                    }
                }

                int t0Count = t0.Count;
                int t1Count = t1.Count;

                if (0 == t0Count)
                {
                    if (JoinType.RIGHT_OUTER_JOIN == type)
                    {
                        for (int j = 0; j < t1Count; j++)
                        {
                            ByteSlice   z        = t1[j];
                            List <byte> valuebuf = ftools.AllocBuffer(DSpace_OutputRecordLength);
                            for (int n = 0; n < DSpace_OutputRecordLength - z.Length; n++)
                            {
                                valuebuf.Add(1); // IsNull=true.
                            }
                            z.AppendTo(valuebuf);
                            output.Add(ByteSlice.Prepare(valuebuf));
                        }
                    }
                }
                else if (0 == t1Count)
                {
                    if (JoinType.LEFT_OUTER_JOIN == type)
                    {
                        for (int i = 0; i < t0Count; i++)
                        {
                            ByteSlice   z        = t0[i];
                            List <byte> valuebuf = ftools.AllocBuffer(DSpace_OutputRecordLength);
                            z.AppendTo(valuebuf);

                            for (int n = 0; n < DSpace_OutputRecordLength - z.Length; n++)
                            {
                                valuebuf.Add(1); // IsNull=true.
                            }

                            output.Add(ByteSlice.Prepare(valuebuf));
                        }
                    }
                }
                else
                {
                    for (int i = 0; i < t0Count; i++)
                    {
                        for (int j = 0; j < t1Count; j++)
                        {
                            ByteSlice x = t0[i];
                            ByteSlice y = t1[j];
#if DEBUG
                            if (DSpace_OutputRecordLength != x.Length + y.Length)
                            {
                                throw new Exception("DEBUG:  JoinOn.Reduce: (DSpace_OutputRecordLength != x.Length + y.Length).  x.Length="
                                                    + x.Length.ToString() + "; y.Length=" + y.Length.ToString() + ";DSpace_OutputRecordLength="
                                                    + DSpace_OutputRecordLength.ToString());
                            }
#endif
                            List <byte> valuebuf = ftools.AllocBuffer(DSpace_OutputRecordLength);
                            x.AppendTo(valuebuf);
                            y.AppendTo(valuebuf);
                            output.Add(ByteSlice.Prepare(valuebuf));
                        }
                    }
                }

                ftools.ResetBuffers();
            }
コード例 #4
0
ファイル: QaSelect.cs プロジェクト: xwyangjshb/qizmt
            public override void Reduce(ByteSlice key, IEnumerator <ByteSlice> values, MapReduceOutput output)
            {
#if DEBUG
                string KeyStringValue = "<not evaluated yet>";
                if (0 != ((key.Length - 1) % 2))
                {
                    KeyStringValue = "<not a string>";
                }
                else
                {
                    //KeyStringValue = System.Text.Encoding.Unicode.GetString(ByteSlice.Prepare(key, 1, key.Length - 1).ToBytes());
                    {
                        System.Text.Encoding ue = new System.Text.UnicodeEncoding(false, false, true);     // throwOnInvalidBytes=true
                        try
                        {
                            KeyStringValue = ue.GetString(ByteSlice.Prepare(key, 1, key.Length - 1).ToBytes());
                        }
                        catch
                        {
                            KeyStringValue = "<not a string>";
                        }
                    }
                }
#endif

                if (null == sOptions)
                {
                    // Done early to detect SFUNC
                    sOptions = (DSpace_ExecArgs.Length > 8) ? DSpace_ExecArgs[8] : "-";

                    WhatFunctions = -1 != sOptions.IndexOf("SFUNC");
                    GroupBy       = -1 != sOptions.IndexOf("GBY");
                }

                if (!WhatFunctions)
                {
                    // Normal: no functions (aggregates or scalars).
                    if (GroupBy)
                    {
                        values.Reset();
                        while (values.MoveNext())
                        {
                            output.Add(values.Current);
                            break;
                        }
                    }
                    else
                    {
                        values.Reset();
                        while (values.MoveNext())
                        {
                            output.Add(values.Current);
                        }
                    }
                }
                else //if(WhatFunctions)
                {
                    if (null == TableName)
                    {
                        InitFields(); // Note: only called if functions (aggregates or scalars)!

                        agtools = new DbFunctionTools();
                    }

                    bool GettingFieldTypeStrings = null == FieldTypeStrings;
                    if (GettingFieldTypeStrings)
                    {
                        FieldTypeStrings = new List <string>();
                    }

                    rows.Clear();
                    outvalues.Clear();
                    values.Reset();
                    while (values.MoveNext())
                    {
                        ByteSlice row = values.Current;
                        rows.Add(row);
                        outvalues.Add(agtools.AllocBuffer(row.Length > 256 ? Entry.Round2Power(row.Length) : 256));
                    }

                    //List<SelectClause> sclauses
                    bool NewSelectClauses = null == sclauses;
                    if (NewSelectClauses)
                    {
                        sclauses = new List <SelectClause>(Whats.Count);
                    }
                    for (int iww = 0; iww < Whats.Count; iww++)
                    {
                        int    wi     = Whats[iww];
                        string Source = awhat[iww];
                        string calls  = null;
                        if (NewSelectClauses)
                        {
                            sclauses.Add(new SelectClause(agtools, cols));
                            calls = Source;
                        }
                        SelectClause   sclause = sclauses[iww];
                        List <DbValue> results = sclause.ProcessSelectPart(calls, rows);
#if DEBUG
                        if (1 != results.Count && results.Count != outvalues.Count)
                        {
                            throw new Exception("DEBUG:  (WhatFunctions) && (1 != results.Count && results.Count != outvalues.Count)");
                        }
#endif

                        int    outvaluesCount = outvalues.Count;
                        DbType rtype          = DbType.PrepareNull(); // Just for init.
                        bool   JustOneResult  = 1 == results.Count;
                        bool   DifferentSize  = false;
                        for (int ir = 0; ir < outvaluesCount; ir++)
                        {
                            ByteSlice bs;
                            int       ix = ir;
                            if (JustOneResult)
                            {
                                ix = 0;
                            }
                            bs = results[ix].Eval(out rtype);
                            bs.AppendTo(outvalues[ir]);
                            for (int vdiff = OutputColumnSizes[iww] - bs.Length; vdiff > 0; vdiff--)
                            {
                                DifferentSize = true;
                                outvalues[ir].Add(0);
                            }
                        }

                        if (GettingFieldTypeStrings)
                        {
                            if (DifferentSize)
                            {
                                rtype = DbType.Prepare(OutputColumnSizes[iww], rtype.ID);
                            }
                            FieldTypeStrings.Add(rtype.Name);
                        }
                    }

                    {
                        int outvaluesCount = outvalues.Count;
                        for (int iv = 0; iv < outvaluesCount; iv++)
                        {
                            output.Add(ByteSlice.Prepare(outvalues[iv]));
                            if (GroupBy)
                            {
                                break;
                            }
                        }
                    }

                    //if(null != agtools)
                    {
                        agtools.ResetBuffers(); // Last. Important!
                    }
                }
            }
コード例 #5
0
        public String Mapreduced(Dictionary <string, string> fields, List <string> movement = null, List <string> objs = null, List <string> locations = null, List <string> users = null, Int64 start = 0, Int64 end = 0)
        {
            var orderfield = "folio";


            try
            {
                List <string> cols = new List <string>();
                foreach (var x in fields)
                {
                    cols.Add(x.Key);
                }
                cols.Add("CreatedTimeStamp");
                string[] arrayfields    = cols.ToArray();
                string[] arraymov       = movement.ToArray();
                string[] arrayobjs      = objs.ToArray();
                string[] arraylocations = locations.ToArray();
                string[] arrayusers     = users.ToArray();

                BsonArray bsonarraymov   = new BsonArray();
                BsonArray bsonarrayobjs  = new BsonArray();
                BsonArray bsonarrayloc   = new BsonArray();
                BsonArray bsonarrayusers = new BsonArray();
                for (int i = 0; i < arraymov.Length; i++)
                {
                    bsonarraymov.Add(arraymov[i]);
                }
                for (int i = 0; i < arrayobjs.Length; i++)
                {
                    bsonarrayobjs.Add(arrayobjs[i]);
                }
                for (int i = 0; i < arraylocations.Length; i++)
                {
                    bsonarrayloc.Add(arraylocations[i]);
                }
                for (int i = 0; i < arrayusers.Length; i++)
                {
                    bsonarrayusers.Add(arrayusers[i]);
                }


                List <BsonDocument> documents = new List <BsonDocument>();

                var query = Query.And(Query.In("movement", bsonarraymov), Query.In("object", bsonarrayobjs), Query.In("location", bsonarrayloc), Query.In("Creator", bsonarrayusers), Query.GTE("CreatedTimeStamp", start), Query.LTE("CreatedTimeStamp", end));

                /*Realiza la conexion */
                MongoCollection Demand    = conection.getCollection("Demand");
                MongoCollection Movements = conection1.getCollection("MovementProfiles");
                MongoCollection Locations = conection2.getCollection("Locations");

                var Movementsfunction = @"function() {
                var key = this._id.valueOf();
                var value={movement:this.name};
                 emit(key,value);
                   }";
                var Locationsfunction = @"function() {
                var key = this._id.valueOf();
                var value={location:this.name};
                 emit(key,value);
                   }";
                var Demandfunction    = @"function() {
                    var key = this.movement;
                     var value = {id:this._id.valueOf(),folio:this.folio,object:this.object,total:this.total,status:this.status,movementFields:this.movementFields,CreatedDate:this.CreatedDate,CreatedTimeStamp:this.CreatedTimeStamp,Creator:this.Creator,system_status:this.system_status};
                     emit(key,value);
                
                   }";
                var Demandfunction2   = @"function() {
                    var key = this.location;
                     var value = {id:this._id.valueOf(),folio:this.folio,object:this.object,total:this.total,status:this.status,movementFields:this.movementFields,CreatedDate:this.CreatedDate,CreatedTimeStamp:this.CreatedTimeStamp,Creator:this.Creator,system_status:this.system_status};
                     emit(key,value);
                
                   }";

                var reducefunction = @"function (key, values) {
               var reduced = {id:'',folio:'',movement:'',location:'',object:'',total:'',status:'',movementFields:'',CreatedDate:'',CreatedTimeStamp:'',Creator:'',system_status:''};
                var index=0;
              values.forEach(function(value){
              
          
                 
                    if(value.movement != null && reduced.movement ==''){
                    
                    reduced.movement = value.movement;
                  
                    }
                    if(value.location != null && reduced.location == ''){
                    
                    reduced.location = value.location;
                  
                    }
                   
                    if(value.id != null){
                       reduced.id= value.id;
                       reduced.folio = value.folio;
                      
                        reduced.object = value.object;
                        reduced.total = value.total;
                        reduced.status = value.status;
                        reduced.movementFields = value.movementFields;
                        reduced.CreatedDate = value.CreatedDate;
                        reduced.CreatedTimeStamp = value.CreatedTimeStamp;
                        reduced.Creator = value.Creator;
                        reduced.system_status = value.system_status;

                     
                    }
           
            });
           
            return reduced;
          }";

                /*Establece las propiedades a seguir del mapreduce */
                MapReduceOptionsBuilder options = new MapReduceOptionsBuilder();
                options.SetOutput(MapReduceOutput.Inline);
                options.SetOutput(MapReduceOutput.Reduce("result", true));



                /* Realiza los mapreduce sobre el mismo resultado */
                MapReduceResult result = Movements.MapReduce(Movementsfunction, reducefunction, options);
                // result = Locations.MapReduce(Locationsfunction, reducefunction, options);

                result = Demand.MapReduce(Demandfunction, reducefunction, options);
                //    result = Demand.MapReduce(Demandfunction2, reducefunction, options);


                /* resultado en json */
                string results = result.GetResults().ToJson();
                return(results);
            }
            catch (Exception ex)
            {
                return(null);
            }
        }
コード例 #6
0
ファイル: QaJoinOn.cs プロジェクト: erisonliang/qizmt
            public override void Reduce(ByteSlice key, IEnumerator<ByteSlice> values, MapReduceOutput output)
            {
                if (JoinType.X == type)
                {
                    ftools = new DbFunctionTools();

                    string stype = DSpace_ExecArgs[1];

                    if (0 == string.Compare("INNER", stype, true))
                    {
                        type = JoinType.INNER_JOIN;
                    }
                    else if (0 == string.Compare("LEFT_OUTER", stype, true))
                    {
                        type = JoinType.LEFT_OUTER_JOIN;
                    }
                    else if (0 == string.Compare("RIGHT_OUTER", stype, true))
                    {
                        type = JoinType.RIGHT_OUTER_JOIN;
                    }
                    else
                    {
                        throw new NotSupportedException("DEBUG:  JOIN type not supported: " + stype);
                    }

                    t0 = new List<ByteSlice>();
                    t1 = new List<ByteSlice>();
                }

                t0.Clear();
                t1.Clear();

                while (values.MoveNext())
                {
                    ByteSlice row = values.Current;

                    int tableid = GetTableID(row);

                    if (0 == tableid)
                    {
                        t0.Add(ByteSlice.Prepare(row, 5, row.Length - 5));
                    }
                    else if (1 == tableid)
                    {
                        t1.Add(ByteSlice.Prepare(row, 5, row.Length - 5));
                    }
                    else
                    {
                        throw new Exception("Reduce: Unexpected TableID: " + tableid);
                    }

                }

                int t0Count = t0.Count;
                int t1Count = t1.Count;

                if (0 == t0Count)
                {
                    if (JoinType.RIGHT_OUTER_JOIN == type)
                    {
                        for (int j = 0; j < t1Count; j++)
                        {
                            ByteSlice z = t1[j];
                            List<byte> valuebuf = ftools.AllocBuffer(DSpace_OutputRecordLength);
                            for (int n = 0; n < DSpace_OutputRecordLength - z.Length; n++)
                            {
                                valuebuf.Add(1); // IsNull=true.
                            }
                            z.AppendTo(valuebuf);
                            output.Add(ByteSlice.Prepare(valuebuf));
                        }

                    }
                }
                else if (0 == t1Count)
                {
                    if (JoinType.LEFT_OUTER_JOIN == type)
                    {
                        for (int i = 0; i < t0Count; i++)
                        {
                            ByteSlice z = t0[i];
                            List<byte> valuebuf = ftools.AllocBuffer(DSpace_OutputRecordLength);
                            z.AppendTo(valuebuf);

                            for (int n = 0; n < DSpace_OutputRecordLength - z.Length; n++)
                            {
                                valuebuf.Add(1); // IsNull=true.
                            }

                            output.Add(ByteSlice.Prepare(valuebuf));
                        }

                    }
                }
                else
                {
                    for (int i = 0; i < t0Count; i++)
                    {
                        for (int j = 0; j < t1Count; j++)
                        {
                            ByteSlice x = t0[i];
                            ByteSlice y = t1[j];
#if DEBUG
                            if (DSpace_OutputRecordLength != x.Length + y.Length)
                            {
                                throw new Exception("DEBUG:  JoinOn.Reduce: (DSpace_OutputRecordLength != x.Length + y.Length).  x.Length=" 
                                    + x.Length.ToString() + "; y.Length=" + y.Length.ToString() + ";DSpace_OutputRecordLength="
                                    + DSpace_OutputRecordLength.ToString());
                            }
#endif
                            List<byte> valuebuf = ftools.AllocBuffer(DSpace_OutputRecordLength);
                            x.AppendTo(valuebuf);
                            y.AppendTo(valuebuf);
                            output.Add(ByteSlice.Prepare(valuebuf));
                        }
                    }
                }

                ftools.ResetBuffers();
            }
コード例 #7
0
ファイル: QaDistinct.cs プロジェクト: xwyangjshb/qizmt
 public override void Reduce(ByteSlice key, IEnumerator <ByteSlice> values, MapReduceOutput output)
 {
     output.Add(key);
 }
コード例 #8
0
 public virtual void Reduce(ByteSlice key, IEnumerator <ByteSlice> values, MapReduceOutput output)
 {
 }
コード例 #9
0
ファイル: MapReduceTask.cs プロジェクト: wangchengqun/NCache
        public MapReduceTask(CacheBase parent, TaskCallback callback,
                             IMapper mapper,
                             ICombinerFactory combiner,
                             IReducerFactory reducer,
                             MapReduceInput inputProvider,
                             MapReduceOutput outputProvider,
                             Filter filter, CacheRuntimeContext context, int chunkSize, int maxExceps)
        {
            this._parent       = parent;
            this.Callback      = callback;
            this._context      = context;
            this.maxExceptions = maxExceps;

            this.mapperTask = new MapperTask(mapper, inputProvider, filter != null ? filter.KeyFilter : null, this);

            if (reducer != null)
            {
                ReducerConfigured = true;
                this.reducerTask  = new ReducerTask(reducer, this);
            }
            else
            {
                ReducerConfigured = true;
                this.reducerTask  = new ReducerTask(new IdentityReducerFactory(), this);
            }
            if (combiner != null)
            {
                CombinerConfigured = true;
                this.combinerTask  = new CombinerTask(combiner, this);
            }

            throttlingMgr       = new MapReduceThrottlingManager(chunkSize, this);
            this.outputProvider = outputProvider;

            if (_parent != null && _parent is LocalCacheImpl)
            {
                isLocal      = true;
                participants = new Hashtable();
                participants.Add(new Address(_parent.Context.Render.IPAddress, _parent.Context.Render.Port), new NodeTaskStatus());
            }
#if !CLIENT
            if (_parent != null && _parent is ClusterCacheBase)
            {
                ArrayList   list = ((ClusterCacheBase)_parent).ActiveServers;
                IEnumerator it   = list.GetEnumerator();
                participants = new Hashtable();
                while (it.MoveNext())
                {
                    participants.Add(it.Current, new NodeTaskStatus());
                }

                if (parent is PartitionedServerCache)
                {
                    distributionMgr = ((PartitionedServerCache)parent).DistributionMgr;
                }
            }
#endif

            //Queue initialization and synchronization
            Queue syncQueue = Queue.Synchronized(reducerDataQueue);
            this.reducerDataQueue = syncQueue;
        }