Exemplo n.º 1
0
 static async void DoMapReduce()
 {
     try
     {
         string mapFile      = File.ReadAllText(GetPath("MapReducer", "Map.js"));
         string ReduceFile   = File.ReadAllText(GetPath("MapReducer", "Reduce.js"));
         string FinalizeFile = File.ReadAllText(GetPath("MapReducer", "Finalize.js"));
         Console.WriteLine("Hello World!");
         MongoDB.Driver.MongoClient client = new MongoDB.Driver.MongoClient("mongodb://192.168.0.40:27088");
         var db          = client.GetDatabase("PEIU");
         var collections = db.GetCollection <BsonDocument>("daegun_meter");
         var options     = new MapReduceOptions <BsonDocument, BsonDocument>();
         options.JavaScriptMode = true;
         options.OutputOptions.ToJson();
         options.OutputOptions = MapReduceOutputOptions.Merge("statistics");
         var asyncResults = collections.MapReduce <BsonDocument>(mapFile, ReduceFile, options);
         var list         = asyncResults.ToList();
         foreach (var list_element in list)
         {
             Console.WriteLine(list_element.ToString());
         }
         Console.ReadKey();
     }
     catch (Exception ex)
     {
         Console.WriteLine(ex.Message);
     }
 }
Exemplo n.º 2
0
        private void DoMapreduce(object state)
        {
            try
            {
                MongoMapReduceTask map_reduce_task = (MongoMapReduceTask)state;
                logger.LogInformation($"Mongo MapReduce is working. {map_reduce_task.ResultDatabase}.{map_reduce_task.ResultCollection}.mapreduce()");

                string mapFile     = File.ReadAllText(map_reduce_task.GetMapFunctionFileName("MapReduceFunction"));
                string ReduceFile  = File.ReadAllText(map_reduce_task.GetReduceFunctionFileName("MapReduceFunction"));
                var    db          = client.GetDatabase(map_reduce_task.Database);
                var    collections = db.GetCollection <BsonDocument>(map_reduce_task.Collection);
                var    options     = new MapReduceOptions <BsonDocument, BsonDocument>();
                options.JavaScriptMode = true;
                options.OutputOptions.ToJson();
                options.OutputOptions = MapReduceOutputOptions.Merge(map_reduce_task.ResultCollection, map_reduce_task.ResultDatabase);
                var asyncResults = collections.MapReduce <BsonDocument>(mapFile, ReduceFile, options);
                if (asyncResults != null)
                {
                    logger.LogInformation($"Mongo MapReduce is completed. {map_reduce_task.ResultDatabase}.{map_reduce_task.ResultCollection}.mapreduce()");
                }
            }
            catch (Exception ex)
            {
                logger.LogError(ex, ex.Message);
            }
        }
Exemplo n.º 3
0
        public IList <BsonDocument> MapReduce(string _map, string _reduce, string collName, string fileter, string outputResult, int timeOutSeconds = 60, bool isDel = false)
        {
            IList <BsonDocument> result = new List <BsonDocument>();
            var map = new BsonJavaScript(_map);

            var reduce = new BsonJavaScript(_reduce);

            //const key = this.Date.getFullYear();

            if (isDel && !string.IsNullOrEmpty(outputResult))
            {
                //删除结果集合
                mDB.DropCollection(outputResult);
            }
            var coll = mDB.GetCollection <BsonDocument>(collName);
            //var options = new MapReduceOptions<BsonDocument, BsonDocument>();//what should be TResult?

            //options.OutputOptions = MapReduceOutputOptions.Inline;
            //查询条件
            FilterDefinition <BsonDocument> _filter = null;

            if (!string.IsNullOrEmpty(fileter))
            {
                _filter = fileter;
            }

            //输出结果集合
            var output = MapReduceOutputOptions.Reduce(outputResult, nonAtomic: true);

            if (string.IsNullOrEmpty(outputResult))
            {
                output = MapReduceOutputOptions.Inline;
            }

            MapReduceOptions <BsonDocument, BsonDocument> options = new MapReduceOptions <BsonDocument, BsonDocument>
            {
                Filter        = _filter,
                MaxTime       = TimeSpan.FromSeconds(timeOutSeconds),
                OutputOptions = output,
                Verbose       = true,
            };

            try
            {
                result = coll.MapReduce(map, reduce, options).ToList();
            }
            catch (Exception ex)
            {
                Console.WriteLine($"Exception occurred {ex.Message}");
            }

            return(result);
        }
Exemplo n.º 4
0
        public async Task DemoMapReduceAsync(IMongoCollection <ClubMember> collection)
        {
            //The match function has to be written in JavaScript
            //the map method outputs a key value pair for each document
            //in this case the key is the Lastname property value and the value is the Age property value
            var map = new BsonJavaScript(@"function() 
                                            {
                                             //Associate each LastName property with the Age value
                                               emit(this.Lastname,this.Age);
                                             }");
            //The MapReduce method  uses the output from the Map method
            //   to produce a list of ages for each unique LastName value.
            //  Then each key and its associated list of values is presented to the Reduce method in turn.
            var reduce = new BsonJavaScript(@"function(lastName,ages) 
                                             {
                                                 return Array.sum(ages);
                                              }");
            //The Reduce method returns the Lastname as the key and the sum of the ages as the value
            //The beauty of this technique is that data can be processed in batches
            //The output of one batch is combined with that of another and fed back through the Reducer
            //This is repeated until the output is reduced to the number of unique keys.
            // The results are output to a new collection named ResultsCollection on the server.
            // This saves on the use of computer memory and enables the results to be queried effectively by using indexes.

            var options = new MapReduceOptions <ClubMember, BsonDocument>
            {
                //Replace (the default) : the content of existing collection is dropped, and the new output go into it
                // Merge  The existing collection is kept.When a result with the same Key(the _id) exists, the value in the collection is replaced with it.
                //Reduce : The existing collection is kept.When a result with the same Key(the _id) exists,
                //MongoDB runs the reduce function against the two of them, and stores the result.
                OutputOptions = MapReduceOutputOptions.Reduce("ResultsCollection")
            };
            var resultAsBsonDocumentList = await collection.MapReduce(map, reduce, options).ToListAsync();

            Console.WriteLine("The total age for every member of each family  is ....");
            var reduction =
                resultAsBsonDocumentList.Select(
                    doc => new { family = doc["_id"].AsString, age = (int)doc["value"].AsDouble });

            foreach (var anon in reduction)
            {
                Console.WriteLine("{0} Family Total Age {1}", anon.family, anon.age);
            }
        }
Exemplo n.º 5
0
        //public static void test()
        //{

        //    var coll = mDB.GetCollection<ProtocolByPub>("publog");



        //    string mapFunction = @"function(){
        //                            emit(this.cusid, this.price);
        //                        };";

        //    string reduceFunction = @"function(cusid, prices){
        //                            var total = 0;
        //                            total = Array.sum(prices);
        //                            return { sum: total };
        //                        };";


        //    FilterDefinition<BsonDocument> filter = "{ x : { $regex : /ABC/ } }";
        //    var options = new MapReduceOptions<BsonDocument, ProtocolByPub>();//what should be TResult?
        //    options.Filter = filter;
        //    options.OutputOptions = MapReduceOutputOptions.Inline;
        //    var cusid_prices_results = coll.MapReduce<ProtocolByPub>(mapFunction, reduceFunction);
        //}
        //this.Date.getFullYear()
        //        private const string MapJs = @"function mapF() {
        //    const key = this.Date.getFullYear();
        //    const valuePerYear = { total: 1};

        //    emit(key, valuePerYear);
        //}; ";

        //        private const string ReduceJS = @"function reduceF(year, values) {
        //    var sum = 0;
        //    values.forEach(function(v) {
        //        sum += v.total;
        //    });
        //    return {total: NumberInt(sum)};
        //}";


        public void test1()
        {
            string MapJs    = @"function mapF() {
    const key = this.firstName;
    const ages = this.age;

    emit(key, ages);
}; ";
            string ReduceJS = @"function reduceF(key, ages) {
    var sum = 0;
    ages.forEach(function(v) {
        sum += v.age;
    });
    return NumberInt(sum);
}";

            //string mongoConnectionString = "my-connection-string";

            IMongoCollection <BsonDocument> collection = mDB.GetCollection <BsonDocument>("test2");
            BsonJavaScript map    = new BsonJavaScript(MapJs);
            BsonJavaScript reduce = new BsonJavaScript(ReduceJS);
            FilterDefinitionBuilder <BsonDocument> filterBuilder = new FilterDefinitionBuilder <BsonDocument>();
            var f = filterBuilder.Eq("age", 12);
            //     FilterDefinition<BsonDocument> filter = new BsonDocument{
            //  {"_id",12}
            //};
            MapReduceOptions <BsonDocument, BsonDocument> options = new MapReduceOptions <BsonDocument, BsonDocument>
            {
                Filter        = f,
                MaxTime       = TimeSpan.FromMinutes(1),
                OutputOptions = MapReduceOutputOptions.Reduce("Result", nonAtomic: true),
                Verbose       = true
            };

            try
            {
                var result = collection.MapReduce(map, reduce, options).ToList();
            }
            catch (Exception ex)
            {
                Console.WriteLine($"Exception occurred {ex.Message}");
            }
        }
Exemplo n.º 6
0
        static void Main(string[] args)
        {
            var client   = new MongoDB.Driver.MongoClient("mongodb://localhost");
            var database = client.GetDatabase("MapReduce");

            if (!database.ListCollections().Any())
            {
                SeedOrders(database);
                SeedCustomers(database);
            }

            var map     = GetFunction("CustomerMapFunction");
            var reduce  = GetFunction("CustomerReduceFunction");
            var options = new MapReduceOptions <Customer, BsonDocument>();

            options.Filter = Builders <Customer> .Filter.Eq(o => o.CustomerId, 5000);

            options.Verbose       = false;
            options.OutputOptions = MapReduceOutputOptions.Replace("Out");
            database.GetCollection <Customer>(CUSTOMER_COLLECTION).MapReduce <BsonDocument>(new BsonJavaScript(map), new BsonJavaScript(reduce), options);


            var map2     = GetFunction("OrderMapFunction");
            var reduce2  = GetFunction("OrderReduceFunction");
            var options2 = new MapReduceOptions <Order, BsonDocument>();

            options2.Filter = Builders <Order> .Filter.Eq(o => o.CustomerId, 5000);

            options2.Verbose       = false;
            options2.OutputOptions = MapReduceOutputOptions.Reduce("Out");
            var res = database.GetCollection <Order>(ORDER_COLLECTION).MapReduce <BsonDocument>(new BsonJavaScript(map2), new BsonJavaScript(reduce2), options2);

            var output = database.GetCollection <BsonDocument>("Out");

            foreach (var doc in output.Find(b => true).ToList())
            {
                Console.WriteLine();
                Console.WriteLine(doc.ToJson());
            }

            Console.Read();
        }
Exemplo n.º 7
0
        /// <summary>
        ///
        /// </summary>
        /// <param name="donutScript">The reduce script to execute.</param>
        /// <param name="inputDocumentsLimit"></param>
        /// <param name="orderBy"></param>
        /// <returns></returns>
        public async Task Reduce(string donutScript, uint inputDocumentsLimit = 0, SortDefinition <BsonDocument> orderBy = null)
        {
            MapReduceExpression mapReduce = new DonutSyntaxReader(new PrecedenceTokenizer(new DonutTokenDefinitions())
                                                                  .Tokenize(donutScript)).ReadMapReduce();
            MapReduceJsScript script = MapReduceJsScript.Create(mapReduce);
            var targetCollection     = OutputDestinationCollection.ReducedOutputCollection;
            var mapReduceOptions     = new MapReduceOptions <BsonDocument, BsonDocument>
            {
                Sort           = orderBy,
                JavaScriptMode = true,
                OutputOptions  = MapReduceOutputOptions.Replace(targetCollection)
            };

            if (inputDocumentsLimit > 0)
            {
                mapReduceOptions.Limit = inputDocumentsLimit;
            }
            var collection = MongoHelper.GetCollection(OutputDestinationCollection.OutputCollection);
            await collection.MapReduceAsync <BsonDocument>(script.Map, script.Reduce, mapReduceOptions);
        }
Exemplo n.º 8
0
        private void CalculateFastK_MR(int dataCount)
        {
            if (dataCount < NumberOfData + FastKPeriod - 1)
            {
                NumberOfData = dataCount - FastKPeriod + 1;
            }

            var limit  = FastKPeriod + NumberOfData - 1;
            var filter = new BsonDocument {
                { "Kod", Code }, { "Tarih", new BsonDocument("$lte", new BsonDateTime(TargetDate)) }
            };
            var sort       = new BsonDocument("Tarih", -1);
            var projection = new BsonDocument {
                { "_id", 0 }, { "Tarih", 1 }
            };

            BsonJavaScript mapper = new BsonJavaScript(@"
                function() {
	                if (!equalityFunction(this.Tarih, limitDate) && this.Tarih < limitDate)
		                return;
	                else if (!equalityFunction(this.Tarih, lastDate) && this.Tarih > lastDate)
		                return;
	
	                var dateIndex = binarySearch(dates, this.Tarih);

	                if (dateIndex == -1)
		                return;
	
	                for (var i = 0; i < period && dateIndex >= 0; i++, dateIndex--) {
		                if (dates[dateIndex] > startingDate || equalityFunction(dates[dateIndex], startingDate))
			                emit(dates[dateIndex], {close: this.Kapanis, low: this.Dusuk, high: this.Yuksek});
	                }
                }
            ");

            BsonJavaScript reducer = new BsonJavaScript(@"
                function(key, values) {
                    var result = {close: values[0].close, low: values[0].low, high: values[0].high};
                    for (var i = 1; i < values.length; i++) {
                        result.low = values[i].low < result.low ? values[i].low : result.low;
                        result.high = values[i].high > result.high ? values[i].high : result.high;
                    }
                    return result;
                }
            ");

            BsonJavaScript finalizer = new BsonJavaScript(@"
                function(key, reducedValue) {
                    reducedValue = 100 * (reducedValue.close - reducedValue.low) / (reducedValue.high - reducedValue.low);
                    return reducedValue;
                }
            ");

            var startingDate = MongoDBService.GetService().FindOneSortProjectSkip(filter, sort, projection, NumberOfData - 1).GetElement(0).Value.ToLocalTime();
            var lastDate     = MongoDBService.GetService().FindOneSortProject(filter, sort, projection).GetElement(0).Value.ToLocalTime();
            var limitDate    = MongoDBService.GetService().FindOneSortProjectSkip(filter, sort, projection, limit - 1).GetElement(0).Value.ToLocalTime();
            var dates        = IndicatorService.GetData(Code, TargetDate, "Tarih", limit).Select(p => p.GetElement(0).Value.ToLocalTime()).ToArray();
            var scope        = new BsonDocument {
                { "startingDate", startingDate }, { "lastDate", lastDate }, { "limitDate", limitDate }, { "period", FastKPeriod }, { "numberOfData", NumberOfData }, { "binarySearch", MapReduceHelpers.BinarySearch }, { "equalityFunction", MapReduceHelpers.IsDatesEqual }, { "dates", new BsonArray(dates) }
            };

            var options = new MapReduceOptions <BsonDocument, BsonDocument>()
            {
                Filter        = filter,
                Sort          = sort,
                Scope         = scope,
                OutputOptions = MapReduceOutputOptions.Replace("stochasticsOut", "financialData", false),
                Finalize      = finalizer
            };

            FastK = new double[NumberOfData];
            var resultSet = MongoDBService.GetService().MapReduceMany(mapper, reducer, options);

            for (int i = 0, j = resultSet.Count - 1; i < NumberOfData; i++, j--)
            {
                FastK[i] = resultSet.ElementAt(j).GetElement(1).Value.ToDouble();
            }
        }
Exemplo n.º 9
0
        public static double[] WsrMR(string code, DateTime targetDate, int period = 14, int numberOfData = 1)
        {
            if (period <= 0)
            {
                throw new IndicatorException("Period must be positive.");
            }
            else if (numberOfData <= 0)
            {
                throw new IndicatorException("Number of data to be processed must be positive.");
            }

            int dataCount = IndicatorService.DataCount(code, targetDate);

            if (dataCount < period)
            {
                throw new IndicatorException(IndicatorException.DATA_NOT_ENOUGH_MESSAGE);
            }

            if (dataCount < numberOfData + period - 1)
            {
                numberOfData = dataCount - period + 1;
            }

            var limit  = period + numberOfData - 1;
            var filter = new BsonDocument {
                { "Kod", code }, { "Tarih", new BsonDocument("$lte", new BsonDateTime(targetDate)) }
            };
            var sort       = new BsonDocument("Tarih", -1);
            var projection = new BsonDocument {
                { "_id", 0 }, { "Tarih", 1 }
            };

            BsonJavaScript mapper = new BsonJavaScript(@"
                function() {
	                if (!equalityFunction(this.Tarih, limitDate) && this.Tarih < limitDate)
		                return;
	                else if (!equalityFunction(this.Tarih, lastDate) && this.Tarih > lastDate)
		                return;
	
	                var dateIndex = binarySearch(dates, this.Tarih);

	                if (dateIndex == -1)
		                return;
	
	                for (var i = 0; i < period && dateIndex >= 0; i++, dateIndex--) {
		                if (dates[dateIndex] > startingDate || equalityFunction(dates[dateIndex], startingDate))
			                emit(dates[dateIndex], {close: this.Kapanis, low: this.Dusuk, high: this.Yuksek});
	                }
                }
            ");

            BsonJavaScript reducer = new BsonJavaScript(@"
                function(key, values) {
                    var result = {close: values[0].close, low: values[0].low, high: values[0].high};
                    for (var i = 1; i < values.length; i++) {
                        result.low = values[i].low < result.low ? values[i].low : result.low;
                        result.high = values[i].high > result.high ? values[i].high : result.high;
                    }
                    return result;
                }
            ");

            BsonJavaScript finalizer = new BsonJavaScript(@"
                function(key, reducedValue) {
                    reducedValue = -100 * (reducedValue.high - reducedValue.close) / (reducedValue.high - reducedValue.low);
                    return reducedValue;
                }
            ");

            var startingDate = MongoDBService.GetService().FindOneSortProjectSkip(filter, sort, projection, numberOfData - 1).GetElement(0).Value.ToLocalTime();
            var lastDate     = MongoDBService.GetService().FindOneSortProject(filter, sort, projection).GetElement(0).Value.ToLocalTime();
            var limitDate    = MongoDBService.GetService().FindOneSortProjectSkip(filter, sort, projection, limit - 1).GetElement(0).Value.ToLocalTime();
            var dates        = IndicatorService.GetData(code, targetDate, "Tarih", limit).Select(p => p.GetElement(0).Value.ToLocalTime()).ToArray();
            var scope        = new BsonDocument {
                { "startingDate", startingDate }, { "lastDate", lastDate }, { "limitDate", limitDate }, { "period", period }, { "numberOfData", numberOfData }, { "binarySearch", MapReduceHelpers.BinarySearch }, { "equalityFunction", MapReduceHelpers.IsDatesEqual }, { "dates", new BsonArray(dates) }
            };

            var options = new MapReduceOptions <BsonDocument, BsonDocument>()
            {
                Filter        = filter,
                Sort          = sort,
                Scope         = scope,
                OutputOptions = MapReduceOutputOptions.Replace("wsrOut", "financialData", false),
                Finalize      = finalizer
            };

            double[] wsr       = new double[numberOfData];
            var      resultSet = MongoDBService.GetService().MapReduceMany(mapper, reducer, options);

            for (int i = 0, j = resultSet.Count - 1; i < numberOfData; i++, j--)
            {
                wsr[i] = resultSet.ElementAt(j).GetElement(1).Value.ToDouble();
            }

            return(wsr);
        }
        public static double[] RsiMR(string code, DateTime targetDate, int period = 14, int numberOfData = 1)
        {
            if (period <= 0)
            {
                throw new IndicatorException("Periyot pozitif sayı olmalıdır.");
            }
            else if (numberOfData <= 0)
            {
                throw new IndicatorException("Gösterilecek veri sayısı pozitif sayı olmalıdır.");
            }

            int dataCount = IndicatorService.DataCount(code, targetDate);

            if (dataCount < period)
            {
                throw new IndicatorException(IndicatorException.DATA_NOT_ENOUGH_MESSAGE);
            }

            if (dataCount < numberOfData + period - 1)
            {
                numberOfData = dataCount - period + 1;
            }

            int limit = numberOfData + period - 1;

            var filter = new BsonDocument {
                { "Kod", code }, { "Tarih", new BsonDocument("$lte", new BsonDateTime(targetDate)) }
            };
            var sort = new BsonDocument("Tarih", -1);

            var projection = new BsonDocument {
                { "_id", 0 }, { "Tarih", 1 }
            };
            var startingDate = MongoDBService.GetService().FindOneSortProjectSkip(filter, sort, projection, numberOfData - 1).GetElement(0).Value.ToLocalTime();
            var lastDate     = MongoDBService.GetService().FindOneSortProject(filter, sort, projection).GetElement(0).Value.ToLocalTime();
            var limitDate    = MongoDBService.GetService().FindOneSortProjectSkip(filter, sort, projection, limit - 1).GetElement(0).Value.ToLocalTime();
            var dates        = IndicatorService.GetData(code, targetDate, "Tarih", limit).Select(p => p.GetElement(0).Value.ToLocalTime()).ToArray();

            MapReduceOptions <BsonDocument, BsonDocument> options = new MapReduceOptions <BsonDocument, BsonDocument>
            {
                Filter = new BsonDocument {
                    { "Kod", code }, { "Tarih", new BsonDocument("$lte", new BsonDateTime(targetDate)) }
                },
                Sort  = new BsonDocument("Tarih", -1),
                Scope = new BsonDocument {
                    { "startingDate", startingDate }, { "lastDate", lastDate }, { "limitDate", limitDate }, { "period", period }, { "numberOfData", numberOfData }, { "binarySearch", MapReduceHelpers.BinarySearch }, { "equalityFunction", MapReduceHelpers.IsDatesEqual }, { "dates", new BsonArray(dates) }
                },
                OutputOptions = MapReduceOutputOptions.Replace("rsiOut", "financialData", false)
            };

            double[]       avg;
            BsonJavaScript mapper = new BsonJavaScript(@"
                function() {
                    if (!equalityFunction(this.Tarih, limitDate) && this.Tarih < limitDate)
                        return;
                    else if (!equalityFunction(this.Tarih, lastDate) && this.Tarih > lastDate)
                        return;

                    var dateIndex;

                    dateIndex = binarySearch(dates, this.Tarih);
                    if (dateIndex == -1)
                        return;

                    for (var i = 0; i < period && dateIndex >= 0; i++, dateIndex--) {
                        if (dates[dateIndex] > startingDate || equalityFunction(dates[dateIndex], startingDate))
                            emit(dates[dateIndex], {date: this.Tarih, headClose : this.Kapanis, tailClose : this.Kapanis, gain: 0 , loss: 0});
                    }
                }
            ");

            BsonJavaScript reducer = new BsonJavaScript(@"
                function(key, values) {
	                values.sort(function(a, b) {
		                return a.date < b.date;
	                });

                    sumGain = values[0].gain; 
                    sumLoss = values[0].loss;
                    var diff;
    
                    for(var i = 0; i < values.length - 1; i++) {
                        sumGain += values[i+1].gain;   
                        sumLoss += values[i+1].loss;
                        diff = values[i].tailClose - values[i+1].headClose;
                        if(diff > 0) 
                            sumGain += diff;
                        else 
                            sumLoss -= diff;
                    }

                    return {date: values[0].date, headClose: values[0].headClose, tailClose: values[values.length-1].tailClose, gain: sumGain, loss: sumLoss};
                }
            ");

            BsonJavaScript finalizer = new BsonJavaScript(@"
                function(key, reducedValue) {
                    if (reducedValue.loss == 0) return 100;
                    reducedValue = 100 - 100 / (1 + (reducedValue.gain / reducedValue.loss));
                    return reducedValue;
                }
            ");

            options.Finalize = finalizer;

            List <BsonDocument> resultSet = MongoDBService.GetService().MapReduceMany(mapper, reducer, options);

            avg = new double[numberOfData];

            for (int i = 0, j = numberOfData - 1; i < numberOfData; i++, j--)
            {
                avg[i] = resultSet.ElementAt(j).GetElement(1).Value.ToDouble();
            }

            return(avg);
        }
Exemplo n.º 11
0
        public object mapReduce(object map, object reduce, object options)
        {
            var mapReduceOptions = new MapReduceOptions <BsonDocument, BsonDocument>();

            object sort = options.GetProperty("sort");

            if (sort != null)
            {
                mapReduceOptions.Sort = new BsonDocumentSortDefinition <BsonDocument>(sort.ToBsonDocument());
            }

            object query = options.GetProperty("query");

            if (query != null)
            {
                mapReduceOptions.Filter = new BsonDocumentFilterDefinition <BsonDocument>(query.ToBsonDocument());
            }

            object scope = options.GetProperty("scope");

            if (scope != null)
            {
                mapReduceOptions.Scope = scope.ToBsonDocument();
            }

            object limit = options.GetProperty("limit");

            if (limit != null)
            {
                mapReduceOptions.Limit = limit.ToIntOrDefault();
            }

            object verbose = options.GetProperty("verbose");

            if (limit != null)
            {
                mapReduceOptions.Verbose = verbose.ToBoolOrDefault();
            }

            object jsMode = options.GetProperty("jsMode");

            if (jsMode != null)
            {
                mapReduceOptions.JavaScriptMode = jsMode.ToBoolOrDefault();
            }

            object bypassDocumentValidation = options.GetProperty("bypassDocumentValidation");

            if (bypassDocumentValidation != null)
            {
                mapReduceOptions.BypassDocumentValidation = bypassDocumentValidation.ToBoolOrDefault();
            }

            object finalize = options.GetProperty("finalize");

            if (finalize != null)
            {
                mapReduceOptions.Finalize = new BsonJavaScript(finalize.ToStringOrDefault());
            }

            object outCollection = options.GetProperty("out");

            if (outCollection is string)
            {
                mapReduceOptions.OutputOptions = MapReduceOutputOptions.Replace(outCollection.ToStringOrDefault());
            }
            if (outCollection is IDictionary <string, object> )
            {
                var    outDicionary = (IDictionary <string, object>)outCollection;
                object inline       = outDicionary.GetValueOrDefault2("inline");
                if (inline != null)
                {
                    mapReduceOptions.OutputOptions = MapReduceOutputOptions.Inline;
                }
                else
                {
                    string replaceAction = outDicionary.GetValueOrDefault2("replace").ToStringOrDefault();
                    string mergeAction   = outDicionary.GetValueOrDefault2("merge").ToStringOrDefault();
                    string reduceAction  = outDicionary.GetValueOrDefault2("reduce").ToStringOrDefault();

                    string db        = outDicionary.GetValueOrDefault2("db").ToStringOrDefault();
                    bool?  nonAtomic = outDicionary.GetValueOrDefault2("nonAtomic").ToBoolOrDefault();
                    bool?  sharded   = outDicionary.GetValueOrDefault2("sharded").ToBoolOrDefault();

                    if (replaceAction != null)
                    {
                        mapReduceOptions.OutputOptions = MapReduceOutputOptions.Replace(replaceAction, db, sharded);
                    }
                    if (mergeAction != null)
                    {
                        mapReduceOptions.OutputOptions = MapReduceOutputOptions.Merge(mergeAction, db, sharded, nonAtomic);
                    }
                    if (reduceAction != null)
                    {
                        mapReduceOptions.OutputOptions = MapReduceOutputOptions.Reduce(reduceAction, db, sharded, nonAtomic);
                    }
                }
            }

            return(new MongoEvaluatedCursor <BsonDocument>(
                       _collection.MapReduce <BsonDocument>(
                           new BsonJavaScript(map.ToStringOrDefault()),
                           new BsonJavaScript(reduce.ToStringOrDefault()),
                           mapReduceOptions)
                       ));
        }
        public async void GroupByAccountFromCommand(string companyName)
        {
            try
            {
                ConsoleLogger.LogInformation(
                    string.Concat("Calculation Test: Mapreduce on Sharded Cluster: ",
                                  companyName, " Started."));

                const string reduceCollectionName = "calculationresults";

                var accountMapScript       = Helpers.GetScriptContent(@"SampleMongoScripts\MapReduce\accountMap.mongojs");
                var accountFactorMapScript =
                    Helpers.GetScriptContent(@"SampleMongoScripts\MapReduce\accountFactorMap.mongojs");
                var reduceScript   = Helpers.GetScriptContent(@"SampleMongoScripts\MapReduce\reduce.mongojs");
                var finalizeScript = Helpers.GetScriptContent(@"SampleMongoScripts\MapReduce\finalize.mongojs");

                var mapReduceAggregationScript =
                    File.ReadAllText(Path.Combine(Directory.GetCurrentDirectory(),
                                                  @"SampleMongoScripts\MapReduce\aggregation.mongojs"))
                    .Replace("[[TargetCollection]]", reduceCollectionName)
                    .Replace("[[CompanyName]]", companyName);


                var timer = Stopwatch.StartNew();

                Task.Run(async() =>
                {
                    await MongoDataContext.Repository <Account>()
                    .GetCollection()
                    .MapReduceAsync(accountMapScript, reduceScript,
                                    new MapReduceOptions <Account, BsonDocument>
                    {
                        Finalize      = finalizeScript,
                        OutputOptions =
                            MapReduceOutputOptions.Reduce(reduceCollectionName,
                                                          MongoDataContext.Database.DatabaseNamespace.DatabaseName)
                    });

                    await MongoDataContext.Repository <AccountFactor>()
                    .GetCollection()
                    .MapReduceAsync(accountFactorMapScript, reduceScript,
                                    new MapReduceOptions <AccountFactor, BsonDocument>
                    {
                        Finalize      = finalizeScript,
                        OutputOptions =
                            MapReduceOutputOptions.Reduce(reduceCollectionName,
                                                          MongoDataContext.Database.DatabaseNamespace.DatabaseName)
                    });
                    ConsoleLogger.LogWarning(string.Concat("Mapreduce finished. New collection created: ",
                                                           reduceCollectionName));
                }).Wait();

                ConsoleLogger.LogWarning(string.Concat("Document count for ", reduceCollectionName, ": ",
                                                       MongoDataContext.Database.GetCollection <BsonDocument>(reduceCollectionName)
                                                       .AsQueryable()
                                                       .Count()
                                                       .ToString("N0")));

                var result =
                    await
                    MongoDataContext.Database.GetCommandResultAsync <MapReduceCalculationResult>(
                        mapReduceAggregationScript);

                timer.Stop();

                Console.WriteLine(string.Concat(Environment.NewLine, "### MapreduceOnShardedCluster >> ",
                                                companyName, " ###"));
                ConsoleTable.From(result).Write(Format.Alternative);

                var timespan = timer.Elapsed;
                ConsoleLogger.LogInformation(
                    string.Concat("Calculation Test: Mapreduce on Sharded Cluster: ",
                                  companyName, " Finished. Duration: ",
                                  string.Format("{0:00}:{1:00}:{2:00}", timespan.Minutes, timespan.Seconds,
                                                timespan.Milliseconds / 10)));

                _testCounter++;
                ConsoleLogger.LogWarning(string.Concat("Test(s) left: ", _testCounter, "/", _testCount));
            }
            catch (Exception ex)
            {
                ConsoleLogger.LogCritical(ex.GetAllMessages());
            }
        }
Exemplo n.º 13
0
        public static double[] ExponentialMR(string code, DateTime targetDate, int period = 14, int numberOfData = 30)
        {
            if (period <= 0)
            {
                throw new IndicatorException("Period must be positive.");
            }

            int dataCount = IndicatorService.DataCount(code, targetDate);

            if (dataCount < numberOfData)
            {
                numberOfData = dataCount;
            }

            double alpha = 2.0 / (1 + period);
            double beta  = 1 - alpha;
            double minimumCoefficient = 0.001;

            BsonJavaScript mapper = new BsonJavaScript(@"
                function() {
	                if (!equalityFunction(this.Tarih, limitDate) && this.Tarih < limitDate)
		                return;
	                else if (!equalityFunction(this.Tarih, lastDate) && this.Tarih > lastDate)
		                return;
	                var dateIndex = binarySearch(dates, this.Tarih);
	                if (dateIndex == -1)
		                return;
	                
                    var epoch = Math.log(minimumCoefficient / (alpha * this.Kapanis)) / Math.log(beta);
                    var value;
	                if (dateIndex == numberOfData-1) {
		                value = this.Kapanis;
		                for (var i = dateIndex, j = 0; j < epoch && i >= 0; i--, j++) {
			                emit(dates[i], value);
			                value = value * beta;
		                }
	                } else {
		                value = this.Kapanis * alpha;
		                for (var i = dateIndex, j = 0; j < epoch && i >= 0; i--, j++) {
			                emit(dates[i], value);
			                value = value * beta;
		                }
	                }
                }
            ");

            BsonJavaScript reducer = new BsonJavaScript(@"
                function(key,values){
                    return Array.sum(values);
                }
            ");

            var filter = new BsonDocument {
                { "Kod", code }, { "Tarih", new BsonDocument("$lte", new BsonDateTime(targetDate)) }
            };
            var sort         = new BsonDocument("Tarih", -1);
            var mongoService = MongoDBService.GetService();

            var projection = new BsonDocument {
                { "_id", 0 }, { "Tarih", 1 }
            };
            var lastDate  = MongoDBService.GetService().FindOneSortProject(filter, sort, projection).GetElement(0).Value.ToLocalTime();
            var limitDate = MongoDBService.GetService().FindOneSortProjectSkip(filter, sort, projection, numberOfData - 1).GetElement(0).Value.ToLocalTime();
            var dates     = IndicatorService.GetData(code, targetDate, "Tarih", numberOfData).Select(p => p.GetElement(0).Value.ToLocalTime()).ToArray();
            var scope     = new BsonDocument {
                { "numberOfData", numberOfData }, { "minimumCoefficient", minimumCoefficient }, { "alpha", alpha }, { "beta", beta }, { "lastDate", lastDate }, { "limitDate", limitDate }, { "dates", new BsonArray(dates) }, { "binarySearch", MapReduceHelpers.BinarySearch }, { "equalityFunction", MapReduceHelpers.IsDatesEqual }
            };

            MapReduceOptions <BsonDocument, BsonDocument> options = new MapReduceOptions <BsonDocument, BsonDocument>
            {
                Filter        = filter,
                Sort          = sort,
                Scope         = scope,
                OutputOptions = MapReduceOutputOptions.Replace("emaOut", "financialData", false)
            };

            double[] values = MongoDBService.GetService().MapReduceMany(mapper, reducer, options).Select(p => p.GetElement("value").Value.ToDouble()).ToArray();
            Array.Reverse(values);
            return(values);
        }
Exemplo n.º 14
0
        public static double[] WeightedMR(string code, DateTime targetDate, int period = 14, int numberOfData = 30)
        {
            if (period <= 0)
            {
                throw new IndicatorException("Periyot pozitif sayı olmalıdır.");
            }
            else if (numberOfData <= 0)
            {
                throw new IndicatorException("Gösterilecek veri sayısı pozitif sayı olmalıdır.");
            }

            int dataCount = IndicatorService.DataCount(code, targetDate);

            if (dataCount < period)
            {
                throw new IndicatorException(IndicatorException.DATA_NOT_ENOUGH_MESSAGE);
            }

            if (dataCount < numberOfData + period - 1)
            {
                numberOfData = dataCount - period + 1;
            }

            int    limit       = numberOfData + period - 1;
            double denominator = (period * (period + 1)) >> 1;

            var filter = new BsonDocument {
                { "Kod", code }, { "Tarih", new BsonDocument("$lte", new BsonDateTime(targetDate)) }
            };
            var sort = new BsonDocument("Tarih", -1);

            var projection = new BsonDocument {
                { "_id", 0 }, { "Tarih", 1 }
            };
            var startingDate = MongoDBService.GetService().FindOneSortProjectSkip(filter, sort, projection, numberOfData - 1).GetElement(0).Value.ToLocalTime();
            var lastDate     = MongoDBService.GetService().FindOneSortProject(filter, sort, projection).GetElement(0).Value.ToLocalTime();
            var limitDate    = MongoDBService.GetService().FindOneSortProjectSkip(filter, sort, projection, limit - 1).GetElement(0).Value.ToLocalTime();
            var dates        = IndicatorService.GetData(code, targetDate, "Tarih", limit).Select(p => p.GetElement(0).Value.ToLocalTime()).ToArray();

            MapReduceOptions <BsonDocument, BsonDocument> options = new MapReduceOptions <BsonDocument, BsonDocument>
            {
                Filter = new BsonDocument {
                    { "Kod", code }, { "Tarih", new BsonDocument("$lte", new BsonDateTime(targetDate)) }
                },
                Sort          = new BsonDocument("Tarih", -1),
                OutputOptions = MapReduceOutputOptions.Replace("wmaOut", "financialData", false),
                Scope         = new BsonDocument {
                    { "startingDate", startingDate }, { "lastDate", lastDate }, { "limitDate", limitDate }, { "period", period }, { "denominator", denominator }, { "numberOfData", numberOfData }, { "binarySearch", MapReduceHelpers.BinarySearch }, { "equalityFunction", MapReduceHelpers.IsDatesEqual }, { "dates", new BsonArray(dates) }
                }
            };

            double[]       avg;
            BsonJavaScript mapper = new BsonJavaScript(@"
                function() {
	                if (!equalityFunction(this.Tarih, limitDate) && this.Tarih < limitDate)
		                return;
	                else if (!equalityFunction(this.Tarih, lastDate) && this.Tarih > lastDate)
		                return;
	
	                var dateIndex;
	
	                dateIndex = binarySearch(dates, this.Tarih);
	                if (dateIndex == -1)
		                return;
	
	                var factor = period;
	                for (var i = 0; i < period && dateIndex >= 0; i++, dateIndex--) {
		                if (dates[dateIndex] > startingDate || equalityFunction(dates[dateIndex], startingDate))
			                emit(dates[dateIndex], this.Kapanis * factor);
		                factor--;
	                }
                }
            ");

            BsonJavaScript reducer = new BsonJavaScript(@"
                function(key, values) {
	                return Array.sum(values);
                }
            ");

            BsonJavaScript finalizer = new BsonJavaScript(@"
                function(key, reducedValue) {
	                return reducedValue / denominator;
                }
            ");

            options.Finalize = finalizer;

            List <BsonDocument> resultSet = MongoDBService.GetService().MapReduceMany(mapper, reducer, options);

            avg = new double[numberOfData];

            for (int i = 0, j = numberOfData - 1; i < numberOfData; i++, j--)
            {
                avg[i] = resultSet.ElementAt(j).GetElement(1).Value.ToDouble();
            }

            return(avg);
        }
Exemplo n.º 15
0
        public virtual IEnumerable <TResult> MapReduce <TEntity, TResult>(MapReduceOptionsProxy <TEntity, TResult> options) where TEntity : CollectionEntityBase, new()
        {
            if (string.IsNullOrEmpty(options.Map))
            {
                throw new Exception("map is must");
            }
            if (string.IsNullOrEmpty(options.Reduce))
            {
                throw new Exception("reduce is must");
            }
            var o = new MapReduceOptions <TEntity, TResult>();

            if (options.OutputEnum.Equals(MapReduceOutputOptionsEnum.Inline))
            {
                o.OutputOptions = MapReduceOutputOptions.Inline;
            }
            else
            {
                if (string.IsNullOrEmpty(options.DatabaseName) || string.IsNullOrEmpty(options.CollectionName))
                {
                    throw new Exception("DatabaseName and CollectionName is must");
                }
                if (options.OutputEnum.Equals(MapReduceOutputOptionsEnum.Merge))
                {
                    o.OutputOptions = MapReduceOutputOptions.Reduce(options.CollectionName, options.DatabaseName);
                }
                else if (options.OutputEnum.Equals(MapReduceOutputOptionsEnum.Reduce))
                {
                    o.OutputOptions = MapReduceOutputOptions.Reduce(options.CollectionName, options.DatabaseName);
                }
                else if (options.OutputEnum.Equals(MapReduceOutputOptionsEnum.Replace))
                {
                    o.OutputOptions = MapReduceOutputOptions.Replace(options.CollectionName, options.DatabaseName);
                }
            }
            o.BypassDocumentValidation = options.BypassDocumentValidation;
            if (!string.IsNullOrEmpty(options.Filter))
            {
                o.Filter = options.Filter;
            }
            if (!string.IsNullOrEmpty(options.Finalize))
            {
                o.Finalize = options.Finalize;
            }
            o.JavaScriptMode = options.JavaScriptMode;
            o.Limit          = options.Limit;
            o.MaxTime        = options.MaxTime;
            if (!string.IsNullOrEmpty(options.Sort))
            {
                o.Sort = options.Sort;
            }
            o.Verbose = options.Verbose;
            var r = this.CurrentCollection <TEntity>().MapReduce <TResult>(new MongoDB.Bson.BsonJavaScript(options.Map), new MongoDB.Bson.BsonJavaScript(options.Reduce), o);

            if (options.OutputEnum.Equals(MapReduceOutputOptionsEnum.Inline))
            {
                return(r.Current);
            }
            else
            {
                return(null);
            }
        }