public void Start(string streamConfigFileName, string searchConfigFileName)
        {
            StreamingQueries streamingQueryObj = ParseStreamConfig(streamConfigFileName);
            List<string> streamingQueries = streamingQueryObj.streamingQueries;
            List<string> streamingQueriesIDs = streamingQueryObj.streamingQueriesIDs;
            Dictionary<string, string> streamingQueriesEntities = streamingQueryObj.streamingQueriesEntities;

            ConfigParser searchConfigParser = new ConfigParser();
            ConfigParser.SearchQueries searchQueriesTokenized = searchConfigParser.GetTokenizedSearchQueries(searchConfigFileName);
            List<ArrayList<ArrayList<QueryToken>>> searchQueriesTokens = searchQueriesTokenized.searchQueriesTokens;
            List<string> searchQueriesIDs = searchQueriesTokenized.searchQueriesIDs;

            List<List<string>> streamingQueryGroups = SeparateQueriesIntoGroups(streamingQueries);
            List<List<string>> streamingQueryIDGroups = SeparateQueriesIntoGroups(streamingQueriesIDs);
            List<List<string>> streamingAuthData = LoadAuthData();

            if (streamingQueryGroups.Count > streamingAuthData.Count)
            {
                StreamLogger.writeErrorToConsoleAndExit(String.Format("Number of query groups ({0}) exceeds the number of available authentication groups ({1})!", streamingQueryGroups.Count, streamingAuthData.Count),
                    "TwitterStreamingDacq.StreamAcquisition", "Start");
            }

            int numThreads = streamingQueryGroups.Count;
            List<Task> tasks = new List<Task>();

            for (int iThr = 0; iThr < numThreads; ++iThr)
            {
                List<string> taskQueries = streamingQueryGroups[iThr];
                List<string> taskQueriesIDs = streamingQueryIDGroups[iThr];
                Dictionary<string, string> taskEntities = streamingQueriesEntities;
                List<ArrayList<ArrayList<QueryToken>>> taskSearchQueriesTokens = searchQueriesTokens;
                List<string> taskSearchQueriesIDs = searchQueriesIDs;
                List<string> taskAuth = streamingAuthData[iThr];
                StreamTask streamTask = new StreamTask();
                int taskId = iThr + 1;
                if (numThreads == 1)
                    taskId = StreamLogger.EMPTY_TASK_ID;

                tasks.Add(Task.Factory.StartNew(() =>
                {
                    streamTask.Start(taskId, taskQueries, taskQueriesIDs, taskSearchQueriesTokens, taskSearchQueriesIDs, taskEntities, taskAuth);
                }));
            }
            Task.WaitAll(tasks.ToArray());
        }
Exemple #2
0
 static void Main(string[] args)
 {
     ConfigParser cp = new ConfigParser();
     cp.ConvertSearchToStreamingConfig("TwitterDacqDAX30.cfg", "TwitterStreamingDacqDAX30.cfg");
 }