public void Start(string streamConfigFileName, string searchConfigFileName)
{
StreamingQueries streamingQueryObj = ParseStreamConfig(streamConfigFileName);
List<string> streamingQueries = streamingQueryObj.streamingQueries;
List<string> streamingQueriesIDs = streamingQueryObj.streamingQueriesIDs;
Dictionary<string, string> streamingQueriesEntities = streamingQueryObj.streamingQueriesEntities;
ConfigParser searchConfigParser = new ConfigParser();
ConfigParser.SearchQueries searchQueriesTokenized = searchConfigParser.GetTokenizedSearchQueries(searchConfigFileName);
List<ArrayList<ArrayList<QueryToken>>> searchQueriesTokens = searchQueriesTokenized.searchQueriesTokens;
List<string> searchQueriesIDs = searchQueriesTokenized.searchQueriesIDs;
List<List<string>> streamingQueryGroups = SeparateQueriesIntoGroups(streamingQueries);
List<List<string>> streamingQueryIDGroups = SeparateQueriesIntoGroups(streamingQueriesIDs);
List<List<string>> streamingAuthData = LoadAuthData();
if (streamingQueryGroups.Count > streamingAuthData.Count)
{
StreamLogger.writeErrorToConsoleAndExit(String.Format("Number of query groups ({0}) exceeds the number of available authentication groups ({1})!", streamingQueryGroups.Count, streamingAuthData.Count),
"TwitterStreamingDacq.StreamAcquisition", "Start");
}
int numThreads = streamingQueryGroups.Count;
List<Task> tasks = new List<Task>();
for (int iThr = 0; iThr < numThreads; ++iThr)
{
List<string> taskQueries = streamingQueryGroups[iThr];
List<string> taskQueriesIDs = streamingQueryIDGroups[iThr];
Dictionary<string, string> taskEntities = streamingQueriesEntities;
List<ArrayList<ArrayList<QueryToken>>> taskSearchQueriesTokens = searchQueriesTokens;
List<string> taskSearchQueriesIDs = searchQueriesIDs;
List<string> taskAuth = streamingAuthData[iThr];
StreamTask streamTask = new StreamTask();
int taskId = iThr + 1;
if (numThreads == 1)
taskId = StreamLogger.EMPTY_TASK_ID;
tasks.Add(Task.Factory.StartNew(() =>
{
streamTask.Start(taskId, taskQueries, taskQueriesIDs, taskSearchQueriesTokens, taskSearchQueriesIDs, taskEntities, taskAuth);
}));
}
Task.WaitAll(tasks.ToArray());
}
static void Main(string[] args)
{
ConfigParser cp = new ConfigParser();
cp.ConvertSearchToStreamingConfig("TwitterDacqDAX30.cfg", "TwitterStreamingDacqDAX30.cfg");
}
