Exemplo n.º 1
0
        public void Import(List <ImportGuideChannel> skipChannels, ImportDataCallback importDataCallback, ProgressCallback progressCallback, FeedbackCallback feedbackCallback, KeepImportServiceAliveCallback keepImportServiceAliveCallback)
        {
            List <ImportGuideChannel> channelsToImport = GetAllImportChannels(skipChannels);
            int      importedCount      = 0;
            DateTime _lastKeepAliveTime = DateTime.Now;

            foreach (ImportGuideChannel channel in channelsToImport)
            {
                if (!skipChannels.Contains(channel))
                {
                    GiveFeedback(feedbackCallback, String.Format("Importing channel {0}.", channel.ChannelName));
                    List <GuideProgram> guideData = _dbReader.GetGuideDataByClickFinderChannel(channel);
                    importDataCallback(channel, ChannelType.Television, guideData.ToArray(), false);
                }
                importedCount++;
                int procentDone = ((100 * importedCount) / channelsToImport.Count);
                if (progressCallback != null)
                {
                    progressCallback(procentDone);
                }
                if ((procentDone % 5) == 0)
                {
                    Trace.WriteLine(String.Format("Progress : {0} % done.", procentDone));
                }

                if (_lastKeepAliveTime.AddSeconds(_keepAliveIntervalSeconds) <= DateTime.Now)
                {
                    keepImportServiceAliveCallback();
                    _lastKeepAliveTime = DateTime.Now;
                }
            }
        }
Exemplo n.º 2
0
        public void Import(List <ImportGuideChannel> skipChannels, ImportDataCallback importDataCallback, ProgressCallback progressCallback, FeedbackCallback feedbackCallback, KeepImportServiceAliveCallback keepImportServiceAliveCallback)
        {
            keepImportServiceAliveCallback();
            DateTime todeey = DateTime.Now;

            DateTime startDate = todeey;
            DateTime endDate   = todeey.AddDays(ConfigInstance.Current.NrOfDaysToImport);

            if (ConfigInstance.Current.UpdateMode == Config.UpDateMode.NextDayOnly)
            {
                endDate = startDate;
                DateTime lastImportedDay = ConvertIntAsDateTime(ConfigInstance.Current.LastImportedDay);

                if (lastImportedDay > startDate)
                {
                    Logger.Write(FormatForLogger(DateTime.Now.ToLongTimeString() + " : Calling Import for updates for one day : " + startDate.ToLongTimeString()));

                    Import(skipChannels, importDataCallback, progressCallback, feedbackCallback, keepImportServiceAliveCallback, ConvertStartDateTimeToString(startDate, true), ConvertEndDateTimeToString(startDate.AddHours(25), true));
                    Logger.Write(FormatForLogger(DateTime.Now.ToLongTimeString() + " : End of one day Import"));

                    startDate = lastImportedDay;
                }

                endDate = todeey.AddDays(ConfigInstance.Current.NrOfDaysToImport);
                if (endDate > startDate)
                {
                    Logger.Write(FormatForLogger(DateTime.Now.ToLongTimeString() + " : Calling Import for new data : " + startDate.ToLongTimeString()));

                    Import(skipChannels, importDataCallback, progressCallback, feedbackCallback, keepImportServiceAliveCallback, ConvertStartDateTimeToString(startDate), ConvertEndDateTimeToString(endDate));
                }
            }
            else
            {
                Import(skipChannels, importDataCallback, progressCallback, feedbackCallback, keepImportServiceAliveCallback, ConvertStartDateTimeToString(startDate), ConvertEndDateTimeToString(endDate));
            }
            Logger.Write(FormatForLogger(DateTime.Now.ToLongTimeString() + " : End of new data Import"));

            ConfigInstance.Current.LastImportedDay = ConvertDateTimeAsDateInt(endDate);
            ConfigInstance.Save();
        }
Exemplo n.º 3
0
        private void Import(List <ImportGuideChannel> skipChannels,
                            ImportDataCallback importDataCallback,
                            ProgressCallback progressCallback,
                            FeedbackCallback feedbackCallback,
                            KeepImportServiceAliveCallback keepImportServiceAliveCallback,
                            string startDate, string endDate)
        {
            GiveFeedback(feedbackCallback, "Calling SchedulesDirect WebService ...");

            if (progressCallback != null)
            {
                progressCallback(0);
            }

            keepImportServiceAliveCallback();

            xtvdResponse response = CallSchedulesDirectWebService(startDate, endDate);

            if (response != null)
            {
                GiveFeedback(feedbackCallback, "Processing SchedulesDirect data ...");

                //DataDumper.DumpResponse(response, @"d:\steph\temp\response.txt");
                keepImportServiceAliveCallback();

                // Process results, create dictionaries for easier access

                // Stations
                Dictionary <int, ImportGuideChannel> guideChannels = BuildImportGuideChannelDictionary(response, ConfigInstance.Current.ChannelNameFormat);
                GuideChannelStore.Save(AvailableChannelsConfigFile, new List <ImportGuideChannel>(guideChannels.Values));

                // Programs
                Dictionary <string, Program> programs = BuildProgramsDictionary(response);

                // Add genres to programs
                AddGenresToPrograms(response, programs);

                // Add actors and directors to programs
                AddActorsAndDirectorsToPrograms(response, programs);

                // Start the real import :
                List <GuideProgram> guideData           = new List <GuideProgram>();
                ImportGuideChannel  currentGuideChannel = new ImportGuideChannel(String.Empty, String.Empty);
                int importedCount = 0;
                int procentDone   = 0;

                foreach (schedulesSchedule schedule in response.xtvd.schedules)
                {
                    if (guideChannels.ContainsKey(schedule.station) && !skipChannels.Contains(guideChannels[schedule.station]))
                    {
                        if (!currentGuideChannel.ExternalId.Equals(schedule.station.ToString()))
                        {
                            if (guideData.Count > 0)
                            {
                                GiveFeedback(feedbackCallback, String.Format("Importing channel {0}, {1} schedule(s).", currentGuideChannel.ChannelName, guideData.Count));

                                importDataCallback(currentGuideChannel, ChannelType.Television, guideData.ToArray(), ConfigInstance.Current.UpdateChannelNames);
                                guideData = new List <GuideProgram>();
                            }
                            currentGuideChannel = guideChannels[schedule.station];
                        }
                        if (programs.ContainsKey(schedule.program))
                        {
                            GuideProgram guideProgram = FillGuideProgram(schedule, programs[schedule.program]);
                            if (guideProgram != null)
                            {
                                guideData.Add(guideProgram);
                            }
                            else
                            {
                                Logger.Write(FormatForLogger("FillGuideProgram returned null for schedule program : " + schedule.program));
                            }
                        }
                        else
                        {
                            Logger.Write(FormatForLogger("FillGuideProgram : unknown schedule program : " + schedule.program));
                        }
                    }
                    importedCount++;

                    int currentProcentDone = ((100 * importedCount) / response.xtvd.schedules.Length);
                    if (currentProcentDone != procentDone)
                    {
                        procentDone = currentProcentDone;
                        if (progressCallback != null)
                        {
                            progressCallback(procentDone);
                        }
                        else if ((procentDone % 5) == 0)
                        {
                            Logger.Write(FormatForLogger(String.Format("Progress : {0} % done.", procentDone)));
                        }
                    }
                }
                if (guideData.Count > 0)
                {
                    importDataCallback(currentGuideChannel, ChannelType.Television, guideData.ToArray(), ConfigInstance.Current.UpdateChannelNames);
                }
            }
        }
        /// <summary>
        /// Do the actual guide update
        /// </summary>
        /// <param name="skipChannels"></param>
        /// <param name="importDataCallback"></param>
        /// <param name="progressCallback"></param>
        /// <param name="feedbackCallback"></param>
        /// <param name="keepImportServiceAliveCallback"></param>
        public void Import(
            List <ImportGuideChannel> skipChannels,
            ImportDataCallback importDataCallback,
            ProgressCallback progressCallback,
            FeedbackCallback feedbackCallback,
            KeepImportServiceAliveCallback keepImportServiceAliveCallback)
        {
            try {
                GiveFeedback(feedbackCallback, "Staring SD JSON guide data import...");

                DateTime importStart = DateTime.Now;

                if (progressCallback != null)
                {
                    progressCallback(0);
                }

                keepImportServiceAliveCallback();

                // get station list
                List <string> stationIdList = getStationIdList(skipChannels);

                Logger.Info("Getting guide data for {0} channels", stationIdList.Count);

                WebClient wc = WebClient.getInstance();

                TokenRequest tr = new TokenRequest(ConfigInstance.Current.SDUserName, ConfigInstance.Current.SDPassword);

                // make sure SD site is online
                StatusResponse status = wc.getStatus(tr);
                if (status != null && status.systemStatus != null)
                {
                    string st = status.systemStatus[0].status.ToLower();
                    if (!"online".Equals(st))
                    {
                        Logger.Error("The SD server is not online: {0} - {1) - {2}", status.serverID, status.systemStatus[0].status, status.systemStatus[0].details);
                        throw new SystemException("The SD Server is not online.  See log for details");
                    }
                }

                // get lineup info
                GiveFeedback(feedbackCallback, "Getting channel information...");
                List <LineupInfoResponse> llir = wc.getAssignedLineupInfoList(tr);
                // map is keyed by stationId or externalId... the number that SD uses to identify the channel
                Dictionary <string, ImportGuideChannel> digc = ChannelFactory.makeImportChannelMap(llir, ConfigInstance.Current.ChannelNameFormat);

                // SD asks that no more than 5000 stations be queried at once
                // ignore this for now as not many will have 5000+ channels
                GiveFeedback(feedbackCallback, "Getting schedules...");
                Logger.Info("Get schedules for {0} days", ConfigInstance.Current.NrOfDaysToImport);

                // lsr contains the schedules for a station for a single day
                List <SchedulesResponse> lsr = wc.getSchedules(tr, stationIdList, ConfigInstance.Current.NrOfDaysToImport);

                // combine all days' schedules into one entry for each station
                Dictionary <string, SchedulesResponse> srMap = new Dictionary <string, SchedulesResponse>();



                // 1) Get all programs at once


                // make a global HashSet of program IDs and populate ScheduleResponse map
                HashSet <string> globalProgramSet = new HashSet <string>();
                foreach (SchedulesResponse sr in lsr)
                {
                    if (sr.programs != null)
                    {
                        // make list of program IDs to get
                        foreach (SchedulesResponse.Program pg in sr.programs)
                        {
                            // we only get a programID once, even if it's on multiple schedules
                            globalProgramSet.Add(pg.programID);
                        }

                        // populate map of schedule responses
                        if (srMap.ContainsKey(sr.stationID))
                        {
                            srMap[sr.stationID].programs.AddRange(sr.programs);
                        }
                        else
                        {
                            srMap.Add(sr.stationID, sr);
                        }
                    }
                }

                Logger.Info("There are {0} programs in this timeframe", globalProgramSet.Count);

                // take the programs in chunks of _programBlockSize... value below 5000
                // SD won't allow a query of more than 5000 programs at once
                int progLoops        = globalProgramSet.Count / _programBlockSize;
                int progsInFinalLoop = globalProgramSet.Count % _programBlockSize;

                if (progsInFinalLoop > 0)
                {
                    progLoops++;
                }

                // this map will contain *all* program responses across all stations
                Dictionary <string, ProgramResponseInstance> globalProgramResponseMap = new Dictionary <string, ProgramResponseInstance>();

                // keep a list for navigating in for loop
                List <string> globalProgramList = new List <string>();
                globalProgramList.AddRange(globalProgramSet);

                GiveFeedback(feedbackCallback, "About to get program information for all programs...");

                for (int i = 0; i < progLoops; i++)
                {
                    // is this the last loop?
                    int blockSize = _programBlockSize;
                    if (i == progLoops - 1)
                    {
                        blockSize = progsInFinalLoop;
                    }
                    string message = string.Format("Getting program information for items {0} to {1}",
                                                   i * _programBlockSize + 1,
                                                   (i * _programBlockSize) + blockSize);
                    GiveFeedback(feedbackCallback, message);

                    List <string> programSubset = globalProgramList.GetRange(i * _programBlockSize, blockSize);

                    List <ProgramResponseInstance> lpri = wc.getPrograms(tr, programSubset);

                    if (lpri != null)
                    {
                        foreach (ProgramResponseInstance pri in lpri)
                        {
                            globalProgramResponseMap.Add(pri.programID, pri);
                        }
                    }
                }

                // tackle each station at a time...
                int count = 1;
                foreach (SchedulesResponse sr in srMap.Values)
                {
                    keepImportServiceAliveCallback();

                    List <GuideProgram> lgp = new List <GuideProgram>();
                    try {
                        // take the schedules and programs and create the guide programs to put into Argus
                        // in a try block to catch issues with programfactory
                        lgp = ProgramFactory.makeGuidePrograms(globalProgramResponseMap, sr);
                    } catch (Exception ex) {
                        Logger.Error("Could not extract program information: {0}", ex.StackTrace);
                        throw;
                    }
                    ImportGuideChannel currentGuideChannel = digc[sr.stationID];

                    double percentDone = 100 * ((double)count / srMap.Values.Count);

                    int percentInt = Convert.ToInt32(percentDone);

                    if (progressCallback != null)
                    {
                        progressCallback(percentInt);
                    }

                    string programMessage =
                        string.Format("Processing station: {0}, ID: {1}, Program Count: {2}, {3} done",
                                      digc[sr.stationID].ChannelName,
                                      sr.stationID,
                                      sr.programs.Count,
                                      string.Format("{0}%", percentInt));

                    GiveFeedback(feedbackCallback, programMessage);

                    // save the programs in the Argus db
                    importDataCallback(
                        currentGuideChannel,
                        ChannelType.Television,
                        lgp.ToArray(),
                        ConfigInstance.Current.UpdateChannelNames);


                    count++;
                }


                DateTime endTime = DateTime.Now;

                TimeSpan ts      = endTime.Subtract(importStart);
                int      seconds = ts.Seconds;
                int      minutes = ts.Minutes;
                int      hours   = ts.Hours;

                Logger.Info("Import complete.  Took {0} hours, {1} minutes and {2} seconds", hours, minutes, seconds);
                Logger.Info("Channels processed: {0}", srMap.Values.Count);

                GiveFeedback(feedbackCallback, "Completed SD JSON guide data import.");
            } catch (Exception ex) {
                Logger.Error("There was an error importing the guide data: {0}/n{1}", ex.Message, ex.StackTrace);
                throw;
            }
        }
Exemplo n.º 5
0
        private void Import(List<ImportGuideChannel> skipChannels, 
                           ImportDataCallback importDataCallback, 
                           ProgressCallback progressCallback, 
                           FeedbackCallback feedbackCallback, 
                           KeepImportServiceAliveCallback keepImportServiceAliveCallback,
                           string startDate, string endDate )
        {
            GiveFeedback(feedbackCallback, "Calling SchedulesDirect WebService ...");

            if (progressCallback != null)
            {
                progressCallback(0);
            }

            keepImportServiceAliveCallback();

            xtvdResponse response = CallSchedulesDirectWebService(startDate, endDate);
            if (response != null)
            {
                GiveFeedback(feedbackCallback, "Processing SchedulesDirect data ...");

                //DataDumper.DumpResponse(response, @"d:\steph\temp\response.txt");
                keepImportServiceAliveCallback();

                // Process results, create dictionaries for easier access

                // Stations
                Dictionary<int, ImportGuideChannel> guideChannels = BuildImportGuideChannelDictionary(response, ConfigInstance.Current.ChannelNameFormat);
                GuideChannelStore.Save(AvailableChannelsConfigFile, new List<ImportGuideChannel>(guideChannels.Values));

                // Programs
                Dictionary<string, Program> programs = BuildProgramsDictionary(response);

                // Add genres to programs
                AddGenresToPrograms(response, programs);

                // Add actors and directors to programs
                AddActorsAndDirectorsToPrograms(response, programs);

                // Start the real import :
                List<GuideProgram> guideData = new List<GuideProgram>();
                ImportGuideChannel currentGuideChannel = new ImportGuideChannel(String.Empty, String.Empty);
                int importedCount = 0;
                int procentDone = 0;

                foreach (schedulesSchedule schedule in response.xtvd.schedules)
                {
                    if (guideChannels.ContainsKey(schedule.station) && !skipChannels.Contains(guideChannels[schedule.station]))
                    {
                        if (!currentGuideChannel.ExternalId.Equals(schedule.station.ToString()))
                        {
                            if (guideData.Count > 0)
                            {
                                GiveFeedback(feedbackCallback, String.Format("Importing channel {0}, {1} schedule(s).", currentGuideChannel.ChannelName, guideData.Count));

                                importDataCallback(currentGuideChannel, ChannelType.Television, guideData.ToArray(), ConfigInstance.Current.UpdateChannelNames);
                                guideData = new List<GuideProgram>();
                            }
                            currentGuideChannel = guideChannels[schedule.station];
                        }
                        if (programs.ContainsKey(schedule.program))
                        {
                            GuideProgram guideProgram = FillGuideProgram(schedule, programs[schedule.program]);
                            if (guideProgram != null)
                            {
                                guideData.Add(guideProgram);
                            }
                            else
                            {
                                Logger.Write(FormatForLogger("FillGuideProgram returned null for schedule program : " + schedule.program));
                            }
                        }
                        else
                        {
                            Logger.Write(FormatForLogger("FillGuideProgram : unknown schedule program : " + schedule.program));
                        }
                    }
                    importedCount++;

                    int currentProcentDone = ((100 * importedCount) / response.xtvd.schedules.Length);
                    if (currentProcentDone != procentDone)
                    {
                        procentDone = currentProcentDone;
                        if (progressCallback != null)
                        {
                            progressCallback(procentDone);
                        }
                        else if ((procentDone % 5) == 0)
                        {
                            Logger.Write(FormatForLogger(String.Format("Progress : {0} % done.", procentDone)));
                        }
                    }
                }
                if (guideData.Count > 0)
                {
                    importDataCallback(currentGuideChannel, ChannelType.Television, guideData.ToArray(), ConfigInstance.Current.UpdateChannelNames);
                }
            }
        }
Exemplo n.º 6
0
        public void Import(List<ImportGuideChannel> skipChannels, ImportDataCallback importDataCallback, ProgressCallback progressCallback, FeedbackCallback feedbackCallback, KeepImportServiceAliveCallback keepImportServiceAliveCallback)
        {
            keepImportServiceAliveCallback();
            DateTime todeey = DateTime.Now;

            DateTime startDate = todeey;
            DateTime endDate = todeey.AddDays(ConfigInstance.Current.NrOfDaysToImport);

            if (ConfigInstance.Current.UpdateMode == Config.UpDateMode.NextDayOnly)
            {
                endDate = startDate;
                DateTime lastImportedDay = ConvertIntAsDateTime(ConfigInstance.Current.LastImportedDay);

                if (lastImportedDay > startDate)
                {
                    Logger.Write(FormatForLogger(DateTime.Now.ToLongTimeString() + " : Calling Import for updates for one day : " + startDate.ToLongTimeString()));

                    Import(skipChannels, importDataCallback, progressCallback, feedbackCallback, keepImportServiceAliveCallback, ConvertStartDateTimeToString(startDate, true), ConvertEndDateTimeToString(startDate.AddHours(25), true));
                    Logger.Write(FormatForLogger(DateTime.Now.ToLongTimeString() + " : End of one day Import"));

                    startDate = lastImportedDay;
                }

                endDate = todeey.AddDays(ConfigInstance.Current.NrOfDaysToImport);
                if (endDate > startDate)
                {
                    Logger.Write(FormatForLogger(DateTime.Now.ToLongTimeString() + " : Calling Import for new data : " + startDate.ToLongTimeString()));

                    Import(skipChannels, importDataCallback, progressCallback, feedbackCallback, keepImportServiceAliveCallback, ConvertStartDateTimeToString(startDate), ConvertEndDateTimeToString(endDate));
                }
            }
            else
            {
                Import(skipChannels, importDataCallback, progressCallback, feedbackCallback, keepImportServiceAliveCallback, ConvertStartDateTimeToString(startDate), ConvertEndDateTimeToString(endDate));
            }
            Logger.Write(FormatForLogger(DateTime.Now.ToLongTimeString() + " : End of new data Import"));

            ConfigInstance.Current.LastImportedDay = ConvertDateTimeAsDateInt(endDate);
            ConfigInstance.Save();
        }
Exemplo n.º 7
0
        public void Import(List<ImportGuideChannel> skipChannels, ImportDataCallback importDataCallback, ProgressCallback progressCallback, FeedbackCallback feedbackCallback, KeepImportServiceAliveCallback keepImportServiceAliveCallback)
        {
            List<ImportGuideChannel> channelsToImport = GetAllImportChannels(skipChannels);
            int importedCount = 0;
            DateTime _lastKeepAliveTime = DateTime.Now;

            foreach (ImportGuideChannel channel in channelsToImport)
            {
                if (!skipChannels.Contains(channel))
                {
                    GiveFeedback(feedbackCallback, String.Format("Importing channel {0}.", channel.ChannelName));
                    List<GuideProgram> guideData = _dbReader.GetGuideDataByClickFinderChannel(channel);
                    importDataCallback(channel, ChannelType.Television, guideData.ToArray(), false);
                }
                importedCount++;
                int procentDone = ((100 * importedCount) / channelsToImport.Count);
                if (progressCallback != null)
                {
                    progressCallback(procentDone);
                }
                if((procentDone % 5)==0)
                {
                    Trace.WriteLine(String.Format("Progress : {0} % done.", procentDone));
                }

                if (_lastKeepAliveTime.AddSeconds(_keepAliveIntervalSeconds) <= DateTime.Now)
                {
                    keepImportServiceAliveCallback();
                    _lastKeepAliveTime = DateTime.Now;
                }
            }
        }
        /// <summary>
        /// Do the actual guide update
        /// </summary>
        /// <param name="skipChannels"></param>
        /// <param name="importDataCallback"></param>
        /// <param name="progressCallback"></param>
        /// <param name="feedbackCallback"></param>
        /// <param name="keepImportServiceAliveCallback"></param>
        public void Import(
			List<ImportGuideChannel> skipChannels, 
			ImportDataCallback importDataCallback, 
			ProgressCallback progressCallback, 
			FeedbackCallback feedbackCallback, 
			KeepImportServiceAliveCallback keepImportServiceAliveCallback)
        {
            try {
                GiveFeedback(feedbackCallback,"Staring SD JSON guide data import...");

                DateTime importStart = DateTime.Now;

                if (progressCallback != null)
                {
                    progressCallback(0);
                }

                keepImportServiceAliveCallback();

                // get station list
                List<string> stationIdList = getStationIdList(skipChannels);

                Logger.Info("Getting guide data for {0} channels", stationIdList.Count);

                WebClient wc = WebClient.getInstance();

                TokenRequest tr = new TokenRequest(ConfigInstance.Current.SDUserName, ConfigInstance.Current.SDPassword);

                // make sure SD site is online
                StatusResponse status = wc.getStatus(tr);
                if (status != null && status.systemStatus != null) {
                    string st = status.systemStatus[0].status.ToLower();
                    if (!"online".Equals(st)) {
                        Logger.Error("The SD server is not online: {0} - {1) - {2}", status.serverID, status.systemStatus[0].status, status.systemStatus[0].details);
                        throw new SystemException("The SD Server is not online.  See log for details");
                    }
                }

                // get lineup info
                GiveFeedback(feedbackCallback, "Getting channel information...");
                List<LineupInfoResponse> llir = wc.getAssignedLineupInfoList(tr);
                // map is keyed by stationId or externalId... the number that SD uses to identify the channel
                Dictionary<string,ImportGuideChannel> digc = ChannelFactory.makeImportChannelMap(llir, ConfigInstance.Current.ChannelNameFormat);

                // SD asks that no more than 5000 stations be queried at once
                // ignore this for now as not many will have 5000+ channels
                GiveFeedback(feedbackCallback, "Getting schedules...");
                Logger.Info("Get schedules for {0} days", ConfigInstance.Current.NrOfDaysToImport);

                // lsr contains the schedules for a station for a single day
                List<SchedulesResponse> lsr = wc.getSchedules(tr, stationIdList, ConfigInstance.Current.NrOfDaysToImport);

                // combine all days' schedules into one entry for each station
                Dictionary<string, SchedulesResponse> srMap = new Dictionary<string, SchedulesResponse>();

                // 1) Get all programs at once

                // make a global HashSet of program IDs and populate ScheduleResponse map
                HashSet<string> globalProgramSet = new HashSet<string>();
                foreach (SchedulesResponse sr in lsr) {
                    if (sr.programs != null) {
                        // make list of program IDs to get
                        foreach (SchedulesResponse.Program pg in sr.programs) {
                            // we only get a programID once, even if it's on multiple schedules
                            globalProgramSet.Add(pg.programID);
                        }

                        // populate map of schedule responses
                        if (srMap.ContainsKey(sr.stationID)) {
                            srMap[sr.stationID].programs.AddRange(sr.programs);
                        } else {
                            srMap.Add(sr.stationID, sr);
                        }
                    }
                }

                Logger.Info("There are {0} programs in this timeframe", globalProgramSet.Count);

                // take the programs in chunks of _programBlockSize... value below 5000
                // SD won't allow a query of more than 5000 programs at once
                int progLoops = globalProgramSet.Count / _programBlockSize;
                int progsInFinalLoop = globalProgramSet.Count % _programBlockSize;

                if (progsInFinalLoop > 0) {
                    progLoops++;
                }

                // this map will contain *all* program responses across all stations
                Dictionary<string,ProgramResponseInstance> globalProgramResponseMap = new Dictionary<string, ProgramResponseInstance>();

                // keep a list for navigating in for loop
                List<string> globalProgramList = new List<string>();
                globalProgramList.AddRange(globalProgramSet);

                GiveFeedback(feedbackCallback,"About to get program information for all programs...");

                for (int i = 0; i < progLoops; i++) {
                    // is this the last loop?
                    int blockSize = _programBlockSize;
                    if (i == progLoops - 1) {
                        blockSize = progsInFinalLoop;
                    }
                    string message = string.Format("Getting program information for items {0} to {1}",
                                i * _programBlockSize + 1,
                                (i * _programBlockSize) + blockSize);
                    GiveFeedback(feedbackCallback, message);

                    List<string> programSubset = globalProgramList.GetRange(i * _programBlockSize, blockSize);

                    List<ProgramResponseInstance> lpri = wc.getPrograms(tr, programSubset);

                    if (lpri != null) {
                        foreach (ProgramResponseInstance pri in lpri) {
                            globalProgramResponseMap.Add(pri.programID, pri);
                        }
                    }

                }

                // tackle each station at a time...
                int count = 1;
                foreach (SchedulesResponse sr in srMap.Values) {

                    keepImportServiceAliveCallback();

                    List<GuideProgram> lgp = new List<GuideProgram>();
                    try {
                        // take the schedules and programs and create the guide programs to put into Argus
                        // in a try block to catch issues with programfactory
                        lgp = ProgramFactory.makeGuidePrograms(globalProgramResponseMap, sr);
                    } catch (Exception ex) {
                        Logger.Error("Could not extract program information: {0}", ex.StackTrace);
                        throw;
                    }
                    ImportGuideChannel currentGuideChannel = digc[sr.stationID];

                    double percentDone = 100 * ((double)count/srMap.Values.Count);

                    int percentInt = Convert.ToInt32(percentDone);

                    if (progressCallback != null)
                    {
                        progressCallback(percentInt);
                    }

                    string programMessage =
                        string.Format("Processing station: {0}, ID: {1}, Program Count: {2}, {3} done",
                                      digc[sr.stationID].ChannelName,
                                      sr.stationID,
                                      sr.programs.Count,
                                      string.Format("{0}%",percentInt));

                    GiveFeedback(feedbackCallback, programMessage);

                    // save the programs in the Argus db
                    importDataCallback(
                        currentGuideChannel,
                        ChannelType.Television,
                        lgp.ToArray(),
                        ConfigInstance.Current.UpdateChannelNames);

                    count++;
                }

                DateTime endTime = DateTime.Now;

                TimeSpan ts = endTime.Subtract(importStart);
                int seconds = ts.Seconds;
               	int minutes = ts.Minutes;
               	int hours = ts.Hours;

                Logger.Info("Import complete.  Took {0} hours, {1} minutes and {2} seconds", hours, minutes, seconds);
                Logger.Info("Channels processed: {0}", srMap.Values.Count);

                GiveFeedback(feedbackCallback,"Completed SD JSON guide data import.");
            } catch (Exception ex) {
                Logger.Error("There was an error importing the guide data: {0}/n{1}", ex.Message, ex.StackTrace);
                throw;
            }
        }