Example #1
0
            public void StartUpdatingResourceGroup(int groupId, ResourceGroupUpdateCallbackSet callbackSet, object context)
            {
                if (!IsReady)
                {
                    throw new InvalidOperationException("Not ready.");
                }

                if (!AvailableResourceGroupIds.Contains(groupId))
                {
                    throw new ArgumentException(Utility.Text.Format("Resource group '{0}' is not available.", groupId));
                }

                if (GetResourceGroupStatus(groupId) == ResourceGroupStatus.UpToDate)
                {
                    throw new InvalidOperationException(Utility.Text.Format("Resource group '{0}' is already up-to-date.", groupId));
                }

                if (AvailableResourceGroupIds.Contains(0) && groupId != 0 && GetResourceGroupStatus(0) != ResourceGroupStatus.UpToDate)
                {
                    throw new InvalidOperationException("You have to update resource group 0 first.");
                }

                if (m_ResourceGroupsBeingUpdated.ContainsKey(groupId))
                {
                    throw new InvalidOperationException($"Resource group '{groupId}' is being updated.");
                }

                var resourceSummary = ResourceSummaries[groupId];
                var resourceGroup   = new ResourceGroupBeingUpdated
                {
                    GroupId         = groupId,
                    Summary         = resourceSummary,
                    CallbackSet     = callbackSet,
                    CallbackContext = context,
                };

                m_ResourceGroupsBeingUpdated.Add(groupId, resourceGroup);

                foreach (var resourceToUpdate in resourceSummary)
                {
                    var downloadContext = new DownloadContext
                    {
                        RootUrlIndex    = 0,
                        ResourcePath    = resourceToUpdate.Key,
                        ResourceGroupId = groupId,
                    };
                    var resourceInfo     = m_Owner.m_RemoteIndex.ResourceInfos[resourceToUpdate.Key];
                    var downloadTaskInfo = new DownloadTaskInfo(
                        Utility.Text.Format("{0}/{1}_{2}{3}", RootUrls[0], resourceToUpdate.Key, resourceInfo.Crc32,
                                            Constant.ResourceFileExtension),
                        Path.Combine(m_Owner.ReadWritePath, resourceToUpdate.Key + Constant.ResourceFileExtension),
                        resourceInfo.Size, resourceInfo.Crc32, new DownloadCallbackSet
                    {
                        OnFailure  = m_OnDownloadFailure,
                        OnSuccess  = m_OnDownloadSuccess,
                        OnProgress = m_OnDownloadProgress,
                    }, downloadContext);
                    resourceGroup.DownloadTaskIds.Add(m_Owner.DownloadModule.StartDownloading(downloadTaskInfo));
                }
            }
        public IReadOnlyCollection <DownloadEntry> Download(DownloadContext downloadContext, BoerseUser boerseUser)
        {
            var threadHandler = _boerseLinkThreadHandlerFactory.Create(downloadContext.BoerseLinkProvider);
            var result        = threadHandler.GetAllEntries(downloadContext, boerseUser);

            return(result);
        }
Example #3
0
        private async Task Context_OnDownloadLocally(DownloadContext item)
        {
            // TODO: Add cancel
            await this.downloadService.Download(item, default);

            await item.UpdateLocallyAvailable();
        }
        internal IReadOnlyCollection <DownloadEntry> GetAllEntries(DownloadContext downloadContext, BoerseUser boerseUser)
        {
            var threadNode = DownloadAndProcessHtml(downloadContext, boerseUser);
            var entries    = ReadAllDownloadEntriesFromThread(threadNode, downloadContext);

            return(entries);
        }
Example #5
0
        private DownloadEntry CreateEntry(HtmlNode threadHtmlNode, DownloadContext downloadContext, HtmlNode uploadedSpan)
        {
            var linkElement = uploadedSpan.ParentNode;

            var externalLink = GetExternalUrl(linkElement);

            string entryName;
            bool   isSeasonPack;

            if (CheckIfIsAllEpisodesSpan(uploadedSpan))
            {
                entryName    = Constants.DOWNLOADENTRYTITLE_SEASONPACK;
                isSeasonPack = true;
            }
            else
            {
                entryName    = GetDownloadEntryTitle(linkElement);
                isSeasonPack = false;
            }

            var lastUploadDate = GetLastThreadUpdate(threadHtmlNode);
            var result         = new DownloadEntry(downloadContext.Name, externalLink, entryName, lastUploadDate, isSeasonPack);

            return(result);
        }
        private string DownloadRawHtml(DownloadContext downloadContext, BoerseUser boerseUser)
        {
            var adapter = _webBrowserAdapterFactory.Create(downloadContext);
            var result  = adapter.DownloadThreadHtml(downloadContext, boerseUser);

            return(result);
        }
Example #7
0
        public AccountHelper(IConfiguration configuration)
        {
            var optionsBuilder = new DbContextOptionsBuilder <DownloadContext>();

            //optionsBuilder.UseMySql(configuration.GetConnectionString("MySQLConnection"));
            optionsBuilder.UseSqlite("Data Source=database.db");
            _context = new DownloadContext(optionsBuilder.Options);
        }
Example #8
0
 private void DownloadService_OnBytesDownloaded(DownloadContext context, long receivedBytes)
 {
     context.DownloadedBytes = receivedBytes;
     if (!context.Status.HasFlag(PackageStatus.Downloading))
     {
         context.SetDownloading();
     }
 }
Example #9
0
        private static void OnDownloadFileCompleted(HTTPRequest request, HTTPResponse response)
        {
            DownloadContext downloadContext = request.Tag as DownloadContext;

            switch (request.State)
            {
            case HTTPRequestStates.Finished:
                if (response.IsSuccess)
                {
                    Debug.Log((object)("ApiFile: Download complete: " + request.Uri.OriginalString));
                    if (downloadContext.OnSuccess != null)
                    {
                        downloadContext.OnSuccess(response.Data);
                    }
                }
                else
                {
                    Debug.LogError((object)string.Format("ApiFile: Download: " + request.Uri.OriginalString + " - Request finished Successfully, but the server sent an error. Status Code: {0}-{1} Message: {2}", response.StatusCode, response.Message, response.DataAsText));
                    if (downloadContext.OnError != null)
                    {
                        downloadContext.OnError("(" + response.StatusCode + ") " + ((response.Message == null) ? string.Empty : response.Message) + ((response.DataAsText == null) ? string.Empty : (": " + response.DataAsText)));
                    }
                }
                break;

            case HTTPRequestStates.Error:
                Debug.LogError((object)("ApiFile: Download: " + request.Uri.OriginalString + " - Request Finished with Error! " + ((request.Exception == null) ? "(No Exception)" : request.Exception.Message)));
                if (downloadContext.OnError != null)
                {
                    downloadContext.OnError("Request Finished with Error: " + ((request.Exception == null) ? "(No Exception)" : request.Exception.Message));
                }
                break;

            case HTTPRequestStates.Aborted:
                Debug.LogWarning((object)("ApiFile: Download: " + request.Uri.OriginalString + " - Request Aborted"));
                if (downloadContext.OnError != null)
                {
                    downloadContext.OnError("The download was cancelled");
                }
                break;

            case HTTPRequestStates.ConnectionTimedOut:
                Debug.LogError((object)("ApiFile: Download: " + request.Uri.OriginalString + " - Connection Timed Out!"));
                if (downloadContext.OnError != null)
                {
                    downloadContext.OnError("Connection timed out");
                }
                break;

            case HTTPRequestStates.TimedOut:
                Debug.LogError((object)("ApiFile: Download: " + request.Uri.OriginalString + " - Processing the request Timed Out!"));
                if (downloadContext.OnError != null)
                {
                    downloadContext.OnError("Processing the request timed out");
                }
                break;
            }
        }
Example #10
0
        static void Main(string[] args)
        {
            System.Threading.Thread.Sleep(TimeSpan.FromSeconds(5));

            var appContext = ContextBuilder.BuildContext();



#pragma warning disable CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed
            Task.Run(async() =>
            {
                var rabbitMQ = appContext.GetObject <RabbitMQ.Client.IModel>();
                string queue = appContext.GetObject <string>("CONFIG:DevWeek:RabbitMQ:DownloadPipeline:Queue");
                while (true)
                {
                    RabbitMQ.Client.BasicGetResult result = rabbitMQ.BasicGet(queue, false);
                    if (result != null)
                    {
                        try
                        {
                            string messageBody = System.Text.Encoding.UTF8.GetString(result.Body);

                            Services.Downloader.Download messageObject = Newtonsoft.Json.JsonConvert.DeserializeObject <Services.Downloader.Download>(messageBody);

                            var pipeline = appContext.GetObject <DevWeek.Services.Downloader.DownloadPipeline>();

                            var context = new DownloadContext()
                            {
                                MediaUrl       = messageObject.OriginalMediaUrl,
                                OutputFileName = $"{Guid.NewGuid().ToString("N")}.mp4",
                                Download       = messageObject
                            };

                            await pipeline.Run(context);

                            rabbitMQ.BasicAck(result.DeliveryTag, false);
                        }
                        catch (InvalidOperationException ex) when(ex.Message.Contains("#invalidUrl"))
                        {
                            rabbitMQ.BasicAck(result.DeliveryTag, false);
                        }
                        catch (Exception ex)
                        {
                            rabbitMQ.BasicNack(result.DeliveryTag, false, true);
                        }
                    }
                    else
                    {
                        System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1));
                    }
                }
            });
#pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed



            Console.WriteLine("Hello World!");
        }
        internal string DownloadThreadHtml(DownloadContext downloadContext, BoerseUser boerseUser)
        {
            _htmlprocessed   = false;
            _downloadContext = downloadContext;
            NavigateToThread();
            AssureUserIsLoggedIn(boerseUser);

            return(_html);
        }
Example #12
0
        public Stream Download(string clientDescription, WebMediaType type, int?provider, string itemId, long?position)
        {
            // validate source first
            MediaSource source = new MediaSource(type, provider, itemId);

            if (!source.Exists)
            {
                throw new FileNotFoundException();
            }

            // create context
            DownloadContext context = new DownloadContext()
            {
                ClientDescription = clientDescription,
                Source            = source,
                StartTime         = DateTime.Now,
                Stream            = new ReadTrackingStreamWrapper(source.Retrieve()),
                MediaInfo         = MediaInfoHelper.LoadMediaInfoOrSurrogate(source) // for playerposition view
            };

            // seek to start position if wanted/needed
            if (position != null && position > 0)
            {
                if (context.Stream.CanSeek)
                {
                    context.Stream.Seek(position.Value, SeekOrigin.Begin);
                }
                else
                {
                    Log.Warn("Download: Cannot seek on stream, failed to set start position to {0}", position);
                }
            }

            // see comment in Streaming.cs:151
            string realIp = WCFUtil.GetHeaderValue("forwardedFor", "X-Forwarded-For");

            context.ClientIP = realIp == null?WCFUtil.GetClientIPAddress() : String.Format("{0} (via {1})", realIp, WCFUtil.GetClientIPAddress());

            // set headers for downloading
            WCFUtil.AddHeader("Content-Disposition", "attachment; filename=\"" + source.GetFileInfo().Name + "\"");
            if (source.MediaType != WebMediaType.TV)
            {
                WCFUtil.SetContentLength(source.GetFileInfo().Size);
            }

            // FIXME: there has to be a better way to do this
            string mime = MIME.GetFromFilename(source.GetFileInfo().Name);

            if (mime != null)
            {
                WCFUtil.SetContentType(mime);
            }

            // finally, save the context and return
            runningDownloads.Add(context);
            return(context.Stream);
        }
        public void Playlist()
        {
            IPipelineActivity activity = new EntryPointRegisterPipelineActivity(null);
            DownloadContext   context  = BuildContext("https://www.youtube.com/watch?v=gFTE__qOMqI&t=306s&index=1&list=PLbDr6zG2yjZPD-FvY6cBQQAB8j7tjXD11");

            Assert.Throws <InvalidOperationException>(() => {
                activity.ExecuteAsync(context).GetAwaiter().GetResult();
            });
        }
        public void OkShortUrl()
        {
            IPipelineActivity activity = new EntryPointRegisterPipelineActivity(null);
            DownloadContext   context  = BuildContext("https://youtu.be/PH0SqgTSocs");

            Assert.Throws <NullReferenceException>(() => {
                activity.ExecuteAsync(context).GetAwaiter().GetResult();
            });
        }
        public void NullAddress()
        {
            IPipelineActivity activity = new EntryPointRegisterPipelineActivity(null);
            DownloadContext   context  = BuildContext(null);

            Assert.Throws <ArgumentNullException>(() => {
                activity.ExecuteAsync(context).GetAwaiter().GetResult();
            });
        }
        public void GenericGoogleAddress()
        {
            IPipelineActivity activity = new EntryPointRegisterPipelineActivity(null);
            DownloadContext   context  = BuildContext("https://www.google.com");

            Assert.Throws <InvalidOperationException>(() => {
                activity.ExecuteAsync(context).GetAwaiter().GetResult();
            });
        }
Example #17
0
        public MoveManagerService(IConfiguration configuration, DownloadSettings settings)
        {
            var    optionsBuilder = new DbContextOptionsBuilder <DownloadContext>();
            string datapath       = Path.Combine(settings.DownloadFolder, "database.db");

            optionsBuilder.UseSqlite("Data Source=" + datapath);

            _context  = new DownloadContext(optionsBuilder.Options);
            _settings = DownloadSettings.Load();
        }
Example #18
0
        /// <summary>
        /// Asynchronously download the raw bytes of an image file, with support for cancellation.
        /// </summary>
        /// <param name="id">The id the of the image file to download.</param>
        /// <param name="cancellationToken">
        /// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
        /// </param>
        /// <returns>An array of bytes, representing the image content.</returns>
        public async Task <byte[]> DownloadFileAsync(
            int id,
            CancellationToken cancellationToken)
        {
            var context = new DownloadContext <File>(EndpointName.FilesRaw, new FileDownloadFilter(id));

            await ExecuteOperationAsync(context, cancellationToken).ConfigureAwait(false);

            return(context.RawResponseContent);
        }
        public ExtractManagerService(DownloadSettings settings, IConfiguration configuration)
        {
            _settings = settings;
            var    optionsBuilder = new DbContextOptionsBuilder <DownloadContext>();
            string datapath       = Path.Combine(settings.DownloadFolder, "database.db");

            optionsBuilder.UseSqlite("Data Source=" + datapath);

            _context = new DownloadContext(optionsBuilder.Options);
        }
        private WebBrowserAdapter GetCreateChachedAdapter(DownloadContext downloadContext)
        {
            WebBrowserAdapter result;

            if (!_adapterCache.TryGetValue(downloadContext.Name, out result))
            {
                result = Common.Singletons.UnityContainerSingleton.Instance.Resolve <WebBrowserAdapter>();
                _adapterCache.TryAdd(downloadContext.Name, result);
            }

            return(result);
        }
Example #21
0
        public async Task TestDirectDownloadWithProgressAsync_Unknown_File_Size()
        {
            var packageSource      = new NugetOrgOfficialV3PackageSources().Items.Single();
            var percentages        = new List <int>();
            var progressSourceMock = new Mock <INugetServiceProgressSource>();

            progressSourceMock.Setup(x => x.Raise(
                                         It.IsAny <int>(),
                                         It.IsAny <long>(),
                                         It.IsAny <long>(),
                                         It.IsAny <long>()))
            .Callback((int percentage, long bytesRead, long totalBytesSoFar, long totalBytesDownloaded) =>
            {
                percentages.Add(percentage);
            });

            var downloadContext = new DownloadContext
            {
                PackageIdentity = new PackageIdentity("LibLog", NuGetVersion.Parse("5.0.6")),
                PackageFileSize = 0,
                MaxTries        = 3
            };

            using (var downloadResourceResult = await _nugetService.DownloadAsyncWithProgressAsync(packageSource, downloadContext,
                                                                                                   progressSourceMock.Object, CancellationToken.None))
            {
                const long expectedPackageSize = 64196;

                Assert.NotNull(downloadResourceResult);
                Assert.Equal(expectedPackageSize, downloadResourceResult.PackageStream.Length);
                Assert.Equal(0, downloadResourceResult.PackageStream.Position);

                progressSourceMock.Verify(x => x.Raise(
                                              It.Is <int>(v => v == 0),
                                              It.Is <long>(v => v == 0),
                                              It.Is <long>(v => v == 0),
                                              It.Is <long>(v => v == 0)), Times.Once);

                progressSourceMock.Verify(x => x.Raise(
                                              It.Is <int>(v => v == 50),
                                              It.IsAny <long>(),
                                              It.IsAny <long>(),
                                              It.Is <long>(v => v == 0)), Times.AtLeastOnce);

                progressSourceMock.Verify(x => x.Raise(
                                              It.Is <int>(v => v == 100),
                                              It.IsAny <long>(),
                                              It.Is <long>(v => v == expectedPackageSize),
                                              It.Is <long>(v => v == 0)), Times.Once);
            }

            Assert.Equal(progressSourceMock.Invocations.Count, percentages.Count);
        }
Example #22
0
        public Stream Download(string clientDescription, WebMediaType type, int? provider, string itemId, long? position)
        {
            // validate source first
            MediaSource source = new MediaSource(type, provider, itemId);
            if (!source.Exists)
            {
                throw new FileNotFoundException();
            }

            // create context
            DownloadContext context = new DownloadContext()
            {
                ClientDescription = clientDescription,
                Source = source,
                StartTime = DateTime.Now,
                Stream = new ReadTrackingStreamWrapper(source.Retrieve()),
                MediaInfo = MediaInfoHelper.LoadMediaInfoOrSurrogate(source) // for playerposition view
            };

            // seek to start position if wanted/needed
            if (position != null && position > 0)
            {
                if (context.Stream.CanSeek)
                {
                    context.Stream.Seek(position.Value, SeekOrigin.Begin);
                }
                else
                {
                    Log.Warn("Download: Cannot seek on stream, failed to set start position to {0}", position);
                }
            }

            // see comment in Streaming.cs:151
            string realIp = WCFUtil.GetHeaderValue("forwardedFor", "X-Forwarded-For");
            context.ClientIP = realIp == null ? WCFUtil.GetClientIPAddress() : String.Format("{0} (via {1})", realIp, WCFUtil.GetClientIPAddress());

            // set headers for downloading
            WCFUtil.AddHeader("Content-Disposition", "attachment; filename=\"" + source.GetFileInfo().Name + "\"");
            if (source.MediaType != WebMediaType.TV)
                WCFUtil.SetContentLength(source.GetFileInfo().Size);

            // FIXME: there has to be a better way to do this
            string mime = MIME.GetFromFilename(source.GetFileInfo().Name);
            if (mime != null)
            {
                WCFUtil.SetContentType(mime);
            }

            // finally, save the context and return
            runningDownloads.Add(context);
            return context.Stream;
        }
Example #23
0
 private async Task Context_OnDownload(DownloadContext item)
 {
     // TODO: ADD CANCEL
     await this.client.DownloadAsync(new DownloadRequest()
     {
         DownloadRequest_ = new DownloadRequest.Types.Request()
         {
             BotName       = item.Pack.Packs.First().Key,
             FileName      = item.Pack.Name,
             PackageNumber = (long)item.Pack.Packs.First().Value,
         }
     }, cancellationToken : default);
 }
Example #24
0
        public async Task TestDownloadLatestAsync_Local_Directory_PackageSource(NuGetProtocolVersion protocolVersion)
        {
            await using var testPackageDirectory = new DisposableDirectory(_baseFixture.WorkingDirectory, _snapFilesystem);

            var packageSource = new PackageSource(testPackageDirectory, "test", true)
            {
                ProtocolVersion = (int)protocolVersion
            };

            var initialPackageIdentity = new PackageIdentity("test", NuGetVersion.Parse("1.0.0"));

            var initialPackageFilenameAbsolute = _snapFilesystem.PathCombine(testPackageDirectory,
                                                                             $"{initialPackageIdentity.Id}.{initialPackageIdentity.Version.ToNormalizedString()}.nupkg");

            await using (var packageOutputStream = _snapFilesystem.FileReadWrite(initialPackageFilenameAbsolute))
            {
                await using var nupkgStream = BuildNupkg(initialPackageIdentity);
                await nupkgStream.CopyToAsync(packageOutputStream);
            }

            var secondPackageIdentity = new PackageIdentity("test", NuGetVersion.Parse("2.0.0"));

            var secondPackageFilenameAbsolute = _snapFilesystem.PathCombine(testPackageDirectory,
                                                                            $"{secondPackageIdentity.Id}.{secondPackageIdentity.Version.ToNormalizedString()}.nupkg");

            int packageFileSize;

            await using (var packageOutputStream = _snapFilesystem.FileReadWrite(secondPackageFilenameAbsolute))
            {
                await using var nupkgStream = BuildNupkg(secondPackageIdentity);
                await nupkgStream.CopyToAsync(packageOutputStream);

                packageFileSize = (int)nupkgStream.Length;
            }

            var downloadContext = new DownloadContext
            {
                PackageIdentity = secondPackageIdentity,
                PackageFileSize = packageFileSize
            };

            using var downloadResourceResult = await _nugetService.DownloadLatestAsync(secondPackageIdentity.Id, packageSource, true, true);

            Assert.NotNull(downloadResourceResult);
            Assert.Equal(downloadContext.PackageFileSize, downloadResourceResult.PackageStream.Length);
            Assert.Equal(0, downloadResourceResult.PackageStream.Position);

            using var package = new PackageArchiveReader(downloadResourceResult.PackageStream);
            Assert.Equal(package.GetIdentity(), secondPackageIdentity);
        }
        private bool TryDownloadingDataAuthorNode(DownloadContext downloadContext, BoerseUser boerseUser, out HtmlNode dataAuthorNode)
        {
            const string ATTR_DATA_AUTHOR = "data-author";

            var rawHtml = DownloadRawHtml(downloadContext, boerseUser);
            var htmlDoc = new HtmlDocument();

            htmlDoc.LoadHtml(rawHtml);

            dataAuthorNode = htmlDoc.DocumentNode.Descendants().Where(f => f.Name == "li").FirstOrDefault(f => f.Attributes[ATTR_DATA_AUTHOR]?.Value == downloadContext.BoerseLinkProvider.GetNativeName());
            if (dataAuthorNode == null)
            {
                return(false);
            }

            return(true);
        }
        public void PipelineFactory_GivenDownloadContext_BuildsExpectedPipeline()
        {
            var context = new DownloadContext <TestEntity>(
                EndpointName.FilesRaw,
                null);

            var pipeline = (RequestPipeline)this.pipelineFactory.GetPipeline(context);

            Type[] expectedElementTypes =
            {
                typeof(RestClientDownloadHandler)
            };

            IPipelineElement[] actualElements = pipeline.PipelineElements.ToArray();

            AssertElements(expectedElementTypes, actualElements);
        }
        private bool TryReadingSeasonPack(HtmlNode threadHtmlNode, DownloadContext downloadContext, out DownloadEntry seasonPackEntry)
        {
            var seasonPackSpan = threadHtmlNode.Descendants("span").FirstOrDefault(f => f.InnerText.Contains("Staffelpack"));

            if (seasonPackSpan != null)
            {
                var parentDiv      = seasonPackSpan.NavigateToElementOfType("div", Model.Enumerations.HtmlNavigationType.Parent);
                var downloadLink   = GetExternalLinkUrl(parentDiv);
                var entryTitle     = Constants.DOWNLOADENTRYTITLE_SEASONPACK;
                var lastUploadDate = GetLastThreadUpdate(threadHtmlNode);
                seasonPackEntry = new DownloadEntry(downloadContext.Name, downloadLink, entryTitle, lastUploadDate, true);
                return(true);
            }

            seasonPackEntry = null;
            return(false);
        }
Example #28
0
        public DownloadContext GetContext(PackMetadata packMetadata)
        {
            if (!this.context.TryGetValue(packMetadata.Name, out var result))
            {
                result             = new DownloadContext(this.packService.Get(packMetadata), this.moduleSettingsService);
                result.OnDownload += async item => await this.Context_OnDownload(item);

                result.OnDownloadLocally += async item => await this.Context_OnDownloadLocally(item);

                this.context.TryAdd(packMetadata.Name, result);
            }

            // This only works bc Pack is not be copied and one instance is
            // shared, maybe not desireable
            this.packService.Update(packMetadata);

            return(result);
        }
Example #29
0
        private IEnumerable <DownloadEntry> DownloadEntries(DownloadContext downloadContext, BoerseUser boerseUser)
        {
            IEnumerable <DownloadEntry> result;

            try
            {
                var integration = _integrationFactory.Create();
                result = integration.Download(downloadContext, boerseUser);
            }
            catch (Exception ex)
            {
                result = new List <DownloadEntry>
                {
                    new DownloadEntry(downloadContext.Name, ex)
                };
            }

            return(result);
        }
        private bool TryReadingSeasonPack(HtmlNode threadHtmlNode, DownloadContext downloadContext, out DownloadEntry seasonPackEntry)
        {
            var staffelpackElement = threadHtmlNode.Descendants("b").FirstOrDefault(f => f.InnerText.Contains("Staffelpack"));

            if (staffelpackElement != null)
            {
                var parentSpan = staffelpackElement.NavigateToElementOfType("span", Model.Enumerations.HtmlNavigationType.Parent);
                var nextBlock  = parentSpan.NavigateToElementOfType("b", Model.Enumerations.HtmlNavigationType.NextSibling);

                var linkElement = nextBlock.Descendants("a").First();

                var link = linkElement.Attributes["href"].Value;
                seasonPackEntry = new DownloadEntry(downloadContext.Name, link, Constants.DOWNLOADENTRYTITLE_SEASONPACK, GetLastThreadUpdate(threadHtmlNode), true);
                return(true);
            }

            seasonPackEntry = null;
            return(false);
        }
Example #31
0
        static void ExcelReader()
        {
            TimeZoneInfo Pacific_Standard_Time = TimeZoneInfo.FindSystemTimeZoneById(timeZone);
            DateTime     dateTime_Pacific      = TimeZoneInfo.ConvertTimeFromUtc(DateTime.UtcNow, Pacific_Standard_Time);
            var          dateString            = dateTime_Pacific.ToString("yyyyMMdd");

            DirectoryInfo dir = new DirectoryInfo(filePath);

            string partialName = $"{fileName}-{dateString}";

            FileInfo[] file = dir.GetFiles(partialName + "*.csv", SearchOption.TopDirectoryOnly).OrderByDescending(p => p.CreationTimeUtc).ToArray();

            if (file == null || file.Length <= 0)
            {
            }

            System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance);
            FileStream                stream      = File.Open(file[0].FullName, FileMode.Open, FileAccess.Read);
            IExcelDataReader          excelReader = ExcelReaderFactory.CreateCsvReader(stream);
            ExcelDataSetConfiguration excelDataSetConfiguration = new ExcelDataSetConfiguration
            {
                ConfigureDataTable = (_) => new ExcelDataTableConfiguration()
                {
                    UseHeaderRow = true
                }
            };
            var dataSet   = excelReader.AsDataSet(excelDataSetConfiguration);
            var dataTable = dataSet.Tables[0];

            IMapper            mapper = config.CreateMapper();
            List <DataRow>     rows   = new List <DataRow>(dataTable.Rows.OfType <DataRow>());
            List <DATRateData> result = mapper.Map <List <DataRow>, List <DATRateData> >(rows);

            using (var context = new DownloadContext())
            {
                context.DatRatesData.AddRange(result);
                context.SaveChanges();
            }

            excelReader.Close();
        }
        protected override void Process(ProcessorArgs args)
        {
            var download = (Download8Args)args;
              var cookies = download.Cookies;
              var localRepository = download.LocalRepository;
              var fileNames = download.FileNames;
              var links = download.Links;

              var cancellation = new CancellationTokenSource();
              var urls = links.Where(link => this.RequireDownloading(fileNames[link], localRepository)).ToArray();
              var n = urls.Length;
              for (int index = 0; index < n; index++)
              {
            var url = urls[index];
            try
            {
              var fileName = fileNames[url];
              fileName = FixFileName(fileName);

              var destFileName = Path.Combine(localRepository, fileName);
              Assert.IsTrue(!FileSystem.FileSystem.Local.File.Exists(destFileName), "The {0} file already exists".FormatWith(destFileName));
              Log.Info("Downloading " + destFileName, this);

              if (this.TryCopyFromExternalRepository(fileName, destFileName))
              {
            this.Controller.SetProgress(index * scale + scale);
            return;
              }

              var downloadOptions = new DownloadOptions
              {
            Uri = url,
            BlockOptions = new BlocksCount(WindowsSettings.AppDownloader8ParallelThreads.Value),
            CancellationToken = cancellation.Token,
            Cookies = cookies,
            FilePath = destFileName,
            RequestTimeout = new TimeSpan(0, WindowsSettings.AppDownloaderTotalTimeout.Value, 0, 0)
              };

              var done = false;
              Exception exception = null;
              var context = new DownloadContext(downloadOptions);
              context.OnProgressChanged += (x, y, z) => this.Controller.SetProgress(index * scale + z);
              context.OnDownloadCompleted += () => done = true;
              context.OnErrorOccurred += ex => exception = ex;
              ApplicationManager.AttemptToClose += (x, y) => cancellation.Cancel(true);
              context.DownloadAsync();
              while (!done && exception == null)
              {
            Thread.Sleep(1000);
              }

              if (exception != null)
              {
            cancellation.Cancel();
            throw new InvalidOperationException("Downloading file '{0}' failed with exception".FormatWith(url), exception);
              }
            }
            catch (Exception ex)
            {
              Log.Warn("An error occurred during downloading files", this, ex);

              cancellation.Cancel();
              throw new InvalidOperationException("An unhandled exception happened during downloading '{0}' file".FormatWith(url), ex);
            }
              }
        }