示例#1
0
        private string ExtractXmlLink(HttpModel xmlPage)
        {
            var html    = xmlPage.GetAsHtml();
            var xmlLink = html.DocumentNode.SelectSingleNode("//a[@name='XML']").Attributes["href"]?.Value;

            return(xmlLink);
        }
示例#2
0
        public static HttpModel AuthenticationToken(IConfiguration _config)
        {
            var                 _urlbase  = _config.GetSection("API_Access:UrlBase").Value;
            HttpClient          http      = new HttpClient();
            HttpResponseMessage respToken = http.PostAsync(
                _urlbase + "login", new StringContent(
                    JsonConvert.SerializeObject(new
            {
                UserID    = _config.GetSection("API_Access:UserID").Value,
                AccessKey = _config.GetSection("API_Access:AccessKey").Value,
            }), Encoding.UTF8, "application/json")).Result;
            string conteudo =
                respToken.Content.ReadAsStringAsync().Result;

            if (respToken.StatusCode == HttpStatusCode.OK)
            {
                Token token = JsonConvert.DeserializeObject <Token>(conteudo);
                if (token.Authenticated)
                {
                    // Associar o token aos headers do objeto
                    // do tipo HttpClient
                    http.DefaultRequestHeaders.Authorization =
                        new AuthenticationHeaderValue("Bearer", token.AccessToken);
                }
            }
            HttpModel httpmodel = new HttpModel();

            httpmodel.Http     = http;
            httpmodel._urlBase = _urlbase;
            return(httpmodel);
        }
示例#3
0
        public string Put(HttpModel model)
        {
            var result  = string.Empty;
            var bytes   = Encoding.UTF8.GetBytes(model.Serialize());
            var request = (HttpWebRequest)WebRequest.Create(model.Uri);

            request.Method        = "PUT";
            request.ContentType   = "application/json";
            request.ContentLength = bytes.Length;

            using (var writer = request.GetRequestStream())
            {
                writer.Write(bytes, 0, bytes.Length);
            }

            try
            {
                var response = (HttpWebResponse)request.GetResponse();
            }
            catch (Exception ex)
            {
                throw;
            }
            result = Get(model);
            return(result);
        }
        private static IReadOnlyCollection <HtmlNode> GetAnchors(HttpModel httpGet, BaseHttpCrawler crawler)
        {
            var html          = httpGet.GetAsHtml();
            var anchors       = html.DocumentNode.SelectNodes(".//a").ToList();
            var resultAnchors = anchors.ToList();

            foreach (var anchor in anchors)
            {
                var href    = anchor.Href();
                var baseUri = new Uri(httpGet.Url);

                var uri = new Uri(baseUri, href);
                if (String.IsNullOrEmpty(uri.Fragment) == false && uri.Fragment.StartsWith("#"))
                {
                    // hrefs with anchors (internal) should be unified because the anchors don't represent
                    // new url - they represent just location in the same document
                    // example: www.google.com#main -> www.google.com
                    var(urlWithoutFragment, _) = uri.AbsoluteUri.TupleSplit("#");
                    uri = new Uri(urlWithoutFragment);
                }

                anchor.SetAttributeValue("href", uri.AbsoluteUri.HtmlDecode());

                var additionalUrls = crawler.GenerateAdditionalUrls(anchor, null);
                resultAnchors.AddRange(additionalUrls.Select(x => HtmlNode.CreateNode($"<a href='{x}'></a>")));
            }

            // add and the caller as anchor
            var callerAnchor = html.CreateElement("a");

            callerAnchor.SetAttributeValue("href", httpGet.Url);
            resultAnchors.Add(callerAnchor);

            return(resultAnchors);
        }
示例#5
0
 public static HttpModel MapRouteParametersToMethodArguments(this HttpModel model)
 {
     foreach (var m in model.Methods)
     {
         m.MapRouteParameters();
     }
     return(model);
 }
示例#6
0
        public override NameModel CreatePackageName(HtmlNode anchor, HttpModel httpGet)
        {
            // example: https://www.ris.bka.gv.at/Dokument.wxe?ResultFunctionToken=8df6bc29-1253-484d-a41d-5db518f80b23&Abfrage=Bvwg&Entscheidungsart=Undefined&SucheNachRechtssatz=True&SucheNachText=True&GZ=&VonDatum=01.01.2014&BisDatum=14.05.2019&Norm=&ImRisSeitVonDatum=&ImRisSeitBisDatum=&ImRisSeit=Undefined&ResultPageSize=100&Suchworte=&Dokumentnummer=BVWGT_20190502_W170_2208106_1_00

            var documentNumber = new Uri(anchor.Href()).GetQueryFragmentsMap()["Dokumentnummer"];

            return NameModel.Create(documentNumber);
        }
示例#7
0
 /// <summary>
 /// 将记录插入配置并清除缓存
 /// </summary>
 public static void InsertHttmodelAndCleartemp()
 {
     if (m_lastReadModel != null)
     {
         HttpSettings.Add(m_lastReadModel);
     }
     m_lastReadModel = null;
 }
示例#8
0
 /// <summary>
 /// 加载配置文件
 /// </summary>
 /// <param name="file"></param>
 public static void LoadFromFile()
 {
     if (File.Exists(ConfigFile))
     {
         HttpSettings.Clear();
         //0表示未知,1表示Common属性,2表示http服务属性
         int                 RecordMode     = 1;
         StreamReader        fileStream     = new StreamReader(ConfigFile);
         List <PropertyInfo> commonPropList = GetPropertyInfos(AppSettings.GetType());
         List <PropertyInfo> httpPropList   = GetPropertyInfos(typeof(HttpModel));
         while (!fileStream.EndOfStream)
         {
             string lineStr = fileStream.ReadLine().Trim();
             if (!(String.IsNullOrEmpty(lineStr) || lineStr.StartsWith("#")))
             {
                 bool isSignal = false;
                 if (lineStr == "[Common]")
                 {
                     RecordMode = 1;
                     isSignal   = true;
                 }
                 else if (lineStr == "[HttpServer]")
                 {
                     RecordMode = 2;
                     isSignal   = true;
                 }
                 if (isSignal)
                 {
                     InsertHttmodelAndCleartemp();
                     if (RecordMode == 2)
                     {
                         m_lastReadModel = new HttpModel();
                     }
                 }
                 else
                 {
                     string[] lineSplit = lineStr.Split('=');
                     if (lineSplit.Length > 1)
                     {
                         string fieldName = lineSplit[0];
                         string value     = lineSplit[1];
                         if (RecordMode == 1)
                         {
                             ReadCommonSetting(fieldName, value, commonPropList);
                         }
                         else if (RecordMode == 2)
                         {
                             ReadHttpSetting(fieldName, value, httpPropList);
                         }
                     }
                 }
             }
         }
         InsertHttmodelAndCleartemp();
         fileStream.Close();
     }
     SaveToFile();
 }
示例#9
0
        public static void CorrectImplementationWithBinding(
            [ServiceBusTrigger("correct-implementation-netstandard", Connection = "ServiceBusConnection")]
            IncomingNetStandardModel netStandardModel,
            [Http(Url = "%BaseAddress%api/Echo/{Identifier}")]
            HttpModel httpModel,
            ILogger log)
        {
            log.LogInformation($"Executing ServiceBus queue trigger. Processing message: {netStandardModel.Identifier}");

            log.LogInformation($"Executed ServiceBus queue trigger. Processing message: {netStandardModel.Identifier}");
        }
示例#10
0
        /// <summary>
        /// Executes http get
        /// </summary>
        /// <param name="url">Request url</param>
        /// <param name="opt">Action for access to the underling http builder. Provides capabilities for additional configuration.</param>
        /// <returns></returns>
        public async Task <HttpModel> GetAsync(String url, Action <HttpRequestBuilder> opt)
        {
            Exception lastException = null;

            var maxRequests = this.retryCount + 1;  // + 1 indicates the initial request

            while (maxRequests > 0)
            {
                try
                {
                    var builder = new HttpRequestBuilder()
                                  .Get()
                                  .WithUrl(url)
                                  .WithTimeout(this.timeout)
                                  .WithAllowRedirect(this.allowRedirect)
                                  .UseGZip();

                    if (this.reUseCookies && this.cookies != null)
                    {
                        builder.WithCookie(this.cookies);
                    }

                    var httpResult = await builder.ExecuteRichAsync();

                    if (this.reUseCookies)
                    {
                        this.cookies = httpResult.Cookies.OfType <Cookie>().ToList();
                    }

                    opt?.Invoke(builder);

                    var encoding           = httpResult.Encoding;
                    var mimeType           = GetMimeType(httpResult);
                    var contentDisposition = httpResult.GetHeaderValue("Content-Disposition");

                    var httpGet = new HttpModel(url, httpResult.ResponseBytes, encoding, mimeType, contentDisposition, httpResult.Cookies, httpResult.StatusCode);
                    if (this.NotFoundPredicate?.Invoke(httpGet) == true)
                    {
                        throw new WebException($"{url} not found");
                    }

                    return(httpGet);
                }
                catch (Exception e)
                {
                    lastException = e;

                    maxRequests--;
                }
            }

            throw lastException;
        }
示例#11
0
        public virtual HttpModel DecodePackage(ref HttpModel httpModel)
        {
            httpModel.HttpVersion = _httpRequestInfo.HttpVersion;
            httpModel.Method      = _httpRequestInfo.Method;
            httpModel.Path        = _httpRequestInfo.Path;
            httpModel.Headers     = _httpRequestInfo.Headers;
            httpModel.Host        = _httpRequestInfo.BaseUrl;
            httpModel.ContentType = _httpRequestInfo.ContentType;
            httpModel.Content     = _httpRequestInfo.ContentLength > 0 ? DataHelper.Compress(_httpRequestInfo.Body) : null;

            return(httpModel);
        }
示例#12
0
        public virtual HttpModel DecodePackage(ref HttpModel httpModel)
        {
            httpModel.HttpVersion = _httpRequestInfo.HttpVersion;
            httpModel.Method      = _httpRequestInfo.Method;
            httpModel.Path        = _httpRequestInfo.Path;
            httpModel.Headers     = _httpRequestInfo.Headers;
            httpModel.Host        = _httpRequestInfo.BaseUrl;
            httpModel.ContentType = _httpRequestInfo.ContentType;
            httpModel.Content     = _httpRequestInfo.Body;

            return(httpModel);
        }
示例#13
0
        static void Main(string[] args)
        {
            if (args.Length == 0)
            {
                Console.WriteLine("Required path to dll");
                return;
            }
            // input.dll outputDir references.txt

            var path                = args[0];
            var outputPath          = args.Length > 1 ? args[1] : null;
            var directory           = Path.GetDirectoryName(path);
            var referencePaths      = args.Length > 2 ? File.ReadAllLines(args[2]) : new string[0];
            var resolver            = new PathAssemblyResolver(referencePaths);
            var corAssembly         = referencePaths.Where(m => m.Contains("mscorlib")).Select(a => AssemblyName.GetAssemblyName(a).FullName).FirstOrDefault();
            var metadataLoadContext = new MetadataLoadContext(resolver, corAssembly);
            var uControllerAssembly = metadataLoadContext.LoadFromAssemblyName(typeof(HttpHandler).Assembly.FullName);
            var handler             = uControllerAssembly.GetType(typeof(HttpHandler).FullName);
            var assembly            = metadataLoadContext.LoadFromAssemblyPath(path);

            var models = new List <HttpModel>();

            foreach (var type in assembly.GetExportedTypes())
            {
                if (handler.IsAssignableFrom(type))
                {
                    var model = HttpModel.FromType(type);
                    models.Add(model);
                }
            }

            if (models.Count > 0 && outputPath != null)
            {
                Directory.CreateDirectory(outputPath);
            }

            foreach (var model in models)
            {
                var gen = new CodeGenerator(model, metadataLoadContext);
                if (outputPath != null)
                {
                    var fileName = Path.Combine(outputPath, model.HandlerType.Name + ".RouteProvider.cs");
                    File.WriteAllText(fileName, gen.Generate());
                }
                else
                {
                    Console.WriteLine(gen.Generate());
                }
            }
        }
示例#14
0
        public string Get(HttpModel model)
        {
            var result  = string.Empty;
            var request = (HttpWebRequest)WebRequest.Create(model.Uri);

            request.Method      = "GET";
            request.ContentType = "application/json";
            using (var response = request.GetResponse())
                using (var stream = response.GetResponseStream())
                    using (var reader = new StreamReader(stream))
                    {
                        result = reader.ReadToEnd();
                    }
            return(result);
        }
示例#15
0
        private HttpModel CreatHttpModel(PacketData data, string message)
        {
            HttpModel ls = new HttpModel();

            ls.IP_DestinationAddress = string.Join(",", data.IP_DestinationAddress);
            ls.IP_SourceAddress      = string.Join(",", data.IP_SourceAddress);
            ls.TCP_DestinationPort   = data.TCP_DestinationPort;
            ls.TCP_SourcePort        = data.TCP_SourcePort;
            ls.Time = data.Time;

            string[] a = message.Split(new string[] { "\r\n\r\n" }, StringSplitOptions.None);
            ls.Content = a[1];
            string[] b = message.Split(new string[] { "\r\n" }, StringSplitOptions.None);
            ls.Head = b[0];
            ls.Body = a[0].Replace(b[0] + "\r\n", "");
            return(ls);
        }
示例#16
0
 public void ProcessData(NatSession session, NatRequestInfo requestInfo, HttpModel httpModel)
 {
     try
     {
         switch (requestInfo.Body.Action)
         {
         case (int)HttpAction.Response:
         {
             var context = ContextDict.FirstOrDefault(c => c.Key == httpModel.SessionId).Value;
             if (context == null)
             {
                 HandleLog.WriteLine($"未找到上下文context,SessionId={httpModel.SessionId}");
                 return;
             }
             using var stream = context.Response.OutputStream;
             if (httpModel.Content?.Length > 0)
             {
                 //解压
                 var byteData = DataHelper.Decompress(httpModel.Content);
                 var res      = byteData.ToUTF8String();
                 foreach (var item in httpModel.Headers)
                 {
                     context.Response.AddHeader(item.Key, item.Value);
                 }
                 context.Response.StatusCode      = httpModel.StatusCode;
                 context.Response.ContentType     = httpModel.ContentType;
                 context.Response.ContentLength64 = byteData.Length;
                 //把处理信息返回到客户端
                 stream.WriteAsync(byteData, 0, byteData.Length).ContinueWith((t) =>
                     {
                         var timeSpan  = (DateTime.Now - httpModel.RequestTime);
                         var totalSize = byteData.Length * 1.00 / 1024;
                         var speed     = Math.Round(totalSize / timeSpan.TotalSeconds, 1);
                         HandleLog.WriteLine($"{session.Client.user_name} {session.Client.name} {context.Request.HttpMethod} {context.Request.Url.AbsoluteUri} {httpModel.StatusCode} {httpModel.StatusMessage} {Math.Round(totalSize, 1)}KB {timeSpan.TotalMilliseconds}ms {speed}KB/s");
                         ContextDict.Remove(httpModel.SessionId);
                     });
             }
         }
         break;
         }
     }
     catch (Exception ex)
     {
         HandleLog.WriteLine($"HttpServer ProcessData穿透处理异常,{ex}");
     }
 }
示例#17
0
 private void Received(HttpSession session, HttpRequestInfo requestInfo)
 {
     try
     {
         var httpModel = new HttpModel
         {
             RequestTime = DateTime.Now,
             ServerId    = ServerId,
             SessionId   = session.SessionId
         };
         var filter = ReceiveFilter as HttpReceiveFilter;
         filter.DecodePackage(ref httpModel);
         //转发请求
         var natSession = NATServer.GetSingle(c => c.MapList.Any(c => c.remote_endpoint == httpModel.Host));
         if (natSession == null)
         {
             //TODO 错误页面
             HandleLog.WriteLine($"穿透客户端未连接到服务器,请求地址:{httpModel.Host}{httpModel.Path}");
             var response = new HttpResponse()
             {
                 Status      = 404,
                 ContentType = "text/html",
                 Body        = Encoding.UTF8.GetBytes("nat client not found")
             };
             //把处理信息返回到客户端
             session.Send(response.Write());
         }
         else
         {
             //转发数据
             var pack = new JsonData()
             {
                 Type   = (int)JsonType.HTTP,
                 Action = (int)HttpAction.Request,
                 Data   = httpModel.ToJson()
             };
             natSession.Send(PackHelper.CreatePack(pack));
             session.NatSession = natSession;
         }
     }
     catch (Exception ex)
     {
         HandleLog.WriteLine($"【{session.Local}】请求地址:{requestInfo.BaseUrl}{requestInfo.Path},处理发生异常:{ex}");
     }
 }
示例#18
0
        public HttpModel MatchHttpModel(string domain, List <HttpModel> httpModels)
        {
            string    webDomain = domain.Split(':').FirstOrDefault();
            HttpModel ret       = httpModels.Where(t =>
            {
                if (webDomain.StartsWith(t.Domain))
                {
                    return(true);
                }
                return(false);
            }).FirstOrDefault();

            if (ret == null)
            {
                ret = httpModels.Where(t => t.Type.ToLower() != "http").FirstOrDefault();
            }
            return(ret);
        }
示例#19
0
        public void ProcessData(NatSession session, NatRequestInfo requestInfo, HttpModel httpModel)
        {
            try
            {
                switch (requestInfo.Body.Action)
                {
                case (int)HttpAction.Response:
                {
                    var context = GetSingle(c => c.SessionId == httpModel.SessionId);
                    if (context == null)
                    {
                        HandleLog.WriteLine($"未找到上下文context,SessionId={httpModel.SessionId}");
                        return;
                    }
                    HttpResponse httpResponse = new HttpResponse()
                    {
                        HttpVersion   = httpModel.HttpVersion,
                        Headers       = httpModel.Headers,
                        Status        = httpModel.StatusCode,
                        StatusMessage = httpModel.StatusMessage
                    };
                    if (httpModel.Content?.Length > 0)
                    {
                        //解压
                        var byteData = DataHelper.Decompress(httpModel.Content);
                        httpResponse.ContentType = httpModel.ContentType;
                        httpResponse.Body        = byteData;
                    }
                    //把处理信息返回到客户端
                    context.Send(httpResponse.Write());

                    var timeSpan  = (DateTime.Now - httpModel.RequestTime);
                    var totalSize = (httpResponse.Body?.Length ?? 0) * 1.00 / 1024;
                    var map       = session.MapList.Find(c => c.remote_endpoint == httpModel.Host);
                    HandleLog.WriteLine($"{session.Client.user_name} {session.Client.name} {map?.name} {httpModel.Method} {httpModel.Path} {httpModel.StatusCode} {httpModel.StatusMessage} {Math.Round(totalSize, 1)}KB {timeSpan.TotalMilliseconds}ms");
                }
                break;
                }
            }
            catch (Exception ex)
            {
                HandleLog.WriteLine($"HttpsServer ProcessData穿透处理异常,{ex}");
            }
        }
示例#20
0
 public static HttpModel MapComplexTypeArgsToFromBody(this HttpModel model)
 {
     foreach (var m in model.Methods)
     {
         if (HttpMethods.IsPost(m.HttpMethod) || HttpMethods.IsPut(m.HttpMethod) || m.HttpMethod == null)
         {
             foreach (var p in m.Parameters)
             {
                 if (p.HasBindingSource)
                 {
                     continue;
                 }
                 // This is an MVC heuristic
                 p.FromBody = !TypeDescriptor.GetConverter(p.ParameterType).CanConvertFrom(typeof(string));
             }
         }
     }
     return(model);
 }
示例#21
0
        public override string GetTwoLetterLanguage(HttpModel httpGet, HtmlNode anchor)
        {
            // example: https://eur-lex.europa.eu/legal-content/BG/TXT/HTML/?uri=CELEX:32015R0848&qid=1557822576221&from=EN
            var twoLetterLang = anchor.Href()
                                .Split(new String[] { "legal-content" }, StringSplitOptions.RemoveEmptyEntries)
                                [1]
                                .Split(new String[] { "/" }, StringSplitOptions.RemoveEmptyEntries)
                                [0];

            var celex = TryGetCelexFromUrl(anchor.Href());

            // sector 8 languages could be 25 in number
            if (celex.StartsWith("8") == false && twoLetterLang.IsIn(allowedLanguages) == false)
            {
                throw new ArgumentException($"Not allowed language for download");
            }

            return(twoLetterLang);
        }
 public override NameModel CreatePackageDocumentName(HtmlNode anchor, HttpModel httpGet)
 {
     if (IsCasePage(anchor))
     {
         return(NameModel.Create("detail"));
     }
     else if (IsJudgmentPdf(anchor))
     {
         return(NameModel.Create("content"));
     }
     else if (IsPressSummaryPdf(anchor))
     {
         return(NameModel.Create("presssummary"));
     }
     else
     {
         throw new ArgumentException($"Unknown anchor pattern for package document name. Url {anchor.Href()}");
     }
 }
示例#23
0
 /// <summary>
 /// 将记录插入配置并清除缓存
 /// </summary>
 public static void InsertHttmodelAndCleartemp()
 {
     if (m_lastReadModel != null && m_lastReadModel.Port > 0)
     {
         if (HttpSettings.ContainsKey(m_lastReadModel.Port))
         {
             HttpSettings[m_lastReadModel.Port].Add(m_lastReadModel);
         }
         else
         {
             List <HttpModel> list = new List <HttpModel>()
             {
                 m_lastReadModel
             };
             HttpSettings.Add(m_lastReadModel.Port, list);
         }
     }
     m_lastReadModel = null;
 }
示例#24
0
        private static void DispatcherPacket(List <HttpModel> Packist)
        {
            IEnumerator itor = Packist.GetEnumerator();

            while (itor.MoveNext())
            {
                HttpModel data = itor.Current as HttpModel;
                if (data.TCP_SourcePort == 80)
                {
                    // res
                    if (!mp.ContainsKey(data.TCP_DestinationPort))
                    {
                        List <HttpModel> newlist = new List <HttpModel>();
                        newlist.Add(data);
                        mp.Add(data.TCP_DestinationPort, newlist);
                    }
                    else
                    {
                        List <HttpModel> oldlist;
                        mp.TryGetValue(data.TCP_DestinationPort, out oldlist);
                        oldlist.Add(data);
                    }
                }
                else if (data.TCP_DestinationPort == 80)
                {
                    // req
                    if (!mp.ContainsKey(data.TCP_SourcePort))
                    {
                        List <HttpModel> newlist = new List <HttpModel>();
                        newlist.Add(data);
                        mp.Add(data.TCP_SourcePort, newlist);
                    }
                    else
                    {
                        List <HttpModel> oldlist;
                        mp.TryGetValue(data.TCP_SourcePort, out oldlist);
                        oldlist.Add(data);
                    }
                }
            }
        }
示例#25
0
        private DocumentModel GetDocumentModel(HttpModel document, string docNumber)
        {
            if (document.MimeType == "application/pdf")
            {
                return(new DocumentModel
                {
                    Format = document.MimeType,
                    Url = document.Url,
                    Raw = document.Raw,
                    Name = docNumber + ".pdf"
                });
            }

            return(new DocumentModel
            {
                Format = document.MimeType,
                Url = document.Url,
                Raw = document.Raw,
                Name = docNumber + ".html"
            });
        }
示例#26
0
        public void Execute(SourceGeneratorContext context)
        {
            // For debugging
            // System.Diagnostics.Debugger.Launch();

            var metadataLoadContext = new MetadataLoadContext(context.Compilation);
            var uControllerAssembly = metadataLoadContext.LoadFromAssemblyName("uController");
            var handler             = uControllerAssembly.GetType(typeof(HttpHandler).FullName);
            var assembly            = metadataLoadContext.MainAssembly;

            var models = new List <HttpModel>();

            foreach (var type in assembly.GetExportedTypes())
            {
                if (handler.IsAssignableFrom(type))
                {
                    var model = HttpModel.FromType(type);
                    models.Add(model);
                }
            }

            foreach (var model in models)
            {
                var gen        = new CodeGenerator(model, metadataLoadContext);
                var rawSource  = gen.Generate();
                var sourceText = SourceText.From(rawSource, Encoding.UTF8);

                // For debugging
                //var comp = context.Compilation.AddSyntaxTrees(CSharpSyntaxTree.ParseText(sourceText));
                //var diagnosrics = comp.GetDiagnostics();

                context.AddSource(model.HandlerType.Name + "RouteExtensions", sourceText);

                //if (gen.FromBodyTypes.Any())
                //{
                //    var jsonGenerator = new JsonCodeGenerator(metadataLoadContext, model.HandlerType.Namespace);
                //    var generatedConverters = jsonGenerator.Generate(gen.FromBodyTypes, out var helperSource);
                //}
            }
        }
示例#27
0
        public override NameModel CreatePackageName(HtmlNode anchor, HttpModel httpGet)
        {
            // example: https://eur-lex.europa.eu/legal-content/BG/TXT/HTML/?uri=CELEX:32015R0848&qid=1557822576221&from=EN

            var twoLetterLang = this.GetTwoLetterLanguage(httpGet, anchor);

            if (TryGetCelexFromUrl(anchor.Href()) is String celex)
            {
                if (celex.StartsWith("8"))
                {
                    return(NameModel.Create(celex.Replace("(", "_").Replace(")", String.Empty)));
                }
                else
                {
                    return(NameModel.Create($"{celex}_{twoLetterLang}"));
                }
            }
            else
            {
                throw new ArgumentException($"Could not package name from: {anchor.Href()}");
            }
        }
示例#28
0
        public static HttpModel MapMethodNamesToHttpMethods(this HttpModel model)
        {
            foreach (var m in model.Methods)
            {
                if (m.MethodInfo.Name.StartsWith("Get", StringComparison.OrdinalIgnoreCase))
                {
                    m.Get();
                }
                else if (m.MethodInfo.Name.StartsWith("Post", StringComparison.OrdinalIgnoreCase))
                {
                    m.Post();
                }
                else if (m.MethodInfo.Name.StartsWith("Put", StringComparison.OrdinalIgnoreCase))
                {
                    m.Put();
                }
                else if (m.MethodInfo.Name.StartsWith("Delete", StringComparison.OrdinalIgnoreCase))
                {
                    m.Delete();
                }
            }

            return(model);
        }
示例#29
0
        private async Task <HttpModel> LogRequest(HttpContext context)
        {
            string requestBody;

            context.Request.EnableBuffering();

            await using var requestStream = _recyclableMemoryStreamManager.GetStream();
            await context.Request.Body.CopyToAsync(requestStream);

            requestBody = ReadStreamInChunks(requestStream);

            context.Request.Body.Position = 0;

            var logHttp = new HttpModel();

            logHttp.RequestMethod = context.Request.Method;
            logHttp.Url           = context.Request.Path;
            logHttp.Path          = context.Request.Path;
            logHttp.Protocol      = context.Request.Scheme;
            logHttp.RequestHeader = context.Request.Headers.ToJson();
            logHttp.RequestBody   = requestBody;

            return(logHttp);
        }
示例#30
0
        private async void ForwardProxy(HttpSession session, HttpModel httpModel, Map map)
        {
            try
            {
                using HttpRequestMessage httpRequest = new HttpRequestMessage()
                      {
                          Method     = new HttpMethod(httpModel.Method),
                          RequestUri = new Uri($"{map.protocol}://{map.local_endpoint}{httpModel.Path}")
                      };
                HandleLog.WriteLine($"{map.name} {httpModel.Method} {httpRequest.RequestUri.AbsoluteUri} {httpModel.Headers.ToJson()}");
                if (httpRequest.Method != HttpMethod.Get && httpModel.Content?.Length > 0)
                {
                    var body    = httpModel.Content;
                    var bodyStr = body.ToUTF8String();
                    //记录请求小于1kb的参数
                    if (httpModel.Content.Length < 1024)
                    {
                        HandleLog.WriteLine($"{map.name} {httpModel.Method} {httpRequest.RequestUri.AbsoluteUri} {bodyStr}");
                    }
                    httpRequest.Content = new StringContent(bodyStr, Encoding.UTF8, httpModel.ContentType.Split(";")[0]);
                }
                using HttpClient _httpClient = new HttpClient();
                //替换Host 不然400 Bad Request
                //httpModel.Headers["Host"] = map.local_endpoint;
                foreach (var item in httpModel.Headers)
                {
                    if (!item.Key.EqualsWhithNoCase("Content-Type"))
                    {
                        if (!httpRequest.Content?.Headers.TryAddWithoutValidation(item.Key, item.Value) ?? true)
                        {
                            _httpClient.DefaultRequestHeaders.TryAddWithoutValidation(item.Key, item.Value);
                        }
                    }
                }
                if (map.protocol == "https")
                {
                    ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12 | SecurityProtocolType.Tls11 | SecurityProtocolType.Tls;
                }
                var response = await _httpClient.SendAsync(httpRequest);

                //回传给浏览器
                var respHttpModel = new HttpModel
                {
                    HttpVersion   = $"{map.protocol.ToUpper()}/{response.Version.ToString()}",
                    StatusCode    = (int)response.StatusCode,
                    StatusMessage = response.StatusCode.ToString(),
                    Local         = map.local_endpoint,
                    Headers       = response.Headers.ToDictionary(),
                    ResponseTime  = DateTime.Now
                };
                foreach (var item in response.Content.Headers)
                {
                    if (item.Key.EqualsWhithNoCase("Content-Type"))
                    {
                        respHttpModel.ContentType = string.Join(";", item.Value);
                    }
                    else
                    {
                        if (item.Key.EqualsWhithNoCase("Content-Length"))
                        {
                            continue;
                        }
                        respHttpModel.Headers.Add(item.Key, string.Join(";", item.Value));
                    }
                }
                respHttpModel.Headers.Remove("Transfer-Encoding");//response收到的是完整的 这个响应头要去掉 不然浏览器解析出错
                respHttpModel.Content = DataHelper.StreamToBytes(response.Content.ReadAsStreamAsync().Result);

                HttpResponse httpResponse = new HttpResponse()
                {
                    HttpVersion   = respHttpModel.HttpVersion,
                    Headers       = respHttpModel.Headers,
                    Status        = respHttpModel.StatusCode,
                    StatusMessage = respHttpModel.StatusMessage
                };
                if (respHttpModel.Content?.Length > 0)
                {
                    httpResponse.ContentType = respHttpModel.ContentType;
                    httpResponse.Body        = respHttpModel.Content;
                }
                //把处理信息返回到客户端
                session.Send(httpResponse.Write());

                var timeSpan  = (DateTime.Now - httpModel.RequestTime);
                var totalSize = (httpResponse.Body?.Length ?? 0) * 1.00 / 1024;
                HandleLog.WriteLine($"{map.user_name} {map.client_name} {map?.name} {httpModel.Method} {httpModel.Path} {respHttpModel.StatusCode} {respHttpModel.StatusMessage} {Math.Round(totalSize, 1)}KB {timeSpan.TotalMilliseconds}ms");
            }
            catch (Exception ex)
            {
                HandleLog.WriteLine($"【{session.Local}】请求地址:{map.protocol}://{httpModel.Host}{httpModel.Path},正向代理异常:{ex}");
                //把处理信息返回到客户端
                session.Write("server error");
            }
        }