Ejemplo n.º 1
0
        private static void SendBuffers(Stream stream, IEnumerable <byte[]> messages)
        {
            var response = new byte[1];

            byte[] header = new byte[16];
            foreach (var message in messages)
            {
                Array.Copy(BitConverter.GetBytes(message.LongLength), 0, header, 0, sizeof(UInt64));            //8 bytes
                Array.Copy(BitConverter.GetBytes(Crc64.Compute(0, message)), 0, header, 8, sizeof(UInt64));     //8 bytes

                stream.Write(header, 0, header.Length);
                //log.Debug($"Written {header.Length} header size, message {message.LongLength} bytes");

                //log.Debug($"Write message.");
                stream.Write(message, 0, message.Length);
                //log.Debug($"Message written.");

                //var bytesRead = stream.Read(response, 0, 1);
                ////log.Debug($"Read response");
                //if (bytesRead == 1 && response[0] == '1')
                //{
                //    //log.Debug($"Message was sent successfully.");
                //}
                //else
                //{
                //    throw new Exception("unable to write to nework stream");
                //}
            }
        }
Ejemplo n.º 2
0
        public static void TestCoreCache(string path, string domain)
        {
            log.Info("GetCoreCacheManual()");

            using (var stream = new System.IO.FileStream(path, System.IO.FileMode.Open))
            {
                log.Debug($"Deserialize.");
                var cache = ProtoBuf.Serializer.Deserialize <Models.Cache>(stream);
                log.Debug($"Deserialized.");

                if (cache.CustomLists != null)
                {
                    log.Debug($"Custom List count = {cache.CustomLists.ToArray().Count()}");
                }
                if (cache.Domains != null)
                {
                    log.Debug($"Domains count = {cache.Domains.ToArray().Count()}");
                }
                if (cache.IPRanges != null)
                {
                    log.Debug($"IPRanges count = {cache.IPRanges.ToArray().Count()}");
                }
                if (cache.Policies != null)
                {
                    log.Debug($"Policies count = {cache.Policies.ToArray().Count()}");
                }

                var crc64 = Crc64.Compute(0, ASCIIEncoding.ASCII.GetBytes(domain));
                if (cache.Domains.Any(t => t.Crc64 == crc64))
                {
                    log.Debug("contains " + domain);
                }
            }
        }
Ejemplo n.º 3
0
        private static List <Models.CacheDomain> LoadDomains()
        {
            var result = new List <Models.CacheDomain>();
            var csv    = new CsvHelper.CsvReader(File.OpenText("domains.csv"));

            while (csv.Read())
            {
                var b0  = Convert.ToInt32(csv.GetField(2));
                var b1  = Convert.ToInt32(csv.GetField(3));
                var b2  = Convert.ToInt32(csv.GetField(4));
                var b3  = Convert.ToInt32(csv.GetField(5));
                var b4  = Convert.ToInt32(csv.GetField(6));
                var b5  = Convert.ToInt32(csv.GetField(7));
                var b6  = Convert.ToInt32(csv.GetField(8));
                var b7  = Convert.ToInt32(csv.GetField(9));
                var b8  = Convert.ToInt32(csv.GetField(10));
                var b9  = Convert.ToInt32(csv.GetField(11));
                var b10 = Convert.ToInt32(csv.GetField(12));
                var b11 = Convert.ToInt32(csv.GetField(13));
                var b12 = Convert.ToInt32(csv.GetField(14));
                var b13 = Convert.ToInt32(csv.GetField(15));
                var b14 = Convert.ToInt32(csv.GetField(16));
                var b15 = Convert.ToInt32(csv.GetField(17));

                var item = new Models.CacheDomain
                {
                    Crc64    = Crc64.Compute(0, ASCIIEncoding.ASCII.GetBytes(csv.GetField(0))),
                    Accuracy = Convert.ToInt32(csv.GetField(1)),
                    Flags    = new [] { b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, b10, b11, b12, b13, b14 }
                };
                result.Add(item);
            }

            return(result);
        }
Ejemplo n.º 4
0
        private static int TaskTCPJob(object args)
        {
            TaskTcpArgs          arg     = (TaskTcpArgs)args;
            int                  port    = arg.port;
            bufferType           buftype = arg.buftype;
            IEnumerable <byte[]> data    = arg.data;

            try
            {
                using (var client = new TcpClient())
                {
                    client.Client.Connect(IPAddress.Parse(Configuration.GetKres()), port);

                    var messageType = (int)buftype;

                    //+ 8 bytes (= last crc64 bytes)
                    var header = BitConverter.GetBytes(messageType)             // 4 bytes -> message type
                                 .Concat(BitConverter.GetBytes(data.Count()));  // 4 bytes -> how many buffers

                    var headerCrc = Crc64.Compute(0, header.ToArray());
                    header = header.Concat(BitConverter.GetBytes(headerCrc));   // 8 byte -> crc of this header

                    //log.Debug($"Get stream");
                    using (var stream = client.GetStream())
                    {
                        if (SendHeader(stream, header))
                        {
                            SendBuffers(stream, data);
                        }

                        //log.Debug($"Closing ip stream.");
                        stream.Flush();
                        stream.Close();
                    }
                    client.Close();

                    log.Debug($"Kres {port} job {buftype} updated");
                }
            }
            catch (Exception ex)
            {
                log.Error($"Unable to connect to kres on port {port}, ex: {ex.Message}");
            }

            return(0);
        }
Ejemplo n.º 5
0
        private static int TaskHDDJob(object args)
        {
            TaskHddArgs          arg      = (TaskHddArgs)args;
            string               filename = arg.filename;
            bufferType           buftype  = arg.buftype;
            IEnumerable <byte[]> data     = arg.data;

            try
            {
                using (var stream = new FileStream(filename, FileMode.Append, FileAccess.Write, FileShare.None))
                {
                    var messageType = (int)buftype;

                    //+ 8 bytes (= last crc64 bytes)
                    var header = BitConverter.GetBytes(messageType)             // 4 bytes -> message type
                                 .Concat(BitConverter.GetBytes(data.Count()));  // 4 bytes -> how many buffers

                    var headerCrc = Crc64.Compute(0, header.ToArray());
                    header = header.Concat(BitConverter.GetBytes(headerCrc));   // 8 byte -> crc of this header

                    //log.Debug($"Get stream");
                    //using (var stream = client.GetStream())
                    {
                        if (SendHeader(stream, header))
                        {
                            SendBuffers(stream, data);
                        }

                        //log.Debug($"Closing ip stream.");
                        stream.Flush();
                        stream.Close();
                    }
                    //client.Close();

                    log.Debug($"File {filename} was updated.");
                }
            }
            catch (Exception ex)
            {
                log.Error($"Unable to write to {filename}. Ex: {ex}");
            }

            return(0);
        }
Ejemplo n.º 6
0
        private static byte[] ReadMessage(FileStream stream)
        {
            byte[] buffer = new byte[16];
            if (stream.Read(buffer, 0, buffer.Length) != buffer.Length)
            {
                throw new Exception("Unable to read messageheader");
            }

            var length = BitConverter.ToInt64(buffer, 0);
            var crc64  = BitConverter.ToInt64(buffer, 8);

            buffer = new byte[length];
            if (stream.Read(buffer, 0, (int)length) != length)
            {
                throw new Exception("Unable to read message");
            }

            var computedCrc = Crc64.Compute(0, buffer);

            return(buffer);
        }
Ejemplo n.º 7
0
        public string Bypass(HttpContext ctx, string clientIpAddress, string domainToWhitelist, string authToken, string base64encodedUrlToRedirectTo)
        {
            log.Info($"Bypass request, ip={clientIpAddress}, {domainToWhitelist}.");

            if (string.Compare(authToken, "BFLMPSVZ", StringComparison.OrdinalIgnoreCase) != 0)
            {
                return("");
            }

            var    idbytes  = Encoding.ASCII.GetBytes(clientIpAddress);
            var    idcrc    = Crc64.Compute(0, idbytes);
            string identity = idcrc.ToString("X");

            IPAddress ip;

            if (!IPAddress.TryParse(clientIpAddress, out ip))
            {
                return(new Exception($"unable to parse ip address {clientIpAddress}.").Message);
            }

            //TODO: check ipv6 reverse
            var bytes = ip.GetAddressBytes().Reverse().ToArray();

            BigMath.Int128 intip;
            if (bytes.Length == 4)
            {
                intip = new BigMath.Int128(0, BitConverter.ToUInt32(bytes, 0));
            }
            else if (bytes.Length == 16)
            {
                intip = new BigMath.Int128(BitConverter.ToUInt64(bytes, 0), BitConverter.ToUInt64(bytes, 8));
            }
            else
            {
                return(new Exception($"unable to parse ip address {clientIpAddress}.").Message);
            }

            List <Models.CacheIPRange>    ipranges;
            List <Models.CacheCustomList> customlists;

            if (CacheLiveStorage.CoreCache.IPRanges != null)
            {
                ipranges = CacheLiveStorage.CoreCache.IPRanges.ToList();
            }
            else
            {
                ipranges = new List <Models.CacheIPRange>();
            }

            if (CacheLiveStorage.CoreCache.IPRanges != null)
            {
                customlists = CacheLiveStorage.CoreCache.CustomLists.ToList();
            }
            else
            {
                customlists = new List <Models.CacheCustomList>();
            }


            ipranges.Add(new Models.CacheIPRange()
            {
                Identity     = identity,
                Proto_IpFrom = Encoding.ASCII.GetBytes(intip.ToString()),
                Proto_IpTo   = Encoding.ASCII.GetBytes(intip.ToString()),
                PolicyId     = 0
            });
            var item = customlists.FirstOrDefault(t => string.Compare(t.Identity, identity, StringComparison.OrdinalIgnoreCase) == 0);

            if (item == null)
            {
                item = new Models.CacheCustomList()
                {
                    Identity  = identity,
                    WhiteList = new List <string>()
                    {
                        domainToWhitelist
                    },
                    BlackList = new List <string>(),
                    PolicyId  = 0
                };
                log.Info($"Identity {identity} now has {domainToWhitelist} whitelisted.");
            }
            else
            {
                if (!item.WhiteList.Contains(domainToWhitelist))
                {
                    var list = item.WhiteList.ToList();
                    list.Add(domainToWhitelist);
                    item.WhiteList = list;
                    foreach (var entry in item.WhiteList)
                    {
                        log.Info($"Identity {identity} now has {entry} whitelisted.");
                    }
                }
                else
                {
                    log.Info($"Identity {identity} has {domainToWhitelist} already whitelisted.");
                }
            }
            customlists.RemoveAll(t => string.Compare(t.Identity, identity, StringComparison.OrdinalIgnoreCase) == 0);
            customlists.Add(item);

            CacheLiveStorage.CoreCache.IPRanges    = ipranges;
            CacheLiveStorage.CoreCache.CustomLists = customlists;

            log.Info($"Updating kres modules.");
            KresUpdater.UpdateSmallCaches();
            KresUpdater.UpdateNow();
            log.Info($"Kres modules have been updated.");

            //var redirectUrl = Base64Decode(base64encodedUrlToRedirectTo);
            //ctx.Response.RedirectLocation = redirectUrl;


            return(null);
        }
Ejemplo n.º 8
0
        public string PassThrough(HttpContext ctx, string postdata)
        {
            try
            {
                var list = ctx.Request.Host.ToString().Split('.');

                if (!string.IsNullOrEmpty(postdata))
                {
                    list = postdata.Split('.');
                }

                for (var i = 0; i < list.Length - 1; i++)
                {
                    var joined = string.Join('.', list, i, list.Length - i);
                    var bytes  = Encoding.ASCII.GetBytes(joined);
                    var crc    = Crc64.Compute(0, bytes);
                    if (CacheLiveStorage.CoreCache.Domains == null)
                    {
                        break;
                    }

                    var domain = CacheLiveStorage.CoreCache.Domains.Where(t => t.Crc64 == crc).FirstOrDefault();
                    if (domain == null)
                    {
                        continue;
                    }

                    var ipaddress = ctx.Connection.RemoteIpAddress;
                    var addrbytes = ipaddress.GetAddressBytes().Reverse().ToArray();
                    var addr      = new Models.Int128();
                    if (addrbytes.Length == 4)
                    {
                        addr.Hi  = 0;
                        addr.Low = BitConverter.ToUInt32(addrbytes, 0);
                    }
                    else if (addrbytes.Length == 16)
                    {
                        addr.Hi  = BitConverter.ToUInt64(addrbytes, 0);
                        addr.Low = BitConverter.ToUInt64(addrbytes, 8);
                    }
                    var ip = new BigMath.Int128(addr.Hi, addr.Low);

                    var ipRange = new List <Models.CacheIPRange>();
                    if (CacheLiveStorage.UdpCache != null)
                    {
                        ipRange = CacheLiveStorage.UdpCache.Select(t => t.Value).ToList();
                        Models.CacheIPRange[] ipRangeCore = null;
                        if (CacheLiveStorage.CoreCache.IPRanges != null)
                        {
                            ipRangeCore = CacheLiveStorage.CoreCache.IPRanges.ToArray();
                            ipRange     = ipRange.Concat(ipRangeCore).ToList();
                        }
                    }

                    var range          = ipRange.Where(t => t.BintFrom >= ip && t.BintTo <= ip).FirstOrDefault();
                    var range_identity = string.Empty;
                    var range_policyid = 0;
                    if (range != null)
                    {
                        range_identity = range.Identity;
                        range_policyid = range.PolicyId;
                    }

                    Models.PublicListenerConfig views;
                    using (var sr = new StreamReader("publiclistenerconfig.json"))
                    {
                        views = JsonConvert.DeserializeObject <Models.PublicListenerConfig>(sr.ReadToEnd());
                    }

                    foreach (var network in views.views)
                    {
                        foreach (var cidr in network.networks)
                        {
                            IPNetwork n = IPNetwork.Parse(cidr.ToString());
                            if (!n.Contains(ipaddress))
                            {
                                continue;
                            }

                            if (!string.IsNullOrEmpty(range_identity))
                            {
                                if (CacheLiveStorage.CoreCache.CustomLists != null)
                                {
                                    var custom = CacheLiveStorage.CoreCache.CustomLists.Where(t => string.Compare(t.Identity, range_identity, StringComparison.OrdinalIgnoreCase) == 0).FirstOrDefault();
                                    if (custom != null)
                                    {
                                        var dmn = string.Join('.', list);
                                        if (custom.WhiteList.Contains(joined) || custom.WhiteList.Contains(dmn))
                                        {
                                            //allow
                                            //return "allow";
                                            return(GenerateContent(ctx, network.accuracy));
                                        }
                                        else if (custom.BlackList.Contains(joined) || custom.BlackList.Contains(dmn))
                                        {
                                            return(GenerateContent(ctx, network.blacklist));
                                        }
                                    }
                                }
                            }

                            if (CacheLiveStorage.CoreCache.Policies != null)
                            {
                                var policy = CacheLiveStorage.CoreCache.Policies.Where(t => t.Policy_id == range_policyid).FirstOrDefault();
                                if (policy == null)
                                {
                                    //no policy
                                    //return "no policy";
                                    return(GenerateContent(ctx, network.blacklist));
                                }
                                else
                                {
                                    var flags = domain.Flags.ToArray()[policy.Policy_id];

                                    if ((flags & (int)KresFlags.flags_accuracy) == (int)KresFlags.flags_accuracy)
                                    {
                                        if (policy.Block > 0 && domain.Accuracy > policy.Block)
                                        {
                                            return(GenerateContent(ctx, network.accuracy));
                                        }
                                        else
                                        {
                                            if (policy.Audit > 0 && domain.Accuracy > policy.Audit)
                                            {
                                                //audit
                                                //return "audit";
                                                return(GenerateContent(ctx, network.blacklist));
                                            }
                                            else
                                            {
                                                //no accuracy action
                                                //return "no accuracy action";
                                                return(GenerateContent(ctx, network.blacklist));
                                            }
                                        }
                                    }
                                    if ((flags & (int)KresFlags.flags_whitelist) == (int)KresFlags.flags_whitelist)
                                    {
                                        //allow whitelist
                                        //return "allow whitelist";
                                        return(GenerateContent(ctx, network.blacklist));
                                    }
                                    if ((flags & (int)KresFlags.flags_blacklist) == (int)KresFlags.flags_blacklist)
                                    {
                                        //block
                                        return(GenerateContent(ctx, network.legal));
                                    }
                                }
                            }
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                log.Error("ERROR: ", ex);
            }

            //return "no-action";
            return(GenerateContent(ctx, new[] { "blacklist.cs.html", "blacklist.en.html", "blacklist.sk.html" }));
        }
Ejemplo n.º 9
0
        private void HDDUpdateCustomLists()
        {
            if (CacheLiveStorage.CoreCache.CustomLists == null)
            {
                log.Debug("No custom list to update");
                return;
            }

            var customlist = CacheLiveStorage.CoreCache.CustomLists.ToArray();
            var count      = customlist.Count();

            log.Debug($"Updating {count} custom lists.");
            for (var i = 0; i < count; i++)
            {
                var cacheIdentity            = new byte[customlist[i].Identity.Length];
                var cachecustomlist_policyid = new byte[sizeof(Int32)];
                Array.Copy(ASCIIEncoding.ASCII.GetBytes(customlist[i].Identity), 0, cacheIdentity, 0, customlist[i].Identity.Length);

                var cachecustomlist_whitelist = new byte[(customlist[i].WhiteList.Count() * sizeof(UInt64))];
                var wlarray  = customlist[i].WhiteList.ToArray();
                var sortedWl = new List <ulong>(wlarray.Length);
                for (var j = 0; j < customlist[i].WhiteList.Count(); j++)
                {
                    var crc = Crc64.Compute(0, ASCIIEncoding.ASCII.GetBytes(wlarray[j]));
                    sortedWl.Add(crc);
                }

                sortedWl.Sort();
                for (var j = 0; j < wlarray.Length; j++)
                {
                    Array.Copy(BitConverter.GetBytes(sortedWl[j]), 0, cachecustomlist_whitelist, j * sizeof(UInt64), sizeof(UInt64));
                }

                var cachecustomlist_blacklist = new byte[(customlist[i].BlackList.Count() * sizeof(UInt64))];
                var blarray  = customlist[i].BlackList.ToArray();
                var sortedBl = new List <ulong>(blarray.Length);
                for (var j = 0; j < customlist[i].BlackList.Count(); j++)
                {
                    var crc = Crc64.Compute(0, ASCIIEncoding.ASCII.GetBytes(blarray[j]));
                }

                sortedBl.Sort();
                for (var j = 0; j < blarray.Length; j++)
                {
                    Array.Copy(BitConverter.GetBytes(sortedBl[j]), 0, cachecustomlist_blacklist, j * sizeof(UInt64), sizeof(UInt64));
                }

                Array.Copy(BitConverter.GetBytes(customlist[i].PolicyId), 0, cachecustomlist_policyid, 0, sizeof(Int32));

                listener.pushCustomListIdentityBuffer(null, new List <byte[]>()
                {
                    cacheIdentity
                });
                listener.pushCustomListWhitelistBuffer(null, new List <byte[]>()
                {
                    cachecustomlist_whitelist
                });
                listener.pushCustomListBlacklistBuffer(null, new List <byte[]>()
                {
                    cachecustomlist_blacklist
                });
                listener.pushCustomListPolicyIdBuffer(null, new List <byte[]>()
                {
                    cachecustomlist_policyid
                });
            }
        }