private static void SendBuffers(Stream stream, IEnumerable <byte[]> messages)
{
var response = new byte[1];
byte[] header = new byte[16];
foreach (var message in messages)
{
Array.Copy(BitConverter.GetBytes(message.LongLength), 0, header, 0, sizeof(UInt64)); //8 bytes
Array.Copy(BitConverter.GetBytes(Crc64.Compute(0, message)), 0, header, 8, sizeof(UInt64)); //8 bytes
stream.Write(header, 0, header.Length);
//log.Debug($"Written {header.Length} header size, message {message.LongLength} bytes");
//log.Debug($"Write message.");
stream.Write(message, 0, message.Length);
//log.Debug($"Message written.");
//var bytesRead = stream.Read(response, 0, 1);
////log.Debug($"Read response");
//if (bytesRead == 1 && response[0] == '1')
//{
// //log.Debug($"Message was sent successfully.");
//}
//else
//{
// throw new Exception("unable to write to nework stream");
//}
}
}
public static void TestCoreCache(string path, string domain)
{
log.Info("GetCoreCacheManual()");
using (var stream = new System.IO.FileStream(path, System.IO.FileMode.Open))
{
log.Debug($"Deserialize.");
var cache = ProtoBuf.Serializer.Deserialize <Models.Cache>(stream);
log.Debug($"Deserialized.");
if (cache.CustomLists != null)
{
log.Debug($"Custom List count = {cache.CustomLists.ToArray().Count()}");
}
if (cache.Domains != null)
{
log.Debug($"Domains count = {cache.Domains.ToArray().Count()}");
}
if (cache.IPRanges != null)
{
log.Debug($"IPRanges count = {cache.IPRanges.ToArray().Count()}");
}
if (cache.Policies != null)
{
log.Debug($"Policies count = {cache.Policies.ToArray().Count()}");
}
var crc64 = Crc64.Compute(0, ASCIIEncoding.ASCII.GetBytes(domain));
if (cache.Domains.Any(t => t.Crc64 == crc64))
{
log.Debug("contains " + domain);
}
}
}
private static List <Models.CacheDomain> LoadDomains()
{
var result = new List <Models.CacheDomain>();
var csv = new CsvHelper.CsvReader(File.OpenText("domains.csv"));
while (csv.Read())
{
var b0 = Convert.ToInt32(csv.GetField(2));
var b1 = Convert.ToInt32(csv.GetField(3));
var b2 = Convert.ToInt32(csv.GetField(4));
var b3 = Convert.ToInt32(csv.GetField(5));
var b4 = Convert.ToInt32(csv.GetField(6));
var b5 = Convert.ToInt32(csv.GetField(7));
var b6 = Convert.ToInt32(csv.GetField(8));
var b7 = Convert.ToInt32(csv.GetField(9));
var b8 = Convert.ToInt32(csv.GetField(10));
var b9 = Convert.ToInt32(csv.GetField(11));
var b10 = Convert.ToInt32(csv.GetField(12));
var b11 = Convert.ToInt32(csv.GetField(13));
var b12 = Convert.ToInt32(csv.GetField(14));
var b13 = Convert.ToInt32(csv.GetField(15));
var b14 = Convert.ToInt32(csv.GetField(16));
var b15 = Convert.ToInt32(csv.GetField(17));
var item = new Models.CacheDomain
{
Crc64 = Crc64.Compute(0, ASCIIEncoding.ASCII.GetBytes(csv.GetField(0))),
Accuracy = Convert.ToInt32(csv.GetField(1)),
Flags = new [] { b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, b10, b11, b12, b13, b14 }
};
result.Add(item);
}
return(result);
}
private static int TaskTCPJob(object args)
{
TaskTcpArgs arg = (TaskTcpArgs)args;
int port = arg.port;
bufferType buftype = arg.buftype;
IEnumerable <byte[]> data = arg.data;
try
{
using (var client = new TcpClient())
{
client.Client.Connect(IPAddress.Parse(Configuration.GetKres()), port);
var messageType = (int)buftype;
//+ 8 bytes (= last crc64 bytes)
var header = BitConverter.GetBytes(messageType) // 4 bytes -> message type
.Concat(BitConverter.GetBytes(data.Count())); // 4 bytes -> how many buffers
var headerCrc = Crc64.Compute(0, header.ToArray());
header = header.Concat(BitConverter.GetBytes(headerCrc)); // 8 byte -> crc of this header
//log.Debug($"Get stream");
using (var stream = client.GetStream())
{
if (SendHeader(stream, header))
{
SendBuffers(stream, data);
}
//log.Debug($"Closing ip stream.");
stream.Flush();
stream.Close();
}
client.Close();
log.Debug($"Kres {port} job {buftype} updated");
}
}
catch (Exception ex)
{
log.Error($"Unable to connect to kres on port {port}, ex: {ex.Message}");
}
return(0);
}
private static int TaskHDDJob(object args)
{
TaskHddArgs arg = (TaskHddArgs)args;
string filename = arg.filename;
bufferType buftype = arg.buftype;
IEnumerable <byte[]> data = arg.data;
try
{
using (var stream = new FileStream(filename, FileMode.Append, FileAccess.Write, FileShare.None))
{
var messageType = (int)buftype;
//+ 8 bytes (= last crc64 bytes)
var header = BitConverter.GetBytes(messageType) // 4 bytes -> message type
.Concat(BitConverter.GetBytes(data.Count())); // 4 bytes -> how many buffers
var headerCrc = Crc64.Compute(0, header.ToArray());
header = header.Concat(BitConverter.GetBytes(headerCrc)); // 8 byte -> crc of this header
//log.Debug($"Get stream");
//using (var stream = client.GetStream())
{
if (SendHeader(stream, header))
{
SendBuffers(stream, data);
}
//log.Debug($"Closing ip stream.");
stream.Flush();
stream.Close();
}
//client.Close();
log.Debug($"File {filename} was updated.");
}
}
catch (Exception ex)
{
log.Error($"Unable to write to {filename}. Ex: {ex}");
}
return(0);
}
private static byte[] ReadMessage(FileStream stream)
{
byte[] buffer = new byte[16];
if (stream.Read(buffer, 0, buffer.Length) != buffer.Length)
{
throw new Exception("Unable to read messageheader");
}
var length = BitConverter.ToInt64(buffer, 0);
var crc64 = BitConverter.ToInt64(buffer, 8);
buffer = new byte[length];
if (stream.Read(buffer, 0, (int)length) != length)
{
throw new Exception("Unable to read message");
}
var computedCrc = Crc64.Compute(0, buffer);
return(buffer);
}
public string Bypass(HttpContext ctx, string clientIpAddress, string domainToWhitelist, string authToken, string base64encodedUrlToRedirectTo)
{
log.Info($"Bypass request, ip={clientIpAddress}, {domainToWhitelist}.");
if (string.Compare(authToken, "BFLMPSVZ", StringComparison.OrdinalIgnoreCase) != 0)
{
return("");
}
var idbytes = Encoding.ASCII.GetBytes(clientIpAddress);
var idcrc = Crc64.Compute(0, idbytes);
string identity = idcrc.ToString("X");
IPAddress ip;
if (!IPAddress.TryParse(clientIpAddress, out ip))
{
return(new Exception($"unable to parse ip address {clientIpAddress}.").Message);
}
//TODO: check ipv6 reverse
var bytes = ip.GetAddressBytes().Reverse().ToArray();
BigMath.Int128 intip;
if (bytes.Length == 4)
{
intip = new BigMath.Int128(0, BitConverter.ToUInt32(bytes, 0));
}
else if (bytes.Length == 16)
{
intip = new BigMath.Int128(BitConverter.ToUInt64(bytes, 0), BitConverter.ToUInt64(bytes, 8));
}
else
{
return(new Exception($"unable to parse ip address {clientIpAddress}.").Message);
}
List <Models.CacheIPRange> ipranges;
List <Models.CacheCustomList> customlists;
if (CacheLiveStorage.CoreCache.IPRanges != null)
{
ipranges = CacheLiveStorage.CoreCache.IPRanges.ToList();
}
else
{
ipranges = new List <Models.CacheIPRange>();
}
if (CacheLiveStorage.CoreCache.IPRanges != null)
{
customlists = CacheLiveStorage.CoreCache.CustomLists.ToList();
}
else
{
customlists = new List <Models.CacheCustomList>();
}
ipranges.Add(new Models.CacheIPRange()
{
Identity = identity,
Proto_IpFrom = Encoding.ASCII.GetBytes(intip.ToString()),
Proto_IpTo = Encoding.ASCII.GetBytes(intip.ToString()),
PolicyId = 0
});
var item = customlists.FirstOrDefault(t => string.Compare(t.Identity, identity, StringComparison.OrdinalIgnoreCase) == 0);
if (item == null)
{
item = new Models.CacheCustomList()
{
Identity = identity,
WhiteList = new List <string>()
{
domainToWhitelist
},
BlackList = new List <string>(),
PolicyId = 0
};
log.Info($"Identity {identity} now has {domainToWhitelist} whitelisted.");
}
else
{
if (!item.WhiteList.Contains(domainToWhitelist))
{
var list = item.WhiteList.ToList();
list.Add(domainToWhitelist);
item.WhiteList = list;
foreach (var entry in item.WhiteList)
{
log.Info($"Identity {identity} now has {entry} whitelisted.");
}
}
else
{
log.Info($"Identity {identity} has {domainToWhitelist} already whitelisted.");
}
}
customlists.RemoveAll(t => string.Compare(t.Identity, identity, StringComparison.OrdinalIgnoreCase) == 0);
customlists.Add(item);
CacheLiveStorage.CoreCache.IPRanges = ipranges;
CacheLiveStorage.CoreCache.CustomLists = customlists;
log.Info($"Updating kres modules.");
KresUpdater.UpdateSmallCaches();
KresUpdater.UpdateNow();
log.Info($"Kres modules have been updated.");
//var redirectUrl = Base64Decode(base64encodedUrlToRedirectTo);
//ctx.Response.RedirectLocation = redirectUrl;
return(null);
}
public string PassThrough(HttpContext ctx, string postdata)
{
try
{
var list = ctx.Request.Host.ToString().Split('.');
if (!string.IsNullOrEmpty(postdata))
{
list = postdata.Split('.');
}
for (var i = 0; i < list.Length - 1; i++)
{
var joined = string.Join('.', list, i, list.Length - i);
var bytes = Encoding.ASCII.GetBytes(joined);
var crc = Crc64.Compute(0, bytes);
if (CacheLiveStorage.CoreCache.Domains == null)
{
break;
}
var domain = CacheLiveStorage.CoreCache.Domains.Where(t => t.Crc64 == crc).FirstOrDefault();
if (domain == null)
{
continue;
}
var ipaddress = ctx.Connection.RemoteIpAddress;
var addrbytes = ipaddress.GetAddressBytes().Reverse().ToArray();
var addr = new Models.Int128();
if (addrbytes.Length == 4)
{
addr.Hi = 0;
addr.Low = BitConverter.ToUInt32(addrbytes, 0);
}
else if (addrbytes.Length == 16)
{
addr.Hi = BitConverter.ToUInt64(addrbytes, 0);
addr.Low = BitConverter.ToUInt64(addrbytes, 8);
}
var ip = new BigMath.Int128(addr.Hi, addr.Low);
var ipRange = new List <Models.CacheIPRange>();
if (CacheLiveStorage.UdpCache != null)
{
ipRange = CacheLiveStorage.UdpCache.Select(t => t.Value).ToList();
Models.CacheIPRange[] ipRangeCore = null;
if (CacheLiveStorage.CoreCache.IPRanges != null)
{
ipRangeCore = CacheLiveStorage.CoreCache.IPRanges.ToArray();
ipRange = ipRange.Concat(ipRangeCore).ToList();
}
}
var range = ipRange.Where(t => t.BintFrom >= ip && t.BintTo <= ip).FirstOrDefault();
var range_identity = string.Empty;
var range_policyid = 0;
if (range != null)
{
range_identity = range.Identity;
range_policyid = range.PolicyId;
}
Models.PublicListenerConfig views;
using (var sr = new StreamReader("publiclistenerconfig.json"))
{
views = JsonConvert.DeserializeObject <Models.PublicListenerConfig>(sr.ReadToEnd());
}
foreach (var network in views.views)
{
foreach (var cidr in network.networks)
{
IPNetwork n = IPNetwork.Parse(cidr.ToString());
if (!n.Contains(ipaddress))
{
continue;
}
if (!string.IsNullOrEmpty(range_identity))
{
if (CacheLiveStorage.CoreCache.CustomLists != null)
{
var custom = CacheLiveStorage.CoreCache.CustomLists.Where(t => string.Compare(t.Identity, range_identity, StringComparison.OrdinalIgnoreCase) == 0).FirstOrDefault();
if (custom != null)
{
var dmn = string.Join('.', list);
if (custom.WhiteList.Contains(joined) || custom.WhiteList.Contains(dmn))
{
//allow
//return "allow";
return(GenerateContent(ctx, network.accuracy));
}
else if (custom.BlackList.Contains(joined) || custom.BlackList.Contains(dmn))
{
return(GenerateContent(ctx, network.blacklist));
}
}
}
}
if (CacheLiveStorage.CoreCache.Policies != null)
{
var policy = CacheLiveStorage.CoreCache.Policies.Where(t => t.Policy_id == range_policyid).FirstOrDefault();
if (policy == null)
{
//no policy
//return "no policy";
return(GenerateContent(ctx, network.blacklist));
}
else
{
var flags = domain.Flags.ToArray()[policy.Policy_id];
if ((flags & (int)KresFlags.flags_accuracy) == (int)KresFlags.flags_accuracy)
{
if (policy.Block > 0 && domain.Accuracy > policy.Block)
{
return(GenerateContent(ctx, network.accuracy));
}
else
{
if (policy.Audit > 0 && domain.Accuracy > policy.Audit)
{
//audit
//return "audit";
return(GenerateContent(ctx, network.blacklist));
}
else
{
//no accuracy action
//return "no accuracy action";
return(GenerateContent(ctx, network.blacklist));
}
}
}
if ((flags & (int)KresFlags.flags_whitelist) == (int)KresFlags.flags_whitelist)
{
//allow whitelist
//return "allow whitelist";
return(GenerateContent(ctx, network.blacklist));
}
if ((flags & (int)KresFlags.flags_blacklist) == (int)KresFlags.flags_blacklist)
{
//block
return(GenerateContent(ctx, network.legal));
}
}
}
}
}
}
}
catch (Exception ex)
{
log.Error("ERROR: ", ex);
}
//return "no-action";
return(GenerateContent(ctx, new[] { "blacklist.cs.html", "blacklist.en.html", "blacklist.sk.html" }));
}
private void HDDUpdateCustomLists()
{
if (CacheLiveStorage.CoreCache.CustomLists == null)
{
log.Debug("No custom list to update");
return;
}
var customlist = CacheLiveStorage.CoreCache.CustomLists.ToArray();
var count = customlist.Count();
log.Debug($"Updating {count} custom lists.");
for (var i = 0; i < count; i++)
{
var cacheIdentity = new byte[customlist[i].Identity.Length];
var cachecustomlist_policyid = new byte[sizeof(Int32)];
Array.Copy(ASCIIEncoding.ASCII.GetBytes(customlist[i].Identity), 0, cacheIdentity, 0, customlist[i].Identity.Length);
var cachecustomlist_whitelist = new byte[(customlist[i].WhiteList.Count() * sizeof(UInt64))];
var wlarray = customlist[i].WhiteList.ToArray();
var sortedWl = new List <ulong>(wlarray.Length);
for (var j = 0; j < customlist[i].WhiteList.Count(); j++)
{
var crc = Crc64.Compute(0, ASCIIEncoding.ASCII.GetBytes(wlarray[j]));
sortedWl.Add(crc);
}
sortedWl.Sort();
for (var j = 0; j < wlarray.Length; j++)
{
Array.Copy(BitConverter.GetBytes(sortedWl[j]), 0, cachecustomlist_whitelist, j * sizeof(UInt64), sizeof(UInt64));
}
var cachecustomlist_blacklist = new byte[(customlist[i].BlackList.Count() * sizeof(UInt64))];
var blarray = customlist[i].BlackList.ToArray();
var sortedBl = new List <ulong>(blarray.Length);
for (var j = 0; j < customlist[i].BlackList.Count(); j++)
{
var crc = Crc64.Compute(0, ASCIIEncoding.ASCII.GetBytes(blarray[j]));
}
sortedBl.Sort();
for (var j = 0; j < blarray.Length; j++)
{
Array.Copy(BitConverter.GetBytes(sortedBl[j]), 0, cachecustomlist_blacklist, j * sizeof(UInt64), sizeof(UInt64));
}
Array.Copy(BitConverter.GetBytes(customlist[i].PolicyId), 0, cachecustomlist_policyid, 0, sizeof(Int32));
listener.pushCustomListIdentityBuffer(null, new List <byte[]>()
{
cacheIdentity
});
listener.pushCustomListWhitelistBuffer(null, new List <byte[]>()
{
cachecustomlist_whitelist
});
listener.pushCustomListBlacklistBuffer(null, new List <byte[]>()
{
cachecustomlist_blacklist
});
listener.pushCustomListPolicyIdBuffer(null, new List <byte[]>()
{
cachecustomlist_policyid
});
}
}
