private void btnLoad_Click(object sender, EventArgs e)
{
Corpus bad = new Corpus();
Corpus good = new Corpus();
bad.LoadFromFile("../../TestData/spam.txt");
good.LoadFromFile("../../TestData/good.txt");
_filter = new SpamFilter();
_filter.Load(good, bad);
// Just for grins, we'll dump out some statistics about the data we just loaded.
txtOut.Text = String.Format(@"{0} {1} {2}{3}"
, _filter.Good.Tokens.Count
, _filter.Bad.Tokens.Count
, _filter.Prob.Count
, Environment.NewLine);
// ... and some probabilities for keys
foreach (string key in _filter.Prob.Keys)
{
if (_filter.Prob[key] > 0.02)
{
txtOut.AppendText(String.Format("{0},{1}{2}", _filter.Prob[key].ToString(".0000"), key, Environment.NewLine));
}
}
}
private void btnLoad_Click(object sender, EventArgs e)
{
Korpus spatny = new Korpus();
Korpus dobry = new Korpus();
spatny.NactiZeSouboru("../../TestData/spam.txt");
dobry.NactiZeSouboru("../../TestData/good.txt");
_filter = new SpamFilter();
_filter.Nacti(dobry, spatny);
// Vypis statistik o nactenych datech.
txtOut.Text = String.Format(@"{0} {1} {2}{3}"
, _filter.Dobre.Tokeny.Count
, _filter.Spatne.Tokeny.Count
, _filter.ListPravdepodobnosti.Count
, Environment.NewLine);
// Vypis pravdepodobnosti
foreach (string key in _filter.ListPravdepodobnosti.Keys)
{
if (_filter.ListPravdepodobnosti[key] > 0.02)
{
txtOut.AppendText(String.Format("{0},{1}{2}", _filter.ListPravdepodobnosti[key].ToString(".0000"), key, Environment.NewLine));
}
}
}
/// <summary>
/// Wird aufgerufen, wenn der Benutzer normalen Text (kein Kommando) eingegeben hat
/// </summary>
/// <param name="authorid"></param>
/// <param name="authorname"></param>
/// <param name="chatid"></param>
/// <param name="text"></param>
/// <returns></returns>
internal async Task processTextInput(long authorid, string authorname, long chatid, string text)
{
try
{
if (this.authors.isPostMode(authorid, authorname))
{
// == NEW POST SUBMITTED ==
if (!SpamFilter.checkPostInput(text, out string posttext, out string message))
{
// spamfilter hat zugeschlagen
await this.telegramInputBot.SendTextMessageAsync(
chatId : chatid,
text : "Abgelehnt, Text ändern und erneut senden. Meldung des Spamfilters: " + message
);
return;
}
this.statistics.increaseInteraction();
this.authors.resetCoolDown(authorid, authorname, Author.InteractionCooldownTimer.Posting);
this.authors.resetCoolDown(authorid, authorname, Author.InteractionCooldownTimer.Default);
this.authors.unsetModes(authorid, authorname);
this.posts.addPosting(posttext, authorid);
await this.telegramInputBot.SendTextMessageAsync(
chatId : chatid,
text : DRaumManager.ReplyPost + "\r\nMeldung des Spamfilters: " + message
);
return;
}
if (this.authors.isFeedbackMode(authorid, authorname))
{
// == Feedback ==
if (!SpamFilter.checkPostInput(text, out string feedbacktext, out string message))
{
// spamfilter hat zugeschlagen
await this.telegramInputBot.SendTextMessageAsync(
chatId : chatid,
text : "Abgelehnt, Text ändern und erneut senden. Meldung des Spamfilters: " + message
);
return;
}
this.statistics.increaseInteraction();
this.authors.resetCoolDown(authorid, authorname, Author.InteractionCooldownTimer.Feedback);
this.authors.resetCoolDown(authorid, authorname, Author.InteractionCooldownTimer.Default);
this.authors.unsetModes(authorid, authorname);
this.feedbackManager.enqueueFeedback(
new FeedbackElement("Von: @" + authorname + " ID(" + authorid + ") : " + feedbacktext, chatid));
await this.telegramInputBot.SendTextMessageAsync(
chatId : chatid,
text : DRaumManager.ReplyFeedback
);
}
}
catch (Exception ex)
{
logger.Error(ex, "Fehler beim Text-Verarbeiten");
}
}
/// <summary>
/// Initializes the natural language class
/// </summary>
/// <param name="spamfile">Spam words definition text file</param>
/// <param name="goodfile">Good words definition text file</param>
public NaturalLanguage(string spamfile, string goodfile)
{
Corpus bad = new Corpus();
Corpus good = new Corpus();
t = new Translator();
bad.LoadFromFile(spamfile);
good.LoadFromFile(goodfile);
filter = new SpamFilter();
filter.Load(good, bad);
}
public void spamFilterTest()
{
string input = "& < > \" und <tag>";
Assert.IsFalse(SpamFilter.checkPostInput(input, out string output, out string message));
for (int i = 0; i < 100; i++)
{
input += "X";
}
input = Utilities.telegramEntitiesToHtml(input, new MessageEntity[0]);
Assert.IsTrue(SpamFilter.checkPostInput(input, out output, out message));
Assert.IsTrue(output.StartsWith("& < > " und <tag>"));
}
public void TestSpamFilterDisabled()
{
SpamFilter sf = new SpamFilter(); //No SpamTime set => Show everything
DateTime time1 = new DateTime(2013, 1, 1, 0, 0, 0);
CardType cType = new CardType();
cType.id = 1;
Auction aBob1 = new Auction("bob", time1, Auction.OfferType.BUY, new Card(1, cType, true),"");
Auction aAlice1 = new Auction("alice", new DateTime(2013, 1, 1, 0, 0, 15), Auction.OfferType.BUY, new Card(1, cType, true),"");
DateTime time2 = new DateTime(2013, 1, 1, 0, 0, 30);
Auction aBob2 = new Auction("bob", time2, Auction.OfferType.BUY, new Card(1, cType, true),"");
sf.addAuction(aBob1);
sf.addAuction(aAlice1);
sf.addAuction(aBob2);
Assert.IsFalse(sf.isFilteredBySpamFilter(aBob1));
Assert.IsFalse(sf.isFilteredBySpamFilter(aAlice1));
Assert.IsFalse(sf.isFilteredBySpamFilter(aBob2));
}
public void TestSpamFilterBlock()
{
SpamFilter sf = new SpamFilter();
sf.setSpamTime(new TimeSpan(0, 1, 0));
DateTime time1 = new DateTime(2013, 1, 1, 0, 0, 0);
CardType cType = new CardType();
cType.id = 1;
Auction aBob1 = new Auction("bob", time1, Auction.OfferType.BUY, new Card(1, cType, true),"");
Auction aAlice1 = new Auction("alice", new DateTime(2013, 1, 1, 0, 0, 15), Auction.OfferType.BUY, new Card(1, cType, true),"");
DateTime time2 = new DateTime(2013, 1, 1, 0, 0, 30);
Auction aBob2 = new Auction("bob", time2, Auction.OfferType.BUY, new Card(1, cType, true),"");
sf.addAuction(aBob1);
sf.addAuction(aAlice1);
sf.addAuction(aBob2);
Assert.IsFalse(sf.isFilteredBySpamFilter(aBob1)); //Block the newer message and show the older one
Assert.IsFalse(sf.isFilteredBySpamFilter(aAlice1));
Assert.IsTrue(sf.isFilteredBySpamFilter(aBob2));
}
private void btnZeSouboru(object sender, EventArgs e)
{
_filter = new SpamFilter();
_filter.zeSouboru("../../TestData/out.txt");
}
private void btnFromFile_Click(object sender, EventArgs e)
{
_filter = new SpamFilter();
_filter.FromFile("../../TestData/out.txt");
}
/// <summary>
/// Entry
/// </summary>
/// <param name="args"></param>
static void Main(string[] args)
{
Console.WriteLine("Start SpamFireNetPipe");
var spamfilter = new SpamFilter();
var configfile = Path.Combine(Environment.CurrentDirectory, "spamfire.cfg");
//load cache
if (File.Exists(configfile) == true)
{
using (var fs = new FileStream(configfile, FileMode.Open, FileAccess.Read))
{
using (var sr = new StreamReader(fs, Encoding.UTF8))
{
spamfilter.ImportList(sr);
}
}
}
//Spam Check Server
//RBL
spamfilter.AddDNSBL("zen.spamhaus.org");
//Custom Link
spamfilter.AddCustomURLLink("hoge.hoge.xyz");
var server = new netcatserver(IPAddress.Any, 8888);
server.OnCreatePipeTerminal += (tcplient, disconnectcall) =>
{
return(new SpamPipeTerminal(tcplient, disconnectcall));
};
server.OnConnect += (pipe) =>
{
bool bRecv = false;
pipe.OnReceive += (targetpipe, receivebin, length) =>
{
if (length > 0)
{
bRecv = true;
}
else
if (length == 0)
{
if (bRecv == true)
{
//Finish receive
(pipe as SpamPipeTerminal).ReceiveStream.Position = 0;
var message = MimeKit.MimeMessage.Load((pipe as SpamPipeTerminal).ReceiveStream, false);
if (spamfilter.Fire(message) == true)
{
pipe.Send("1");
}
else
{
pipe.Send("0");
}
pipe.Dispose();
}
}
(pipe as SpamPipeTerminal).ReceiveStream.Write(receivebin, 0, length);
};
//Cache over 3 hour
spamfilter.CacheOver(10000000L * 60L * 60L * 3L);
};
AppDomain.CurrentDomain.ProcessExit += (exitobj, exitargs) =>
{
//save cache
using (var fs = new FileStream(configfile, FileMode.CreateNew, FileAccess.Write))
{
using (var sw = new StreamWriter(fs, Encoding.UTF8))
{
spamfilter.ExportList(sw);
}
}
Console.WriteLine("Finish SpamFireNetPipe");
};
server.Start();
while (true)
{
System.Threading.Thread.Sleep(1000);
}
}
