public override void Run()
{
var cowriePath = @"C:\Users\thelu\OneDrive\Skrivebord\honeypotlogs 26_04\cowrie";
InFolder = Directory.CreateDirectory(cowriePath);
OutExtension = "json";
var parent = Directory.GetParent(Directory.GetCurrentDirectory()).CreateSubdirectory("CowrieA");
var f = parent.CreateSubdirectory($"{ReplaceInvalidChars(ScriptName)}.{DateTime.Now.ToString(ScriptDateTimeFormat)}");
Trace.WriteLine("Working path: " + f.FullName);
OutFolder = f;
var filepath = @"C:\Users\thelu\OneDrive\Skrivebord\honeypotlogs 26_04\all\cowrie.tar";
//https://stackoverflow.com/questions/42625845/asp-net-read-a-file-from-a-tar-gz-archive
/*using (Stream source = new GZipInputStream(new FileStream(filepath, FileMode.Open))) //wc.OpenRead() create one stream with archive tar.gz from our server
* {*/
using (TarInputStream tarStr = new TarInputStream(new FileStream(filepath, FileMode.Open))) //TarInputStream is a stream from ICSharpCode.SharpZipLib.Tar library(need install SharpZipLib in nutgets)
{
TarEntry te;
try
{
while ((te = tarStr.GetNextEntry()) != null) // Go through all files from archive
{
if (te.Name.Contains("downloads"))
{
if (te.Size > 1000)
{
if (!te.Name.Contains("tmp"))
{
var sha = te.Name.Replace("cowrie/downloads/", "");
if (dls.Any(d => d.Value.size == te.Size))
{
var oldsha = dls.Where(d => d.Value.size == te.Size).FirstOrDefault();
oldsha.Value.count += 1;
oldsha.Value.ips.Add(new IpDate()
{
sha2 = sha
});
dls.Remove(oldsha.Key);
dls.Add(oldsha.Key, oldsha.Value);
continue;
}
dls.Add(sha, new DownloadFile()
{
sha2 = sha, size = te.Size, count = 1, ips = new List <IpDate>()
{
new IpDate()
{
sha2 = sha
}
}
});
}
}
}
}
}
catch (TarException tare)
{
Console.WriteLine(tare.StackTrace);
string text = File.ReadAllText(filepath);
Console.WriteLine(text);
}
}
//}
int limit = 1000;
dls.Count();
var s = dls.ToArray().Split(limit);
int lc = 0;
foreach (var item in s)
{
using (var writer = new StreamWriter(OutFile.Substring(0, OutFile.Length - 5) + "." + (lc++) + ".json"))
{
foreach (var item1 in item)
{
writer.WriteLine(JsonConvert.SerializeObject(item1.Value));
}
}
}
}
public override void Run()
{
var limit = 10;//Per min
var limitCount = 0;
var limitOn = true;
//var md = Fetch("").Result;
using (var writer = new StreamWriter(Path.Combine(OutFolder.FullName, "MalwareData.json")))
{
writer.AutoFlush = true;
foreach (var item in storedDls)
{
var key = item.Key;
var newItem = item;
if (item.Value.malwareDTO != null)
{
dls.Add(key, newItem.Value);
writer.WriteLine(JsonConvert.SerializeObject(item.Value, Formatting.None));
continue;
}
if (limitCount > limit && limitOn)
{
limitCount = 0; Thread.Sleep(1000 * 60);
}
var res = Fetch(key).Result;
var md = res.Item1;
Trace.Write(key);
if (res.Item2 == HttpStatusCode.NotFound)
{
Trace.WriteLine(" -> Not found");
}
else if (md.data == null)
{
limit = 4;
limitCount = limit;
Trace.WriteLine(" -> Error " + res.Item2.ToString() + " If 429, wait until the next hour starts");
var nt = (60 - DateTime.Now.Minute) * 60 * 1000;
Trace.WriteLine("Sleeping for " + (60 - DateTime.Now.Minute) + " min");
Thread.Sleep(nt);
continue;
}
else
{
Trace.WriteLine(" -> Content");
limitCount++;
var t = md.engineResults(md.data.attributes.last_analysis_results);
}
newItem.Value.malwareDTO = md;
foreach (var i in dls)
{
if (i.Value.malwareDTO.data.attributes.size == newItem.Value.malwareDTO.data.attributes.size)
{
i.Value.ips.AddRange(newItem.Value.ips);
}
}
dls.Add(key, newItem.Value);
writer.WriteLine(JsonConvert.SerializeObject(item.Value, Formatting.None));
}
}
var cowriePath = Directory.CreateDirectory(@"C:\Users\thelu\OneDrive\Skrivebord\honeypotlogs 26_04\cowrie");
ReadFilesWriteToFile(cowriePath.FullName, "cowrie.json.*", OutFile.Substring(0, OutFile.Length - 5) + ".json");
}
public BlockingCollection <string> ReadFilesWriteToFile(string folder, string fileNameWC, string toFile)
{
var matchesCollection = new BlockingCollection <string>();
var files = Directory.GetFiles(folder, fileNameWC,
SearchOption.TopDirectoryOnly);
Console.WriteLine("Reading Folder: " + folder);
var exceptions = new ValueTuple <string, Regex>[]
{
("empty-string", new Regex(""))
};
var readTask = Task.Run(() =>
{
//List<string> cmds = new List<string>();
Dictionary <string, EventDTO> events = new Dictionary <string, EventDTO>();
//Dictionary<string, IpUsernamePassword> ipup = new Dictionary<string, IpUsernamePassword>();
Dictionary <string, long> success = new Dictionary <string, long>();
Dictionary <string, long> failed = new Dictionary <string, long>();
try
{
foreach (var file in files)
{
string line2 = "";
try
{
using (var reader = new StreamReader(file))
{
string line;
while ((line = reader.ReadLine()) != null)
{
line2 = line;
EventDTO _event = null;
try
{
_event = JsonConvert.DeserializeObject <EventDTO>(line);
}
catch (Exception)
{
continue;
}
if (_event.eventid == "cowrie.login.success")
{
var key = _event.username.ToString() + "," + _event.password.ToString();
if (success.ContainsKey(key))
{
success.TryGetValue(key, out long l);
success.Remove(key);
success.Add(key, l + 1);
}
else
{
success.Add(key, 1);
}
}
if (_event.eventid == "cowrie.login.failed")
{
var key = _event.username.ToString() + "," + _event.password.ToString();
if (failed.ContainsKey(key))
{
failed.TryGetValue(key, out long l);
failed.Remove(key);
failed.Add(key, l + 1);
}
else
{
failed.Add(key, 1);
}
}
}
}
}
catch (DirectoryNotFoundException e)
{
Console.WriteLine(e.StackTrace);
}
catch (Exception e)
{
Console.WriteLine(line2);
Console.WriteLine(e.StackTrace);
}
/*
* foreach (var item in events)
* {
* if (ipup.ContainsKey(item.Value.src_ip)) {
* ipup.TryGetValue(item.Value.src_ip, out IpUsernamePassword value);
* if (item.Value.eventid.Equals("cowrie.login.failed")) {
* if (value.Failed == null)
* {
* value.Failed = new HashSet<UnamePass>() { new UnamePass() { uname = item.Value.username, pass = item.Value.password, count = 1 } };
* }
* else {
* value.Failed.
* }
* }
* } else {
* AddNewToipup(ipup, item, item.Value.eventid.Equals("cowrie.login.failed"));
* }
* }
*/
}
}
finally
{
matchesCollection.CompleteAdding();
using (var writerSuccess = new StreamWriter(OutFile.Substring(0, OutFile.Length - 5) + ".Success" + ".csv"))
{
writerSuccess.WriteLine("username,password,count");
foreach (var cmd in success)
{
writerSuccess.WriteLine(cmd.Key + "," + cmd.Value);
}
}
using (var writerFailed = new StreamWriter(OutFile.Substring(0, OutFile.Length - 5) + ".Failed" + ".csv"))
{
writerFailed.WriteLine("username,password,count");
foreach (var cmd in failed)
{
writerFailed.WriteLine(cmd.Key + "," + cmd.Value);
}
}
}
});
Task.WaitAll(readTask);
return(matchesCollection);
}