private static void setConfigs()
{
var consoleOutput = ShellHelper.Bash("security find-generic-password -s gel");
if (consoleOutput.Contains("acct"))
{
var start = consoleOutput.IndexOf("\"acct\"<blob>=\"");
start = start + 14;
var end = consoleOutput.IndexOf("\"", start);
var email = consoleOutput.Substring(start, (end - start));
var key = ShellHelper.Bash("security find-generic-password -s gel -w");
start = consoleOutput.IndexOf("\"icmt\"<blob>=\"");
start = start + 14;
end = consoleOutput.IndexOf("\"", start);
var url = consoleOutput.Substring(start, (end - start));
Program.email = email.Trim();
Program.key = key.Trim();
Program.url = url.Trim();
}
else
{
throw new Exception("please set your credentials");
}
}
public ActionResult Clear()
{
string command = Constants.PYTHON_BASE + "stop.py";
string result = ShellHelper.Bash(command);
return(Json(new { Message = result }));
}
public async Task <IActionResult> RunWraithTestAsync([FromForm] string path)
{
if (_chromeP.Length > 1 || _pahntomjsP.Length > 1)
{
Console.WriteLine("Test is running please wait!");
return(BadRequest());
}
else
{
try
{
Console.WriteLine("The test path is " + path);
var execCommand = ("cd " + path + "; pwd; wraith capture configs/capture.yaml");
Console.WriteLine(execCommand + "\n");
ShellHelper.Bash(execCommand);
await Task.Run(() => {
while (_chromeP.Length != 0 || _pahntomjsP.Length != 0)
{
}
});
return(Json(new{ status = "done" }));
}
catch (Exception e)
{
Console.WriteLine(e);
return(Json(e.Message));
}
}
}
//Thread function that checks split info
public void checkSplit(string workingDirectory, string sourceFile, Models.Task task)
{
string sourceSize, folderSize;
int sourceSizeGB, folderSizeGB;
while (!task.splitDone)
{
try
{
sourceSize = ShellHelper.Bash("du -h -k " + sourceFile).Split('\t')[0];
sourceSizeGB = Int32.Parse(sourceSize) / 1024;
folderSize = ShellHelper.Bash("du -sh -k " + workingDirectory + "Run/").Split('\t')[0];
folderSizeGB = Int32.Parse(folderSize) / 1024;
task.s_status = "Splitting file into pieces. Current size: " + folderSizeGB + "KB of " + sourceSizeGB + "KB";
task.s_current = folderSizeGB;
task.s_total = sourceSizeGB;
Thread.Sleep(2000);
}
catch (Exception)
{
Thread.Sleep(2000);
}
}
}
public async Task <byte[]> ShrinkAsync(IShrinkable input)
{
var startTime = DateTime.Now;
var originalFilePath = Path.GetTempFileName();
using (var stream = new FileStream(originalFilePath, FileMode.Create))
{
await input.Content.CopyToAsync(stream);
}
var processedFilePath = Path.GetTempFileName();
string cmd = $"gs -sDEVICE=pdfwrite -dCompatibilityLevel=1.4 -dQUIET -q -dNOPAUSE -dBATCH -dSAFER -dPDFSETTINGS=/screen -dEmbedAllFonts=true -dSubsetFonts=true -dAutoRotatePages=/None -dColorImageDownsampleType=/Bicubic -dColorImageResolution={input.Resolution} -dGrayImageDownsampleType=/Bicubic -dGrayImageResolution={input.Resolution} -dMonoImageDownsampleType=/Bicubic -dMonoImageResolution={input.Resolution} -sOutputFile={processedFilePath} {originalFilePath}";
var output = ShellHelper.Bash(cmd);
var processedFileBytes = File.ReadAllBytes(processedFilePath);
File.Delete(originalFilePath);
File.Delete(processedFilePath);
var elapsedTime = DateTime.Now.Subtract(startTime);
_logger.LogInformation($"File ${input.Content.FileName} shrinked in {elapsedTime.Seconds} seconds.");
return(processedFileBytes);
}
public void OnPreprocessBuild(BuildReport report)
{
var commit = ShellHelper.Bash("git rev-parse HEAD").TrimEnd('\r', '\n');
File.WriteAllText(path, commit);
AssetDatabase.ImportAsset(path, ImportAssetOptions.ForceSynchronousImport);
}
public ActionResult LightPaint()
{
string command = Constants.PYTHON_BASE + "lightpaint.py";
string result = ShellHelper.Bash(command);
return(Json(new { Message = result }));
}
public static bool DeleteDogfoodSignalRService(string extensionScriptsDir, string resourceGroup, string serviceName, bool deleteResourceGroup = true)
{
var errCode = 0;
var result = "";
bool rtn = false;
var cmd = $"cd {extensionScriptsDir}; . ./az_signalr_service.sh; az_login_ASRS_dogfood";
(errCode, result) = ShellHelper.Bash(cmd, handleRes: true);
if (errCode != 0)
{
Util.Log($"Fail to login to dogfood Azure");
return(false);
}
cmd = $"cd {extensionScriptsDir}; . ./az_signalr_service.sh; delete_signalr_service {serviceName} {resourceGroup}";
(errCode, result) = ShellHelper.Bash(cmd, handleRes: true);
if (errCode != 0)
{
Util.Log($"Fail to delete SignalR Service");
rtn = false;
}
else
{
rtn = true;
}
if (deleteResourceGroup)
{
cmd = $"cd {extensionScriptsDir}; . ./az_signalr_service.sh; delete_group {resourceGroup}";
(errCode, result) = ShellHelper.Bash(cmd, handleRes: true);
rtn = true;
}
return(rtn);
}
public static string GetRunTime()
{
string runTime = string.Empty;
try
{
if (IsUnix())
{
string output = ShellHelper.Bash("uptime -s");
output = output.Trim();
runTime = DateTimeHelper.FormatTime(long.Parse((DateTime.Now - ParseToDateTime(output)).TotalMilliseconds.ToString().Split('.')[0]));
}
else
{
string output = ShellHelper.Cmd("wmic", "OS get LastBootUpTime/Value");
string[] outputArr = output.Split("=", StringSplitOptions.RemoveEmptyEntries);
if (outputArr.Length == 2)
{
runTime = DateTimeHelper.FormatTime(long.Parse((DateTime.Now - ParseToDateTime(outputArr[1].Split('.')[0])).TotalMilliseconds.ToString().Split('.')[0]));
}
}
}
catch (Exception) { }
return(runTime);
}
private CpuInfo GetCpu()
{
var output = ShellHelper.Bash("top -bn1 | grep \"Cpu(s)\" | sed \"s/.*, *\\([0-9.]*\\)%* id.*/\\1/\" | awk '{print 100 - $1}'");
var lines = ShellHelper.TryReadFileLines("/proc/cpuinfo");
var coreCountRegex = new Regex(@"^cpu cores\s+:\s+(.+)", RegexOptions.IgnoreCase);
var cpuCoresString = (lines.FirstOrDefault(o => coreCountRegex.Match(o).Success) ?? string.Empty);
var clockSpeedRegex = new Regex(@"^cpu MHz\s+:\s+(.+)", RegexOptions.IgnoreCase);
var clockSpeedString = (lines.FirstOrDefault(o => clockSpeedRegex.Match(o).Success) ?? string.Empty);
var index = clockSpeedString.IndexOf('.');
if (index != -1)
{
clockSpeedString = clockSpeedString.Substring(0, index);
}
var cpu = new CpuInfo();
if (double.TryParse(output.Replace("%", string.Empty), out var totalPercentage))
{
cpu.TotalPercentage = totalPercentage;
}
if (int.TryParse(coreCountRegex.Match(cpuCoresString).Groups[1].Value, out var cpuCores))
{
cpu.NumberOfCores = cpuCores;
}
if (int.TryParse(clockSpeedRegex.Match(clockSpeedString).Groups[1].Value, out var clockSpeed))
{
cpu.CurrentClockSpeed = clockSpeed;
}
return(cpu);
}
static List <string> ScanAarFile(string aarPath)
{
List <string> results = new List <string>();
//1. Scan the aar to find any contained jar files
string scanAar = string.Format("jar tf {0}", aarPath);
string[] scanAarOutput = ShellHelper.Bash(scanAar).Split('\n');
//2. Foreach jar inside the aar
foreach (var entry in scanAarOutput)
{
if (entry.Trim().EndsWith("jar"))
{
string outputPath = Path.Combine(Path.GetTempPath(), "check_jar.jar");
//2.1 unzip this jar file to the tmp folder
string unzip = string.Format("unzip -p {0} {1} > {2}", aarPath, entry.Trim(), outputPath);
ShellHelper.Bash(unzip);
//2.2 scan the extracted jar
results.AddRange(ScanJarFile(outputPath));
File.Delete(outputPath);
}
}
return(results);
}
public string TakeImage()
{
string outputFile = GetOutputFilename();
string command = "streamer -s " + outputResolution + " -f " + outputFormat + " -o " + outputFile;
ShellHelper.Bash(command);
return(outputFile);
}
public Snapshot GetRepositorySnapshot(string svnUrl, string repoName, string snapshotId, string requestId)
{
Directory.CreateDirectory($"../Repos/{requestId}");
repoName = repoName.Replace(' ', '_');
string repoPath = $"../Repos/{requestId}/{repoName}";
string revisionPath = $"../Repos/{requestId}/revision";
try
{
string dumpPath = $"../Repos/{requestId}/{repoName}.svndump";
// Dump external SVN repo
ShellHelper.Bash("svnrdump.exe", $"dump {svnUrl} -F {dumpPath}");
// Create new empty local SVN repo
ShellHelper.Bash("svnadmin.exe", $"create {repoPath}");
// Load external dump to local repo
ShellHelper.Bash("svnadmin.exe", $"load {repoPath} -F {dumpPath}");
string md5 = "";
byte[] repoBytes = new byte[0];
if (snapshotId == "none")
{
md5 = CreateRepositoryChecksum(dumpPath);
repoBytes = compressionService.ZipBytes(repoPath, repoName, $"../Repos/{requestId}");
}
else
{
// Checkout SVN revision
ShellHelper.Bash("svn.exe", $"checkout {svnUrl} {revisionPath}");
DirectoryHelper.SetAttributesNormal(new DirectoryInfo(revisionPath));
Directory.Delete($"{revisionPath}/.svn", true);
md5 = CreateSnapshotChecksum(repoPath, int.Parse(snapshotId));
repoBytes = compressionService.ZipBytes(revisionPath, repoName, $"../Repos/{requestId}");
}
File.Delete(dumpPath);
DeleteRequestDirectory(requestId);
return(new Snapshot()
{
checksum = md5,
zippedBytes = repoBytes
});
}
catch (Exception e)
{
DeleteRequestDirectory(requestId);
throw e;
}
}
public static string CreateDogfoodSignalRService(string extensionScriptsDir, string location, string resourceGroup, string serviceName, string sku, int unit)
{
var errCode = 0;
var result = "";
// Dogfood Azure login
var cmd = $"cd {extensionScriptsDir}; . ./az_signalr_service.sh; az_login_ASRS_dogfood";
Util.Log(cmd);
(errCode, result) = ShellHelper.Bash(cmd, handleRes: true);
if (errCode != 0)
{
Util.Log($"Fail to login to dogfood Azure");
return(null);
}
// Create resource group if it does not exist
cmd = $"cd {extensionScriptsDir}; . ./az_signalr_service.sh; create_group_if_not_exist {resourceGroup} {location}";
Util.Log(cmd);
(errCode, result) = ShellHelper.Bash(cmd, handleRes: true);
if (errCode != 0)
{
Util.Log($"Fail to create resource group");
return(null);
}
// Create SignalR service
cmd = $"cd {extensionScriptsDir}; . ./az_signalr_service.sh; create_signalr_service {resourceGroup} {serviceName} {sku} {unit}";
Util.Log(cmd);
(errCode, result) = ShellHelper.Bash(cmd, handleRes: true);
if (errCode != 0)
{
Util.Log($"Fail to create SignalR Service");
return(null);
}
// Check DNS ready
cmd = $"cd {extensionScriptsDir}; . ./az_signalr_service.sh; check_signalr_service_dns {resourceGroup} {serviceName}";
(errCode, result) = ShellHelper.Bash(cmd, handleRes: true);
if (errCode != 0 || !result.Equals("0"))
{
Util.Log($"SignalR service DNS is not ready to use");
return(null);
}
// Get ConnectionString
cmd = $"cd {extensionScriptsDir}; . ./az_signalr_service.sh; query_connection_string {serviceName} {resourceGroup}";
(errCode, result) = ShellHelper.Bash(cmd, handleRes: true);
if (errCode != 0)
{
Util.Log($"Fail to get connection string");
return(null);
}
return(result);
}
private string CreateSnapshotChecksum(string repoPath, int revNumber)
{
string uuid = ShellHelper.Bash("svnlook.exe", $"uuid {repoPath}");
MD5 mD5 = MD5.Create();
byte[] mD5bytes = mD5.ComputeHash(Encoding.ASCII.GetBytes(uuid + revNumber));
string result = BitConverter.ToString(mD5bytes).Replace("-", string.Empty);
mD5.Dispose();
return(result);
}
public RepoTree GetRepositoryTree(string url)
{
try
{
string logs = ShellHelper.Bash("svn.exe", $"log {url}");
return(logs
.ToRepoCommits(url)
.ToRepoTree());
}
catch (Exception e)
{
throw e;
}
}
private MemoryMetrics GetUnixMetrics()
{
string output = ShellHelper.Bash("free -m");
var lines = output.Split("\n");
var memory = lines[1].Split(" ", StringSplitOptions.RemoveEmptyEntries);
var metrics = new MemoryMetrics();
metrics.Total = double.Parse(memory[1]);
metrics.Used = double.Parse(memory[2]);
metrics.Free = double.Parse(memory[3]);
return(metrics);
}
public static string GetCPURate()
{
string cpuRate = string.Empty;
if (IsUnix())
{
string output = ShellHelper.Bash("top -b -n1 | grep \"Cpu(s)\" | awk '{print $2 + $4}'");
cpuRate = output.Trim();
}
else
{
string output = ShellHelper.Cmd("wmic", "cpu get LoadPercentage");
cpuRate = output.Replace("LoadPercentage", string.Empty).Trim();
}
return(cpuRate);
}
public static void runConfig(Command command)
{
var conf = command.args.Find(arg => arg.name == "set");
if (conf != null)
{
var email = conf.options.Find(opt => opt.name == "--email");
var token = conf.options.Find(opt => opt.name == "--token");
var url = conf.options.Find(opt => opt.name == "--url");
if (email != null && token != null && url != null)
{
ShellHelper.Bash("security delete-generic-password -s gel &>-");
ShellHelper.Bash($"security add-generic-password -a {email.value} -s gel -p {token.value} -U -j {url.value}>-");
}
}
}
public async Task UptimeAsync()
{
var processStart = Process.GetCurrentProcess().StartTime;
var procUptime = DateTime.Now.Subtract(processStart);
var hostUptime = ShellHelper.Bash("cat /proc/uptime");
var hostTimeStr = hostUptime.Substring(0, hostUptime.IndexOf(" "));
var hostTimeSeconds = double.Parse(hostTimeStr);
var hostTime = TimeSpan.FromSeconds(hostTimeSeconds);
var embed = new EmbedBuilder();
embed.WithTitle("Uptime");
embed.AddField("BOT", procUptime.Humanize(2), true);
embed.AddField("HOST", hostTime.Humanize(2), true);
await ReplyAsync(embed : embed.Build());
}
//Method for bulkloading
public void Bulkloader(Models.Task task)
{
string[] files = Directory.GetFiles(Program.myWorkingDirectory);
int counter = 1;
if (files.Length == 0)
{
return;
}
else
{
task.total = files.Length;
foreach (string file in files)
{
Console.Write("Bulk file " + counter + " of " + files.Length + "\n");
task.Status = "Bulk importing " + counter + " of " + files.Length;
task.current = counter;
string bulk = "/opt/mssql-tools/bin/bcp " + Program.myMainTable +
" in " + file +
" -S " + Program.myDbCall +
" -U " + Program.username +
" -P " + Program.password +
" -d " + Program.myDatabaseName +
" -c " +
" -t '\\t' -r '\\n'";
//Bulk insert it
Console.WriteLine("BULK:\n" + bulk);
task.details = (ShellHelper.Bash(bulk).ToString());
Console.WriteLine("RESULT:\n" + task.details);
//Remove it
ShellHelper.Bash("rm " + file);
//Count
counter++;
}
Console.Write("Bulk done: " + counter + " files.");
task.Status = "Done";
task.done = true;
Thread.Sleep(60000);
}
}
static List <string> ScanJarFile(string jarPath)
{
string cmd = string.Format("jar tf {0}", jarPath);
string output = ShellHelper.Bash(cmd);
string[] lines = output.Split('\n');
List <string> definitions = new List <string> ();
foreach (var line in lines)
{
if (line.Trim().EndsWith(".class"))
{
definitions.Add(line.Trim());
}
}
return(definitions);
}
public static void SetDirectoryPermission(string downloadDirectory)
{
var os = DevOpsHelper.GetOsPlatform();
if (os == OSPlatform.Windows)
{
var directoryInfo = new DirectoryInfo(downloadDirectory);
var security = directoryInfo.GetAccessControl();
var domain = Environment.UserDomainName;
var username = Environment.UserName;
security.AddAccessRule(new FileSystemAccessRule($@"{domain}\{username}", FileSystemRights.FullControl, AccessControlType.Allow));
directoryInfo.SetAccessControl(security);
}
else if (new[] { OSPlatform.OSX, OSPlatform.Linux }.Contains(os))
{
ShellHelper.Bash($"sudo chown {Environment.UserName} {downloadDirectory}");
}
}
private NetworkInfo GetNetwork()
{
var tcpConnections = Convert.ToInt32(ShellHelper.Bash("netstat -ant | grep ESTABLISHED | wc -l"));
var networkAdapters = new List <NetworkAdapter>();
foreach (var networkInterface in NetworkInterface.GetAllNetworkInterfaces())
{
var ips = new List <Ip>();
foreach (var ip in networkInterface.GetIPProperties().UnicastAddresses)
{
if (ip.Address.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork)
{
ips.Add(new()
{
Address = ip.Address.ToString(),
});
}
}
var ipStats = networkInterface.GetIPStatistics();
networkAdapters.Add(new()
{
BytesReceived = ipStats.BytesReceived,
BytesSent = ipStats.BytesSent,
IncomingErrors = ipStats.IncomingPacketsWithErrors,
OutgoingErrors = ipStats.OutgoingPacketsWithErrors,
OutgoingDiscarded = ipStats.OutgoingPacketsDiscarded,
Description = networkInterface.Description,
IsReceiveOnly = networkInterface.IsReceiveOnly,
Name = networkInterface.Name,
Ips = ips,
TcpConnectionCount = tcpConnections
});
}
return(new()
{
NetworkAdapters = networkAdapters,
TcpConnections = tcpConnections
});
}
private async Task <string> GetSearchData(SearchViewModel model)
{
SearchSoftwareViewModel soft;
string resultComplete = "[";
for (int i = 0; i < model.Softwares.Count; i++)
{
soft = model.Softwares[i];
for (int y = 0; y < soft.ResearchModules.Count; y++)
{
Console.WriteLine("execSoftwareOsint.sh " + soft.Name + " " + soft.ResearchModules[y] + " " + model.DataEntity);
Console.WriteLine("execSoftwareOsint.sh " + soft.Name + " " + soft.ResearchModules[y] + " " + model.DataEntity);
Console.WriteLine("execSoftwareOsint.sh " + soft.Name + " " + soft.ResearchModules[y] + " " + model.DataEntity);
Console.WriteLine("execSoftwareOsint.sh " + soft.Name + " " + soft.ResearchModules[y] + " " + model.DataEntity);
string result = _shellHelper.Bash("execSoftwareOsint.sh " + soft.Name + " " + soft.ResearchModules[y] + " " + model.DataEntity);
resultComplete += result + ",";
}
}
resultComplete = resultComplete.Substring(0, resultComplete.Length - 1);
resultComplete += "]";
return(resultComplete);
}
public IReadOnlyList <RecognitionResult> RecognizePlate(byte[] imageBytes)
{
var filePath = $"plate_{Guid.NewGuid()}.jpg";
System.IO.File.WriteAllBytes(filePath, imageBytes);
_logger.LogDebug($"image bytes saved to {filePath}");
_logger.LogInformation("Recognition start");
var json = ShellHelper.Bash($"alpr --config openalpr/openalpr.conf --json -c eu -p hr {filePath}");
_logger.LogDebug($"Result: {json}");
_logger.LogInformation("Recognition ended");
System.IO.File.Delete(filePath);
_logger.LogDebug("Deleting temp file");
_logger.LogDebug("Starting to convert object");
var openAlprResult = JsonConvert.DeserializeObject <OpelAlprResult>(json);
var result = _mapper.Map <List <RecognitionResult> >(openAlprResult);
return(result);
}
private void isPMD(String metaname, String directoryPath, String directoryTargetFilePath, MetaTigerAddon addon)
{
if (addon.Name == "PMD")
{
foreach (MetaTigerAction action in addon.Actions)
{
string commandFile = action.Command.Replace("{filepath}", directoryTargetFilePath);
ConsoleHelper.WriteWarningLine(directoryTargetFilePath);
string response = ShellHelper.Bash(commandFile, addon.FilePathName);
response = response.Replace(directoryTargetFilePath, "");
response = response.Replace("/", "");
string directoryTargetFilePathReply = directoryTargetFilePath.inverseBarLeft();
directoryTargetFilePathReply = directoryTargetFilePathReply.inverseBarRight();
response = response.Replace(directoryTargetFilePathReply, "");
response = response.escapeForEmpty();
response = response.Trim();
if (response.isNullOrEmpty())
{
ConsoleHelper.WriteWarningLine(response);
}
}
}
}
public static RepoCommit[] ToRepoCommits(this string svnLogs, string svnUrl)
{
string[] splitLogs = svnLogs.Split("\r\n");
List <RepoCommit> commits = new List <RepoCommit>();
for (int i = 0; i < splitLogs.Length - 2; i += 4) // The last two lines can be disregarded
{
int revNumber = (int)Char.GetNumericValue(splitLogs[i + 1][1]);
string message = $"r{revNumber} {splitLogs[i + 2]}";
if (splitLogs[i + 2] == "")
{
message = $"r{revNumber} (No commit message)";
}
DateTime date = DateTime.ParseExact(splitLogs[i + 1].Split('|')[2].Split('(')[0], " yyyy-MM-dd HH:mm:ss K ", System.Globalization.CultureInfo.InvariantCulture);
string svnList = ShellHelper.Bash("svn.exe", $"list {svnUrl} -R -r {revNumber}");
commits.Add(svnList.ToRepoCommit(message, date, revNumber));
}
return(commits.ToArray());
}
private IEnumerable <DriveInfo> GetDrive()
{
ShellHelper.Bash("df -h -x tmpfs | sed 's/ */,/g' > tempDrivesFile.csv");
var jsonData = CsvHelper.ConvertCsvToJson(File.ReadAllLines("tempDrivesFile.csv"));
var jsonList = JsonConvert.DeserializeObject <List <JObject> >(jsonData);
var driveInfo = new DriveInfo();
foreach (var driveJobject in jsonList)
{
var identString = driveJobject["Filesystem"]?.Value <string>();
var availableString = driveJobject["Avail"]?.Value <string>();
var sizeString = driveJobject["Size"]?.Value <string>();
var usedSting = driveJobject["Used"]?.Value <string>();
var usedPercentSting = driveJobject["Use%"]?.Value <string>();
yield return(new()
{
Available = UnitConverterHelper.ConvertToUnit(availableString, Unit.Gigabyte),
Identifier = identString,
Size = UnitConverterHelper.ConvertToUnit(sizeString, Unit.Gigabyte),
UsedPercentage = Convert.ToDouble(usedPercentSting?.Remove(usedPercentSting.Length - 1, 1)),
Used = UnitConverterHelper.ConvertToUnit(usedSting, Unit.Gigabyte),
});
}
}
/// <summary>
/// Process for Loading the data
/// </summary>
private async void Loader(Models.Task task, string sourceFile, string workingDirectory)
{
Stopwatch stopwatch = new Stopwatch();
List <string> TimeStatistics = new List <string>();
List <int> values = Program.values;
task.splitDone = false;
task.broadcastDone = false;
task.bulkDone = false;
string name = sourceFile.Split('/')[sourceFile.Split('/').Length - 1].Split('.')[0];
#region Delete 'log.txt' if exists
if (System.IO.File.Exists(AppDomain.CurrentDomain.BaseDirectory + "log" + name + ".txt"))
{
System.IO.File.Delete(AppDomain.CurrentDomain.BaseDirectory + "log" + name + ".txt");
}
#endregion
#region Delete 'time_log.txt' if exists
if (System.IO.File.Exists(AppDomain.CurrentDomain.BaseDirectory + "time_log" + name + ".txt"))
{
System.IO.File.Delete(AppDomain.CurrentDomain.BaseDirectory + "time_log" + name + ".txt");
}
#endregion
using (StreamWriter sw = System.IO.File.CreateText(AppDomain.CurrentDomain.BaseDirectory + "log" + name + ".txt"))
{
sw.WriteLine(DateTime.Now + " ::: Mater Load Started");
}
TimeStatistics.Add("\n\nNew measure: \n");
stopwatch.Start();
#region Split the file into pieces
task.Status = "split";
task.s_start = DateTime.Now;
//Thread for checking split status
Thread split_checker = new Thread(() => checkSplit(workingDirectory, sourceFile, task));
split_checker.Start();
//Setting and executing the SPLIT command to execute
//Everything is splited into pieces inside of sourcefile directory ~ workingdirectory/Run/
if (Int32.Parse(ShellHelper.Bash("test -e " + workingDirectory + "Run/ && echo 1 || echo 0")) == 1)
{
ShellHelper.Bash("rm -r " + workingDirectory + "Run/");
}
ShellHelper.Bash("mkdir " + workingDirectory + "Run/");
string splitBash = "split -l 100000 --additional-suffix=.csv " + sourceFile + " " + workingDirectory + "Run/";
task.details = ShellHelper.Bash(splitBash);
//Aborting split thread work after it's done
task.s_status = "Done";
task.splitDone = true;
task.s_end = DateTime.Now;
TimeStatistics.Add(DateTime.Now + ": Splitting the file into 100 000 line ones: " + stopwatch.Elapsed);
stopwatch.Restart();
#endregion
#region Broadcasting the files
task.Status = "broadcast";
task.b_start = DateTime.Now;
//Reading the new files one by one and doing stuff depending on MASTER/SLAVE
//Now it reads all the files from ~ workingdirectory/Run/
string[] files = Directory.GetFiles(workingDirectory + "Run/");
int counter = 1;
foreach (string file in files)
{
Random r = new Random();
int randomNumber = r.Next(0, values.Count);
task.b_status = "Broadcasting file " + counter + " of " + files.Length;
task.b_current = counter;
task.b_total = files.Length;
//The values[randomNumber] is allways a number between 0 (first server from configuration table) and max (last server from configuration table)
//The number will allways be in that scope so that is not a problem!
//Now we just send the file to the adress from the Program.Servers list (the value[randomNumber] will determine which one from the table is the destination!
using (StreamWriter sw = System.IO.File.AppendText(AppDomain.CurrentDomain.BaseDirectory + "log" + name + ".txt"))
{
List <string> result = await Sender2(task, Program.Servers[values[randomNumber]].api_call, "/slave/recieve", file.Split('/')[file.Split('/').Length - 1], counter, file);
task.details = string.Join('\n', result);
foreach (string line in result)
{
//Logging the output
sw.WriteLine(line);
}
}
counter++;
}
TimeStatistics.Add(DateTime.Now + ": Broadcasting the Files: " + stopwatch.Elapsed);
stopwatch.Restart();
task.broadcastDone = true;
task.b_status = "Done";
task.b_end = DateTime.Now;
#endregion
#region Deleting the /Run folder
//With one single bash line
ShellHelper.Bash("rm -r " + workingDirectory + "Run/");
#endregion
#region Bulk Insert Activation
//Activating the bulk insert
task.Status = "bulk";
task.blk_start = DateTime.Now;
List <string> slaveInfo = new List <string>();
foreach (Servers server in Program.Servers)
{
HttpClient client = new HttpClient();
try
{
HttpResponseMessage response = await client.GetAsync(server.api_call + "/slave/start_bulk");
if (response.IsSuccessStatusCode)
{
Stream receiveStream = await response.Content.ReadAsStreamAsync();
StreamReader readStream = new StreamReader(receiveStream, Encoding.UTF8);
string result = readStream.ReadToEnd();
slaveInfo.Add(server.api_call + " :" + result + ". Check it with /master/check_bulk_status");
}
else
{
slaveInfo.Add(server.api_call + " - Error, Slave Not Running, Try activating it manually with /master/start_bulk?slave={host}:{port}");
}
}
catch (Exception)
{
slaveInfo.Add(server.api_call + " - - Error, Slave Not Running, Try activating it manually with /master/start_bulk?slave={host}:{port}");
}
}
//Constantly repeat check! (Every 2 secs)
while (!task.bulkDone)
{
int sumcurrent = 0;
int sumtotal = 0;
string status = "";
bool done = true;
string details = "";
foreach (Servers server in Program.Servers)
{
using (var client = new HttpClient())
{
try
{
HttpResponseMessage response = await client.GetAsync(server.api_call + "/slave/check_bulk");
if (response.IsSuccessStatusCode)
{
Stream receiveStream = await response.Content.ReadAsStreamAsync();
StreamReader readStream = new StreamReader(receiveStream, Encoding.UTF8);
string result = readStream.ReadToEnd();
var res = JsonConvert.DeserializeObject <BulkModel>(result);
status += res.status + "/";
sumcurrent += res.current;
sumtotal += res.total;
done = done && res.done;
details += "Server n: " + res.details;
}
status = "Check if the slave is running";
}
catch (Exception)
{
status = "Check if the slave is running";
}
}
}
task.blk_current = sumcurrent;
task.blk_total = sumtotal;
task.blk_status = "Bulk importing " + sumcurrent + " of " + sumtotal;
task.bulkDone = done;
task.details = details;
task.blk_end = DateTime.Now;
Thread.Sleep(2000);
}
//Kill all tasks on servers (after 10 secs)
Thread.Sleep(2000);
foreach (Servers server in Program.Servers)
{
using (var client = new HttpClient())
{
try
{
HttpResponseMessage response = await client.GetAsync(server.api_call + "/slave/kill_task");
if (response.IsSuccessStatusCode)
{
Stream receiveStream = await response.Content.ReadAsStreamAsync();
StreamReader readStream = new StreamReader(receiveStream, Encoding.UTF8);
string result = readStream.ReadToEnd();
task.details = result;
}
task.details = "Check if the slave is running";
}
catch (Exception)
{
task.details = "Check if the slave is running";
}
}
}
//Bulk done
task.blk_status = "Done";
TimeStatistics.Add(DateTime.Now + ": Activating the bulk insertions: " + stopwatch.Elapsed);
stopwatch.Stop();
#endregion
#region Log the Time stats
task.Status = "time";
//Writing time stats to log file
using (System.IO.StreamWriter file =
new System.IO.StreamWriter(AppDomain.CurrentDomain.BaseDirectory + "time_log" + name + ".txt", true))
{
foreach (string line in TimeStatistics)
{
file.WriteLine(line);
}
}
#endregion
task.Status = "finished";
Thread.Sleep(60000);
//60 seconds the task is still active so the user can see the "Load finished" information before the task is killed.
Startup.taskList.Remove(task);
}