private static void setConfigs() { var consoleOutput = ShellHelper.Bash("security find-generic-password -s gel"); if (consoleOutput.Contains("acct")) { var start = consoleOutput.IndexOf("\"acct\"<blob>=\""); start = start + 14; var end = consoleOutput.IndexOf("\"", start); var email = consoleOutput.Substring(start, (end - start)); var key = ShellHelper.Bash("security find-generic-password -s gel -w"); start = consoleOutput.IndexOf("\"icmt\"<blob>=\""); start = start + 14; end = consoleOutput.IndexOf("\"", start); var url = consoleOutput.Substring(start, (end - start)); Program.email = email.Trim(); Program.key = key.Trim(); Program.url = url.Trim(); } else { throw new Exception("please set your credentials"); } }
public ActionResult Clear() { string command = Constants.PYTHON_BASE + "stop.py"; string result = ShellHelper.Bash(command); return(Json(new { Message = result })); }
public async Task <IActionResult> RunWraithTestAsync([FromForm] string path) { if (_chromeP.Length > 1 || _pahntomjsP.Length > 1) { Console.WriteLine("Test is running please wait!"); return(BadRequest()); } else { try { Console.WriteLine("The test path is " + path); var execCommand = ("cd " + path + "; pwd; wraith capture configs/capture.yaml"); Console.WriteLine(execCommand + "\n"); ShellHelper.Bash(execCommand); await Task.Run(() => { while (_chromeP.Length != 0 || _pahntomjsP.Length != 0) { } }); return(Json(new{ status = "done" })); } catch (Exception e) { Console.WriteLine(e); return(Json(e.Message)); } } }
//Thread function that checks split info public void checkSplit(string workingDirectory, string sourceFile, Models.Task task) { string sourceSize, folderSize; int sourceSizeGB, folderSizeGB; while (!task.splitDone) { try { sourceSize = ShellHelper.Bash("du -h -k " + sourceFile).Split('\t')[0]; sourceSizeGB = Int32.Parse(sourceSize) / 1024; folderSize = ShellHelper.Bash("du -sh -k " + workingDirectory + "Run/").Split('\t')[0]; folderSizeGB = Int32.Parse(folderSize) / 1024; task.s_status = "Splitting file into pieces. Current size: " + folderSizeGB + "KB of " + sourceSizeGB + "KB"; task.s_current = folderSizeGB; task.s_total = sourceSizeGB; Thread.Sleep(2000); } catch (Exception) { Thread.Sleep(2000); } } }
public async Task <byte[]> ShrinkAsync(IShrinkable input) { var startTime = DateTime.Now; var originalFilePath = Path.GetTempFileName(); using (var stream = new FileStream(originalFilePath, FileMode.Create)) { await input.Content.CopyToAsync(stream); } var processedFilePath = Path.GetTempFileName(); string cmd = $"gs -sDEVICE=pdfwrite -dCompatibilityLevel=1.4 -dQUIET -q -dNOPAUSE -dBATCH -dSAFER -dPDFSETTINGS=/screen -dEmbedAllFonts=true -dSubsetFonts=true -dAutoRotatePages=/None -dColorImageDownsampleType=/Bicubic -dColorImageResolution={input.Resolution} -dGrayImageDownsampleType=/Bicubic -dGrayImageResolution={input.Resolution} -dMonoImageDownsampleType=/Bicubic -dMonoImageResolution={input.Resolution} -sOutputFile={processedFilePath} {originalFilePath}"; var output = ShellHelper.Bash(cmd); var processedFileBytes = File.ReadAllBytes(processedFilePath); File.Delete(originalFilePath); File.Delete(processedFilePath); var elapsedTime = DateTime.Now.Subtract(startTime); _logger.LogInformation($"File ${input.Content.FileName} shrinked in {elapsedTime.Seconds} seconds."); return(processedFileBytes); }
public void OnPreprocessBuild(BuildReport report) { var commit = ShellHelper.Bash("git rev-parse HEAD").TrimEnd('\r', '\n'); File.WriteAllText(path, commit); AssetDatabase.ImportAsset(path, ImportAssetOptions.ForceSynchronousImport); }
public ActionResult LightPaint() { string command = Constants.PYTHON_BASE + "lightpaint.py"; string result = ShellHelper.Bash(command); return(Json(new { Message = result })); }
public static bool DeleteDogfoodSignalRService(string extensionScriptsDir, string resourceGroup, string serviceName, bool deleteResourceGroup = true) { var errCode = 0; var result = ""; bool rtn = false; var cmd = $"cd {extensionScriptsDir}; . ./az_signalr_service.sh; az_login_ASRS_dogfood"; (errCode, result) = ShellHelper.Bash(cmd, handleRes: true); if (errCode != 0) { Util.Log($"Fail to login to dogfood Azure"); return(false); } cmd = $"cd {extensionScriptsDir}; . ./az_signalr_service.sh; delete_signalr_service {serviceName} {resourceGroup}"; (errCode, result) = ShellHelper.Bash(cmd, handleRes: true); if (errCode != 0) { Util.Log($"Fail to delete SignalR Service"); rtn = false; } else { rtn = true; } if (deleteResourceGroup) { cmd = $"cd {extensionScriptsDir}; . ./az_signalr_service.sh; delete_group {resourceGroup}"; (errCode, result) = ShellHelper.Bash(cmd, handleRes: true); rtn = true; } return(rtn); }
public static string GetRunTime() { string runTime = string.Empty; try { if (IsUnix()) { string output = ShellHelper.Bash("uptime -s"); output = output.Trim(); runTime = DateTimeHelper.FormatTime(long.Parse((DateTime.Now - ParseToDateTime(output)).TotalMilliseconds.ToString().Split('.')[0])); } else { string output = ShellHelper.Cmd("wmic", "OS get LastBootUpTime/Value"); string[] outputArr = output.Split("=", StringSplitOptions.RemoveEmptyEntries); if (outputArr.Length == 2) { runTime = DateTimeHelper.FormatTime(long.Parse((DateTime.Now - ParseToDateTime(outputArr[1].Split('.')[0])).TotalMilliseconds.ToString().Split('.')[0])); } } } catch (Exception) { } return(runTime); }
private CpuInfo GetCpu() { var output = ShellHelper.Bash("top -bn1 | grep \"Cpu(s)\" | sed \"s/.*, *\\([0-9.]*\\)%* id.*/\\1/\" | awk '{print 100 - $1}'"); var lines = ShellHelper.TryReadFileLines("/proc/cpuinfo"); var coreCountRegex = new Regex(@"^cpu cores\s+:\s+(.+)", RegexOptions.IgnoreCase); var cpuCoresString = (lines.FirstOrDefault(o => coreCountRegex.Match(o).Success) ?? string.Empty); var clockSpeedRegex = new Regex(@"^cpu MHz\s+:\s+(.+)", RegexOptions.IgnoreCase); var clockSpeedString = (lines.FirstOrDefault(o => clockSpeedRegex.Match(o).Success) ?? string.Empty); var index = clockSpeedString.IndexOf('.'); if (index != -1) { clockSpeedString = clockSpeedString.Substring(0, index); } var cpu = new CpuInfo(); if (double.TryParse(output.Replace("%", string.Empty), out var totalPercentage)) { cpu.TotalPercentage = totalPercentage; } if (int.TryParse(coreCountRegex.Match(cpuCoresString).Groups[1].Value, out var cpuCores)) { cpu.NumberOfCores = cpuCores; } if (int.TryParse(clockSpeedRegex.Match(clockSpeedString).Groups[1].Value, out var clockSpeed)) { cpu.CurrentClockSpeed = clockSpeed; } return(cpu); }
static List <string> ScanAarFile(string aarPath) { List <string> results = new List <string>(); //1. Scan the aar to find any contained jar files string scanAar = string.Format("jar tf {0}", aarPath); string[] scanAarOutput = ShellHelper.Bash(scanAar).Split('\n'); //2. Foreach jar inside the aar foreach (var entry in scanAarOutput) { if (entry.Trim().EndsWith("jar")) { string outputPath = Path.Combine(Path.GetTempPath(), "check_jar.jar"); //2.1 unzip this jar file to the tmp folder string unzip = string.Format("unzip -p {0} {1} > {2}", aarPath, entry.Trim(), outputPath); ShellHelper.Bash(unzip); //2.2 scan the extracted jar results.AddRange(ScanJarFile(outputPath)); File.Delete(outputPath); } } return(results); }
public string TakeImage() { string outputFile = GetOutputFilename(); string command = "streamer -s " + outputResolution + " -f " + outputFormat + " -o " + outputFile; ShellHelper.Bash(command); return(outputFile); }
public Snapshot GetRepositorySnapshot(string svnUrl, string repoName, string snapshotId, string requestId) { Directory.CreateDirectory($"../Repos/{requestId}"); repoName = repoName.Replace(' ', '_'); string repoPath = $"../Repos/{requestId}/{repoName}"; string revisionPath = $"../Repos/{requestId}/revision"; try { string dumpPath = $"../Repos/{requestId}/{repoName}.svndump"; // Dump external SVN repo ShellHelper.Bash("svnrdump.exe", $"dump {svnUrl} -F {dumpPath}"); // Create new empty local SVN repo ShellHelper.Bash("svnadmin.exe", $"create {repoPath}"); // Load external dump to local repo ShellHelper.Bash("svnadmin.exe", $"load {repoPath} -F {dumpPath}"); string md5 = ""; byte[] repoBytes = new byte[0]; if (snapshotId == "none") { md5 = CreateRepositoryChecksum(dumpPath); repoBytes = compressionService.ZipBytes(repoPath, repoName, $"../Repos/{requestId}"); } else { // Checkout SVN revision ShellHelper.Bash("svn.exe", $"checkout {svnUrl} {revisionPath}"); DirectoryHelper.SetAttributesNormal(new DirectoryInfo(revisionPath)); Directory.Delete($"{revisionPath}/.svn", true); md5 = CreateSnapshotChecksum(repoPath, int.Parse(snapshotId)); repoBytes = compressionService.ZipBytes(revisionPath, repoName, $"../Repos/{requestId}"); } File.Delete(dumpPath); DeleteRequestDirectory(requestId); return(new Snapshot() { checksum = md5, zippedBytes = repoBytes }); } catch (Exception e) { DeleteRequestDirectory(requestId); throw e; } }
public static string CreateDogfoodSignalRService(string extensionScriptsDir, string location, string resourceGroup, string serviceName, string sku, int unit) { var errCode = 0; var result = ""; // Dogfood Azure login var cmd = $"cd {extensionScriptsDir}; . ./az_signalr_service.sh; az_login_ASRS_dogfood"; Util.Log(cmd); (errCode, result) = ShellHelper.Bash(cmd, handleRes: true); if (errCode != 0) { Util.Log($"Fail to login to dogfood Azure"); return(null); } // Create resource group if it does not exist cmd = $"cd {extensionScriptsDir}; . ./az_signalr_service.sh; create_group_if_not_exist {resourceGroup} {location}"; Util.Log(cmd); (errCode, result) = ShellHelper.Bash(cmd, handleRes: true); if (errCode != 0) { Util.Log($"Fail to create resource group"); return(null); } // Create SignalR service cmd = $"cd {extensionScriptsDir}; . ./az_signalr_service.sh; create_signalr_service {resourceGroup} {serviceName} {sku} {unit}"; Util.Log(cmd); (errCode, result) = ShellHelper.Bash(cmd, handleRes: true); if (errCode != 0) { Util.Log($"Fail to create SignalR Service"); return(null); } // Check DNS ready cmd = $"cd {extensionScriptsDir}; . ./az_signalr_service.sh; check_signalr_service_dns {resourceGroup} {serviceName}"; (errCode, result) = ShellHelper.Bash(cmd, handleRes: true); if (errCode != 0 || !result.Equals("0")) { Util.Log($"SignalR service DNS is not ready to use"); return(null); } // Get ConnectionString cmd = $"cd {extensionScriptsDir}; . ./az_signalr_service.sh; query_connection_string {serviceName} {resourceGroup}"; (errCode, result) = ShellHelper.Bash(cmd, handleRes: true); if (errCode != 0) { Util.Log($"Fail to get connection string"); return(null); } return(result); }
private string CreateSnapshotChecksum(string repoPath, int revNumber) { string uuid = ShellHelper.Bash("svnlook.exe", $"uuid {repoPath}"); MD5 mD5 = MD5.Create(); byte[] mD5bytes = mD5.ComputeHash(Encoding.ASCII.GetBytes(uuid + revNumber)); string result = BitConverter.ToString(mD5bytes).Replace("-", string.Empty); mD5.Dispose(); return(result); }
public RepoTree GetRepositoryTree(string url) { try { string logs = ShellHelper.Bash("svn.exe", $"log {url}"); return(logs .ToRepoCommits(url) .ToRepoTree()); } catch (Exception e) { throw e; } }
private MemoryMetrics GetUnixMetrics() { string output = ShellHelper.Bash("free -m"); var lines = output.Split("\n"); var memory = lines[1].Split(" ", StringSplitOptions.RemoveEmptyEntries); var metrics = new MemoryMetrics(); metrics.Total = double.Parse(memory[1]); metrics.Used = double.Parse(memory[2]); metrics.Free = double.Parse(memory[3]); return(metrics); }
public static string GetCPURate() { string cpuRate = string.Empty; if (IsUnix()) { string output = ShellHelper.Bash("top -b -n1 | grep \"Cpu(s)\" | awk '{print $2 + $4}'"); cpuRate = output.Trim(); } else { string output = ShellHelper.Cmd("wmic", "cpu get LoadPercentage"); cpuRate = output.Replace("LoadPercentage", string.Empty).Trim(); } return(cpuRate); }
public static void runConfig(Command command) { var conf = command.args.Find(arg => arg.name == "set"); if (conf != null) { var email = conf.options.Find(opt => opt.name == "--email"); var token = conf.options.Find(opt => opt.name == "--token"); var url = conf.options.Find(opt => opt.name == "--url"); if (email != null && token != null && url != null) { ShellHelper.Bash("security delete-generic-password -s gel &>-"); ShellHelper.Bash($"security add-generic-password -a {email.value} -s gel -p {token.value} -U -j {url.value}>-"); } } }
public async Task UptimeAsync() { var processStart = Process.GetCurrentProcess().StartTime; var procUptime = DateTime.Now.Subtract(processStart); var hostUptime = ShellHelper.Bash("cat /proc/uptime"); var hostTimeStr = hostUptime.Substring(0, hostUptime.IndexOf(" ")); var hostTimeSeconds = double.Parse(hostTimeStr); var hostTime = TimeSpan.FromSeconds(hostTimeSeconds); var embed = new EmbedBuilder(); embed.WithTitle("Uptime"); embed.AddField("BOT", procUptime.Humanize(2), true); embed.AddField("HOST", hostTime.Humanize(2), true); await ReplyAsync(embed : embed.Build()); }
//Method for bulkloading public void Bulkloader(Models.Task task) { string[] files = Directory.GetFiles(Program.myWorkingDirectory); int counter = 1; if (files.Length == 0) { return; } else { task.total = files.Length; foreach (string file in files) { Console.Write("Bulk file " + counter + " of " + files.Length + "\n"); task.Status = "Bulk importing " + counter + " of " + files.Length; task.current = counter; string bulk = "/opt/mssql-tools/bin/bcp " + Program.myMainTable + " in " + file + " -S " + Program.myDbCall + " -U " + Program.username + " -P " + Program.password + " -d " + Program.myDatabaseName + " -c " + " -t '\\t' -r '\\n'"; //Bulk insert it Console.WriteLine("BULK:\n" + bulk); task.details = (ShellHelper.Bash(bulk).ToString()); Console.WriteLine("RESULT:\n" + task.details); //Remove it ShellHelper.Bash("rm " + file); //Count counter++; } Console.Write("Bulk done: " + counter + " files."); task.Status = "Done"; task.done = true; Thread.Sleep(60000); } }
static List <string> ScanJarFile(string jarPath) { string cmd = string.Format("jar tf {0}", jarPath); string output = ShellHelper.Bash(cmd); string[] lines = output.Split('\n'); List <string> definitions = new List <string> (); foreach (var line in lines) { if (line.Trim().EndsWith(".class")) { definitions.Add(line.Trim()); } } return(definitions); }
public static void SetDirectoryPermission(string downloadDirectory) { var os = DevOpsHelper.GetOsPlatform(); if (os == OSPlatform.Windows) { var directoryInfo = new DirectoryInfo(downloadDirectory); var security = directoryInfo.GetAccessControl(); var domain = Environment.UserDomainName; var username = Environment.UserName; security.AddAccessRule(new FileSystemAccessRule($@"{domain}\{username}", FileSystemRights.FullControl, AccessControlType.Allow)); directoryInfo.SetAccessControl(security); } else if (new[] { OSPlatform.OSX, OSPlatform.Linux }.Contains(os)) { ShellHelper.Bash($"sudo chown {Environment.UserName} {downloadDirectory}"); } }
private NetworkInfo GetNetwork() { var tcpConnections = Convert.ToInt32(ShellHelper.Bash("netstat -ant | grep ESTABLISHED | wc -l")); var networkAdapters = new List <NetworkAdapter>(); foreach (var networkInterface in NetworkInterface.GetAllNetworkInterfaces()) { var ips = new List <Ip>(); foreach (var ip in networkInterface.GetIPProperties().UnicastAddresses) { if (ip.Address.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork) { ips.Add(new() { Address = ip.Address.ToString(), }); } } var ipStats = networkInterface.GetIPStatistics(); networkAdapters.Add(new() { BytesReceived = ipStats.BytesReceived, BytesSent = ipStats.BytesSent, IncomingErrors = ipStats.IncomingPacketsWithErrors, OutgoingErrors = ipStats.OutgoingPacketsWithErrors, OutgoingDiscarded = ipStats.OutgoingPacketsDiscarded, Description = networkInterface.Description, IsReceiveOnly = networkInterface.IsReceiveOnly, Name = networkInterface.Name, Ips = ips, TcpConnectionCount = tcpConnections }); } return(new() { NetworkAdapters = networkAdapters, TcpConnections = tcpConnections }); }
private async Task <string> GetSearchData(SearchViewModel model) { SearchSoftwareViewModel soft; string resultComplete = "["; for (int i = 0; i < model.Softwares.Count; i++) { soft = model.Softwares[i]; for (int y = 0; y < soft.ResearchModules.Count; y++) { Console.WriteLine("execSoftwareOsint.sh " + soft.Name + " " + soft.ResearchModules[y] + " " + model.DataEntity); Console.WriteLine("execSoftwareOsint.sh " + soft.Name + " " + soft.ResearchModules[y] + " " + model.DataEntity); Console.WriteLine("execSoftwareOsint.sh " + soft.Name + " " + soft.ResearchModules[y] + " " + model.DataEntity); Console.WriteLine("execSoftwareOsint.sh " + soft.Name + " " + soft.ResearchModules[y] + " " + model.DataEntity); string result = _shellHelper.Bash("execSoftwareOsint.sh " + soft.Name + " " + soft.ResearchModules[y] + " " + model.DataEntity); resultComplete += result + ","; } } resultComplete = resultComplete.Substring(0, resultComplete.Length - 1); resultComplete += "]"; return(resultComplete); }
public IReadOnlyList <RecognitionResult> RecognizePlate(byte[] imageBytes) { var filePath = $"plate_{Guid.NewGuid()}.jpg"; System.IO.File.WriteAllBytes(filePath, imageBytes); _logger.LogDebug($"image bytes saved to {filePath}"); _logger.LogInformation("Recognition start"); var json = ShellHelper.Bash($"alpr --config openalpr/openalpr.conf --json -c eu -p hr {filePath}"); _logger.LogDebug($"Result: {json}"); _logger.LogInformation("Recognition ended"); System.IO.File.Delete(filePath); _logger.LogDebug("Deleting temp file"); _logger.LogDebug("Starting to convert object"); var openAlprResult = JsonConvert.DeserializeObject <OpelAlprResult>(json); var result = _mapper.Map <List <RecognitionResult> >(openAlprResult); return(result); }
private void isPMD(String metaname, String directoryPath, String directoryTargetFilePath, MetaTigerAddon addon) { if (addon.Name == "PMD") { foreach (MetaTigerAction action in addon.Actions) { string commandFile = action.Command.Replace("{filepath}", directoryTargetFilePath); ConsoleHelper.WriteWarningLine(directoryTargetFilePath); string response = ShellHelper.Bash(commandFile, addon.FilePathName); response = response.Replace(directoryTargetFilePath, ""); response = response.Replace("/", ""); string directoryTargetFilePathReply = directoryTargetFilePath.inverseBarLeft(); directoryTargetFilePathReply = directoryTargetFilePathReply.inverseBarRight(); response = response.Replace(directoryTargetFilePathReply, ""); response = response.escapeForEmpty(); response = response.Trim(); if (response.isNullOrEmpty()) { ConsoleHelper.WriteWarningLine(response); } } } }
public static RepoCommit[] ToRepoCommits(this string svnLogs, string svnUrl) { string[] splitLogs = svnLogs.Split("\r\n"); List <RepoCommit> commits = new List <RepoCommit>(); for (int i = 0; i < splitLogs.Length - 2; i += 4) // The last two lines can be disregarded { int revNumber = (int)Char.GetNumericValue(splitLogs[i + 1][1]); string message = $"r{revNumber} {splitLogs[i + 2]}"; if (splitLogs[i + 2] == "") { message = $"r{revNumber} (No commit message)"; } DateTime date = DateTime.ParseExact(splitLogs[i + 1].Split('|')[2].Split('(')[0], " yyyy-MM-dd HH:mm:ss K ", System.Globalization.CultureInfo.InvariantCulture); string svnList = ShellHelper.Bash("svn.exe", $"list {svnUrl} -R -r {revNumber}"); commits.Add(svnList.ToRepoCommit(message, date, revNumber)); } return(commits.ToArray()); }
private IEnumerable <DriveInfo> GetDrive() { ShellHelper.Bash("df -h -x tmpfs | sed 's/ */,/g' > tempDrivesFile.csv"); var jsonData = CsvHelper.ConvertCsvToJson(File.ReadAllLines("tempDrivesFile.csv")); var jsonList = JsonConvert.DeserializeObject <List <JObject> >(jsonData); var driveInfo = new DriveInfo(); foreach (var driveJobject in jsonList) { var identString = driveJobject["Filesystem"]?.Value <string>(); var availableString = driveJobject["Avail"]?.Value <string>(); var sizeString = driveJobject["Size"]?.Value <string>(); var usedSting = driveJobject["Used"]?.Value <string>(); var usedPercentSting = driveJobject["Use%"]?.Value <string>(); yield return(new() { Available = UnitConverterHelper.ConvertToUnit(availableString, Unit.Gigabyte), Identifier = identString, Size = UnitConverterHelper.ConvertToUnit(sizeString, Unit.Gigabyte), UsedPercentage = Convert.ToDouble(usedPercentSting?.Remove(usedPercentSting.Length - 1, 1)), Used = UnitConverterHelper.ConvertToUnit(usedSting, Unit.Gigabyte), }); } }
/// <summary> /// Process for Loading the data /// </summary> private async void Loader(Models.Task task, string sourceFile, string workingDirectory) { Stopwatch stopwatch = new Stopwatch(); List <string> TimeStatistics = new List <string>(); List <int> values = Program.values; task.splitDone = false; task.broadcastDone = false; task.bulkDone = false; string name = sourceFile.Split('/')[sourceFile.Split('/').Length - 1].Split('.')[0]; #region Delete 'log.txt' if exists if (System.IO.File.Exists(AppDomain.CurrentDomain.BaseDirectory + "log" + name + ".txt")) { System.IO.File.Delete(AppDomain.CurrentDomain.BaseDirectory + "log" + name + ".txt"); } #endregion #region Delete 'time_log.txt' if exists if (System.IO.File.Exists(AppDomain.CurrentDomain.BaseDirectory + "time_log" + name + ".txt")) { System.IO.File.Delete(AppDomain.CurrentDomain.BaseDirectory + "time_log" + name + ".txt"); } #endregion using (StreamWriter sw = System.IO.File.CreateText(AppDomain.CurrentDomain.BaseDirectory + "log" + name + ".txt")) { sw.WriteLine(DateTime.Now + " ::: Mater Load Started"); } TimeStatistics.Add("\n\nNew measure: \n"); stopwatch.Start(); #region Split the file into pieces task.Status = "split"; task.s_start = DateTime.Now; //Thread for checking split status Thread split_checker = new Thread(() => checkSplit(workingDirectory, sourceFile, task)); split_checker.Start(); //Setting and executing the SPLIT command to execute //Everything is splited into pieces inside of sourcefile directory ~ workingdirectory/Run/ if (Int32.Parse(ShellHelper.Bash("test -e " + workingDirectory + "Run/ && echo 1 || echo 0")) == 1) { ShellHelper.Bash("rm -r " + workingDirectory + "Run/"); } ShellHelper.Bash("mkdir " + workingDirectory + "Run/"); string splitBash = "split -l 100000 --additional-suffix=.csv " + sourceFile + " " + workingDirectory + "Run/"; task.details = ShellHelper.Bash(splitBash); //Aborting split thread work after it's done task.s_status = "Done"; task.splitDone = true; task.s_end = DateTime.Now; TimeStatistics.Add(DateTime.Now + ": Splitting the file into 100 000 line ones: " + stopwatch.Elapsed); stopwatch.Restart(); #endregion #region Broadcasting the files task.Status = "broadcast"; task.b_start = DateTime.Now; //Reading the new files one by one and doing stuff depending on MASTER/SLAVE //Now it reads all the files from ~ workingdirectory/Run/ string[] files = Directory.GetFiles(workingDirectory + "Run/"); int counter = 1; foreach (string file in files) { Random r = new Random(); int randomNumber = r.Next(0, values.Count); task.b_status = "Broadcasting file " + counter + " of " + files.Length; task.b_current = counter; task.b_total = files.Length; //The values[randomNumber] is allways a number between 0 (first server from configuration table) and max (last server from configuration table) //The number will allways be in that scope so that is not a problem! //Now we just send the file to the adress from the Program.Servers list (the value[randomNumber] will determine which one from the table is the destination! using (StreamWriter sw = System.IO.File.AppendText(AppDomain.CurrentDomain.BaseDirectory + "log" + name + ".txt")) { List <string> result = await Sender2(task, Program.Servers[values[randomNumber]].api_call, "/slave/recieve", file.Split('/')[file.Split('/').Length - 1], counter, file); task.details = string.Join('\n', result); foreach (string line in result) { //Logging the output sw.WriteLine(line); } } counter++; } TimeStatistics.Add(DateTime.Now + ": Broadcasting the Files: " + stopwatch.Elapsed); stopwatch.Restart(); task.broadcastDone = true; task.b_status = "Done"; task.b_end = DateTime.Now; #endregion #region Deleting the /Run folder //With one single bash line ShellHelper.Bash("rm -r " + workingDirectory + "Run/"); #endregion #region Bulk Insert Activation //Activating the bulk insert task.Status = "bulk"; task.blk_start = DateTime.Now; List <string> slaveInfo = new List <string>(); foreach (Servers server in Program.Servers) { HttpClient client = new HttpClient(); try { HttpResponseMessage response = await client.GetAsync(server.api_call + "/slave/start_bulk"); if (response.IsSuccessStatusCode) { Stream receiveStream = await response.Content.ReadAsStreamAsync(); StreamReader readStream = new StreamReader(receiveStream, Encoding.UTF8); string result = readStream.ReadToEnd(); slaveInfo.Add(server.api_call + " :" + result + ". Check it with /master/check_bulk_status"); } else { slaveInfo.Add(server.api_call + " - Error, Slave Not Running, Try activating it manually with /master/start_bulk?slave={host}:{port}"); } } catch (Exception) { slaveInfo.Add(server.api_call + " - - Error, Slave Not Running, Try activating it manually with /master/start_bulk?slave={host}:{port}"); } } //Constantly repeat check! (Every 2 secs) while (!task.bulkDone) { int sumcurrent = 0; int sumtotal = 0; string status = ""; bool done = true; string details = ""; foreach (Servers server in Program.Servers) { using (var client = new HttpClient()) { try { HttpResponseMessage response = await client.GetAsync(server.api_call + "/slave/check_bulk"); if (response.IsSuccessStatusCode) { Stream receiveStream = await response.Content.ReadAsStreamAsync(); StreamReader readStream = new StreamReader(receiveStream, Encoding.UTF8); string result = readStream.ReadToEnd(); var res = JsonConvert.DeserializeObject <BulkModel>(result); status += res.status + "/"; sumcurrent += res.current; sumtotal += res.total; done = done && res.done; details += "Server n: " + res.details; } status = "Check if the slave is running"; } catch (Exception) { status = "Check if the slave is running"; } } } task.blk_current = sumcurrent; task.blk_total = sumtotal; task.blk_status = "Bulk importing " + sumcurrent + " of " + sumtotal; task.bulkDone = done; task.details = details; task.blk_end = DateTime.Now; Thread.Sleep(2000); } //Kill all tasks on servers (after 10 secs) Thread.Sleep(2000); foreach (Servers server in Program.Servers) { using (var client = new HttpClient()) { try { HttpResponseMessage response = await client.GetAsync(server.api_call + "/slave/kill_task"); if (response.IsSuccessStatusCode) { Stream receiveStream = await response.Content.ReadAsStreamAsync(); StreamReader readStream = new StreamReader(receiveStream, Encoding.UTF8); string result = readStream.ReadToEnd(); task.details = result; } task.details = "Check if the slave is running"; } catch (Exception) { task.details = "Check if the slave is running"; } } } //Bulk done task.blk_status = "Done"; TimeStatistics.Add(DateTime.Now + ": Activating the bulk insertions: " + stopwatch.Elapsed); stopwatch.Stop(); #endregion #region Log the Time stats task.Status = "time"; //Writing time stats to log file using (System.IO.StreamWriter file = new System.IO.StreamWriter(AppDomain.CurrentDomain.BaseDirectory + "time_log" + name + ".txt", true)) { foreach (string line in TimeStatistics) { file.WriteLine(line); } } #endregion task.Status = "finished"; Thread.Sleep(60000); //60 seconds the task is still active so the user can see the "Load finished" information before the task is killed. Startup.taskList.Remove(task); }