/// <summary> /// /// </summary> /// <param name="deviceIdentity"></param> /// <param name="deviceId"></param> private static async void ManageDeviceJob(DeviceIdentity deviceIdentity, string deviceId) { var jobClient = JobClient.CreateFromConnectionString(deviceIdentity.GetConnectionString()); string methodJobId = Guid.NewGuid().ToString(); await deviceIdentity.StartMethodJob(methodJobId, jobClient, deviceId); Console.WriteLine("Started Twin Update Job"); var result1 = deviceIdentity.MonitorJob(methodJobId, jobClient).Result; Console.WriteLine("Job Status : " + result1.Status); string twinUpdateJobId = Guid.NewGuid().ToString(); await deviceIdentity.StartTwinUpdateJob(twinUpdateJobId, jobClient, deviceId); result1 = deviceIdentity.MonitorJob(twinUpdateJobId, jobClient).Result; Console.WriteLine("Job Status : " + result1.Status); Console.WriteLine("Press ENTER to exit."); Console.ReadLine(); }
public async Task OnlyJobProcessorShouldBeCalled_IfOutOfBandAttributeHasBeenApplied_ToNotificationHandler() { var outOfBandHandled = false; OutOfBandNotificationHandler.Handled += (_, __) => outOfBandHandled = true; var syncHandled = false; OutOfBandNotificationHandlerWithoutAttribute.Handled += (_, __) => syncHandled = true; var handlers = Container.GetAllInstances <INotificationHandler <OutOfBandNotification> >().ToList(); Assert.Equal(2, handlers.Count); foreach (var handler in handlers) { await handler.Handle(new OutOfBandNotification(), CancellationToken.None); } JobClient.ReceivedWithAnyArgs(1).Enqueue <JobProcessor>(x => x.ProcessAsync(null !, null !)); Assert.False(outOfBandHandled, "Handle must not be called if handler is marked for out of band processing."); Assert.True(syncHandled); }
public RedisJobServerAsyncTest() { //Configure storage connection var clientConfig = new ClientConfig(); clientConfig.DBConnectionString = "localhost:6379"; clientConfig.StorageMode = "redis"; jobClient = new JobClient(clientConfig); var serverConfig = new ServerConfig(); serverConfig.DBConnectionString = "localhost:6379"; serverConfig.StorageMode = "redis"; serverConfig.ProcessID = "JobServerAsyncTest"; serverConfig.Workers = 1; serverConfig.MaxRunnableJobs = 1; serverConfig.ProgressDBInterval = new TimeSpan(0); serverConfig.AutoDeletePeriod = null; serverConfig.ForceStopServer = true; serverConfig.StopServerDelay = 3000; jobServer = new JobServer(serverConfig); }
/// <exception cref="System.Exception"/> private void EncryptedShuffleWithCerts(bool useClientCerts) { try { Configuration conf = new Configuration(); string keystoresDir = new FilePath(Basedir).GetAbsolutePath(); string sslConfsDir = KeyStoreTestUtil.GetClasspathDir(typeof(TestEncryptedShuffle )); KeyStoreTestUtil.SetupSSLConfig(keystoresDir, sslConfsDir, conf, useClientCerts); conf.SetBoolean(MRConfig.ShuffleSslEnabledKey, true); StartCluster(conf); FileSystem fs = FileSystem.Get(GetJobConf()); Path inputDir = new Path("input"); fs.Mkdirs(inputDir); TextWriter writer = new OutputStreamWriter(fs.Create(new Path(inputDir, "data.txt" ))); writer.Write("hello"); writer.Close(); Path outputDir = new Path("output", "output"); JobConf jobConf = new JobConf(GetJobConf()); jobConf.SetInt("mapred.map.tasks", 1); jobConf.SetInt("mapred.map.max.attempts", 1); jobConf.SetInt("mapred.reduce.max.attempts", 1); jobConf.Set("mapred.input.dir", inputDir.ToString()); jobConf.Set("mapred.output.dir", outputDir.ToString()); JobClient jobClient = new JobClient(jobConf); RunningJob runJob = jobClient.SubmitJob(jobConf); runJob.WaitForCompletion(); NUnit.Framework.Assert.IsTrue(runJob.IsComplete()); NUnit.Framework.Assert.IsTrue(runJob.IsSuccessful()); } finally { StopCluster(); } }
/// <exception cref="System.Exception"/> private long GetTaskCounterUsage(JobClient client, JobID id, int numReports, int taskId, TaskType type) { TaskReport[] reports = null; if (TaskType.Map.Equals(type)) { reports = client.GetMapTaskReports(id); } else { if (TaskType.Reduce.Equals(type)) { reports = client.GetReduceTaskReports(id); } } NUnit.Framework.Assert.IsNotNull("No reports found for task type '" + type.ToString () + "' in job " + id, reports); // make sure that the total number of reports match the expected NUnit.Framework.Assert.AreEqual("Mismatch in task id", numReports, reports.Length ); Counters counters = reports[taskId].GetCounters(); return(counters.GetCounter(TaskCounter.CommittedHeapBytes)); }
private void DownloadImageRootProduct(ImageProductInfo imageProductInfo, JobClient updateImagePathProductJobClient, ProducerBasic thumbImageJobClient) { var fileName = Common.UnicodeToKoDauAndGach(imageProductInfo.Name); if (fileName.Length > 100) { fileName = fileName.Substring(0, 99); } fileName += ".jpg"; try { var folder = Common.GetFolderSaveImageRootProduct(fileName); Common.SaveImageProduct(imageProductInfo.ImageUrls, _pathImageRootProduct + folder, fileName); imageProductInfo.ImagePath = Common.GetImagePathRootProduct(folder, fileName); imageProductInfo.DownloadedTime = DateTime.Now; SendMessageToServiceUpdateImagePath(imageProductInfo, updateImagePathProductJobClient); var fulldirectory = _pathImageProduct.Replace("\\", @"\") + folder.Replace("\\", @"\") + fileName; var thumbImageInfo = new ThumbImageProductInfo() { ProductId = imageProductInfo.Id, FileNameImage = fileName, FolderImage = folder.Replace("\\", @"\"), FullDirectoryImage = fulldirectory, SizeImage = widthHeightImages, TypeProduct = 2 }; SendMessageToServiceThumbImage(thumbImageInfo, thumbImageJobClient); Log.Info(string.Format("RootProduct: ID = {0} download image success!", imageProductInfo.Id)); } catch (Exception exception) { Log.Error(string.Format("RootProduct: ID = {0}. ImageUrl: {1} . DetailUrl: {2}", imageProductInfo.Id, imageProductInfo.ImageUrls, imageProductInfo.DetailUrl), exception); imageProductInfo.ErrorMessage = exception.ToString(); SendErrorDownloadImageToService(imageProductInfo); } }
public FFMpegService(string uri, int threadsPerMachine, IAudioPresetProvider audioPresetProvider, ITimeProvider timeProvider) { _httpClient = new HttpClient { Timeout = TimeSpan.FromSeconds(60) }; _serviceUri = uri; _audioClient = new AudioJobClient(_httpClient) { BaseUrl = _serviceUri }; _audioMuxClient = new MuxJobClient(_httpClient) { BaseUrl = _serviceUri }; _hardSubtitlesClient = new HardSubtitlesJobClient(_httpClient) { BaseUrl = _serviceUri }; _statusClient = new StatusClient(_httpClient) { BaseUrl = _serviceUri }; _healthClient = new HealthCheckClient(_httpClient) { BaseUrl = _serviceUri }; _jobClient = new JobClient(_httpClient) { BaseUrl = _serviceUri }; _timeProvider = timeProvider; _threadsPerMachine = threadsPerMachine; _tasksPerAudioJob = audioPresetProvider.AsDictionary()[StateFormat.audio_od_standard].Count; RetryCount = 3; RetrySleepMs = 10 * 1000; }
/// <exception cref="System.IO.IOException"/> public virtual void TestCollect() { JobConf conf = new JobConf(); Configure(conf); try { JobClient.RunJob(conf); // check if all the values were seen by the reducer if (TestCollect.Reduce.numSeen != (NumCollectsPerThread * NumFeeders)) { throw new IOException("Collect test failed!! Total does not match."); } } catch (IOException ioe) { throw; } finally { FileSystem fs = FileSystem.Get(conf); fs.Delete(OutputDir, true); } }
/// <exception cref="System.Exception"/> public virtual void TestChain() { Path inDir = new Path("testing/chain/input"); Path outDir = new Path("testing/chain/output"); // Hack for local FS that does not have the concept of a 'mounting point' if (IsLocalFS()) { string localPathRoot = Runtime.GetProperty("test.build.data", "/tmp").Replace(' ' , '+'); inDir = new Path(localPathRoot, inDir); outDir = new Path(localPathRoot, outDir); } JobConf conf = CreateJobConf(); conf.SetBoolean("localFS", IsLocalFS()); conf.SetInt("mapreduce.job.maps", 1); CleanFlags(conf); FileSystem fs = FileSystem.Get(conf); fs.Delete(outDir, true); if (!fs.Mkdirs(inDir)) { throw new IOException("Mkdirs failed to create " + inDir.ToString()); } DataOutputStream file = fs.Create(new Path(inDir, "part-0")); file.WriteBytes("1\n2\n"); file.Close(); conf.SetJobName("chain"); conf.SetInputFormat(typeof(TextInputFormat)); conf.SetOutputFormat(typeof(TextOutputFormat)); conf.Set("a", "X"); JobConf mapAConf = new JobConf(false); mapAConf.Set("a", "A"); ChainMapper.AddMapper(conf, typeof(TestChainMapReduce.AMap), typeof(LongWritable) , typeof(Text), typeof(LongWritable), typeof(Text), true, mapAConf); ChainMapper.AddMapper(conf, typeof(TestChainMapReduce.BMap), typeof(LongWritable) , typeof(Text), typeof(LongWritable), typeof(Text), false, null); JobConf reduceConf = new JobConf(false); reduceConf.Set("a", "C"); ChainReducer.SetReducer(conf, typeof(TestChainMapReduce.CReduce), typeof(LongWritable ), typeof(Text), typeof(LongWritable), typeof(Text), true, reduceConf); ChainReducer.AddMapper(conf, typeof(TestChainMapReduce.DMap), typeof(LongWritable ), typeof(Text), typeof(LongWritable), typeof(Text), false, null); JobConf mapEConf = new JobConf(false); mapEConf.Set("a", "E"); ChainReducer.AddMapper(conf, typeof(TestChainMapReduce.EMap), typeof(LongWritable ), typeof(Text), typeof(LongWritable), typeof(Text), true, mapEConf); FileInputFormat.SetInputPaths(conf, inDir); FileOutputFormat.SetOutputPath(conf, outDir); JobClient jc = new JobClient(conf); RunningJob job = jc.SubmitJob(conf); while (!job.IsComplete()) { Sharpen.Thread.Sleep(100); } NUnit.Framework.Assert.IsTrue(GetFlag(conf, "configure.A")); NUnit.Framework.Assert.IsTrue(GetFlag(conf, "configure.B")); NUnit.Framework.Assert.IsTrue(GetFlag(conf, "configure.C")); NUnit.Framework.Assert.IsTrue(GetFlag(conf, "configure.D")); NUnit.Framework.Assert.IsTrue(GetFlag(conf, "configure.E")); NUnit.Framework.Assert.IsTrue(GetFlag(conf, "map.A.value.1")); NUnit.Framework.Assert.IsTrue(GetFlag(conf, "map.A.value.2")); NUnit.Framework.Assert.IsTrue(GetFlag(conf, "map.B.value.1")); NUnit.Framework.Assert.IsTrue(GetFlag(conf, "map.B.value.2")); NUnit.Framework.Assert.IsTrue(GetFlag(conf, "reduce.C.value.2")); NUnit.Framework.Assert.IsTrue(GetFlag(conf, "reduce.C.value.1")); NUnit.Framework.Assert.IsTrue(GetFlag(conf, "map.D.value.1")); NUnit.Framework.Assert.IsTrue(GetFlag(conf, "map.D.value.2")); NUnit.Framework.Assert.IsTrue(GetFlag(conf, "map.E.value.1")); NUnit.Framework.Assert.IsTrue(GetFlag(conf, "map.E.value.2")); NUnit.Framework.Assert.IsTrue(GetFlag(conf, "close.A")); NUnit.Framework.Assert.IsTrue(GetFlag(conf, "close.B")); NUnit.Framework.Assert.IsTrue(GetFlag(conf, "close.C")); NUnit.Framework.Assert.IsTrue(GetFlag(conf, "close.D")); NUnit.Framework.Assert.IsTrue(GetFlag(conf, "close.E")); }
public virtual void TestHeapUsageCounter() { JobConf conf = new JobConf(); // create a local filesystem handle FileSystem fileSystem = FileSystem.GetLocal(conf); // define test root directories Path rootDir = new Path(Runtime.GetProperty("test.build.data", "/tmp")); Path testRootDir = new Path(rootDir, "testHeapUsageCounter"); // cleanup the test root directory fileSystem.Delete(testRootDir, true); // set the current working directory fileSystem.SetWorkingDirectory(testRootDir); fileSystem.DeleteOnExit(testRootDir); // create a mini cluster using the local file system MiniMRCluster mrCluster = new MiniMRCluster(1, fileSystem.GetUri().ToString(), 1); try { conf = mrCluster.CreateJobConf(); JobClient jobClient = new JobClient(conf); // define job input Path inDir = new Path(testRootDir, "in"); // create input data CreateWordsFile(inDir, conf); // configure and run a low memory job which will run without loading the // jvm's heap RunningJob lowMemJob = RunHeapUsageTestJob(conf, testRootDir, "-Xms32m -Xmx1G", 0 , 0, fileSystem, jobClient, inDir); JobID lowMemJobID = lowMemJob.GetID(); long lowMemJobMapHeapUsage = GetTaskCounterUsage(jobClient, lowMemJobID, 1, 0, TaskType .Map); System.Console.Out.WriteLine("Job1 (low memory job) map task heap usage: " + lowMemJobMapHeapUsage ); long lowMemJobReduceHeapUsage = GetTaskCounterUsage(jobClient, lowMemJobID, 1, 0, TaskType.Reduce); System.Console.Out.WriteLine("Job1 (low memory job) reduce task heap usage: " + lowMemJobReduceHeapUsage ); // configure and run a high memory job which will load the jvm's heap RunningJob highMemJob = RunHeapUsageTestJob(conf, testRootDir, "-Xms32m -Xmx1G", lowMemJobMapHeapUsage + 256 * 1024 * 1024, lowMemJobReduceHeapUsage + 256 * 1024 * 1024, fileSystem, jobClient, inDir); JobID highMemJobID = highMemJob.GetID(); long highMemJobMapHeapUsage = GetTaskCounterUsage(jobClient, highMemJobID, 1, 0, TaskType.Map); System.Console.Out.WriteLine("Job2 (high memory job) map task heap usage: " + highMemJobMapHeapUsage ); long highMemJobReduceHeapUsage = GetTaskCounterUsage(jobClient, highMemJobID, 1, 0, TaskType.Reduce); System.Console.Out.WriteLine("Job2 (high memory job) reduce task heap usage: " + highMemJobReduceHeapUsage); NUnit.Framework.Assert.IsTrue("Incorrect map heap usage reported by the map task" , lowMemJobMapHeapUsage < highMemJobMapHeapUsage); NUnit.Framework.Assert.IsTrue("Incorrect reduce heap usage reported by the reduce task" , lowMemJobReduceHeapUsage < highMemJobReduceHeapUsage); } finally { // shutdown the mr cluster mrCluster.Shutdown(); try { fileSystem.Delete(testRootDir, true); } catch (IOException) { } } }
private void UpdateImagePathProduct(ImageProductInfo imageProductInfo, ProductTableAdapter productAdapter, JobClient historyJobClient, JobClient updateRedisJobClient) { while (_isRunning) { try { if (productAdapter.Connection.State == ConnectionState.Closed) { productAdapter.Connection.Open(); } productAdapter.UpdateImagePathAndValidAndIsNews(imageProductInfo.ImagePath, imageProductInfo.Id); Log.Info(string.Format("ProductId {0} : Update ImagePath And Valid success.", imageProductInfo.Id)); break; } catch (Exception exception) { Log.Error(string.Format("ProductId {0} : Update ImagePath And Valid error.", imageProductInfo.Id), exception); Thread.Sleep(60000); } } SendMessageToServiceInsertHistoryDownload(new LogHistoryImageProduct { DateLog = imageProductInfo.DownloadedTime, IsDownloaded = true, ErrorName = string.Empty, ProductId = imageProductInfo.Id, NewsToValid = imageProductInfo.IsNew }, historyJobClient); if (imageProductInfo.IsNew) { SendMessageToServiceUpdateSolrAndRedis(imageProductInfo.Id); } else { SendMessageToServiceUpdateRedis(imageProductInfo.Id, updateRedisJobClient); } }
/// <summary>create and run an Aggregate based map/reduce job.</summary> /// <param name="args">the arguments used for job creation</param> /// <exception cref="System.IO.IOException"/> public static void Main(string[] args) { JobConf job = ValueAggregatorJob.CreateValueAggregatorJob(args); JobClient.RunJob(job); }
public TrackCrawler() { rabbitMQServer = RabbitMQManager.GetRabbitMQServer("rabbitMQ177"); updateProductJobClient_TrackCrawlerProduct = new JobClient("TrackCrawler", GroupType.Topic, "TrackCrawler_TrackInfo", true, rabbitMQServer); }
/// <summary>The main driver for sort program.</summary> /// <remarks> /// The main driver for sort program. /// Invoke this method to submit the map/reduce job. /// </remarks> /// <exception cref="System.IO.IOException"> /// When there is communication problems with the /// job tracker. /// </exception> /// <exception cref="System.Exception"/> public virtual int Run(string[] args) { Configuration conf = GetConf(); JobClient client = new JobClient(conf); ClusterStatus cluster = client.GetClusterStatus(); int num_reduces = (int)(cluster.GetMaxReduceTasks() * 0.9); string join_reduces = conf.Get(ReducesPerHost); if (join_reduces != null) { num_reduces = cluster.GetTaskTrackers() * System.Convert.ToInt32(join_reduces); } Job job = Job.GetInstance(conf); job.SetJobName("join"); job.SetJarByClass(typeof(Sort)); job.SetMapperClass(typeof(Mapper)); job.SetReducerClass(typeof(Reducer)); Type inputFormatClass = typeof(SequenceFileInputFormat); Type outputFormatClass = typeof(SequenceFileOutputFormat); Type outputKeyClass = typeof(BytesWritable); Type outputValueClass = typeof(TupleWritable); string op = "inner"; IList <string> otherArgs = new AList <string>(); for (int i = 0; i < args.Length; ++i) { try { if ("-r".Equals(args[i])) { num_reduces = System.Convert.ToInt32(args[++i]); } else { if ("-inFormat".Equals(args[i])) { inputFormatClass = Sharpen.Runtime.GetType(args[++i]).AsSubclass <InputFormat>(); } else { if ("-outFormat".Equals(args[i])) { outputFormatClass = Sharpen.Runtime.GetType(args[++i]).AsSubclass <OutputFormat>(); } else { if ("-outKey".Equals(args[i])) { outputKeyClass = Sharpen.Runtime.GetType(args[++i]).AsSubclass <WritableComparable >(); } else { if ("-outValue".Equals(args[i])) { outputValueClass = Sharpen.Runtime.GetType(args[++i]).AsSubclass <Writable>(); } else { if ("-joinOp".Equals(args[i])) { op = args[++i]; } else { otherArgs.AddItem(args[i]); } } } } } } } catch (FormatException) { System.Console.Out.WriteLine("ERROR: Integer expected instead of " + args[i]); return(PrintUsage()); } catch (IndexOutOfRangeException) { System.Console.Out.WriteLine("ERROR: Required parameter missing from " + args[i - 1]); return(PrintUsage()); } } // exits // Set user-supplied (possibly default) job configs job.SetNumReduceTasks(num_reduces); if (otherArgs.Count < 2) { System.Console.Out.WriteLine("ERROR: Wrong number of parameters: "); return(PrintUsage()); } FileOutputFormat.SetOutputPath(job, new Path(otherArgs.Remove(otherArgs.Count - 1 ))); IList <Path> plist = new AList <Path>(otherArgs.Count); foreach (string s in otherArgs) { plist.AddItem(new Path(s)); } job.SetInputFormatClass(typeof(CompositeInputFormat)); job.GetConfiguration().Set(CompositeInputFormat.JoinExpr, CompositeInputFormat.Compose (op, inputFormatClass, Sharpen.Collections.ToArray(plist, new Path[0]))); job.SetOutputFormatClass(outputFormatClass); job.SetOutputKeyClass(outputKeyClass); job.SetOutputValueClass(outputValueClass); DateTime startTime = new DateTime(); System.Console.Out.WriteLine("Job started: " + startTime); int ret = job.WaitForCompletion(true) ? 0 : 1; DateTime end_time = new DateTime(); System.Console.Out.WriteLine("Job ended: " + end_time); System.Console.Out.WriteLine("The job took " + (end_time.GetTime() - startTime.GetTime ()) / 1000 + " seconds."); return(ret); }
public JobsSample(JobClient jobClient) { _jobClient = jobClient ?? throw new ArgumentNullException(nameof(jobClient)); }
private void DownloadImageSPGoc(long companyId, bool reloadall = false) { rabbitMQServer = RabbitMQManager.GetRabbitMQServer(rabbitMQServerName); //Jobclient update solr and redis var updateProductJobClient = new JobClient(updateProductExchangeGroupName, GroupType.Topic, updateProductqueueJobName, true, rabbitMQServer); //JobClient download image fails //JobClient downloadImageProductJobClient = new JobClient(updateProductImageGroupName, GroupType.Topic, updateProductImageProductJobName, true, rabbitMQServer); //JobClient delete thumb var deleteThumbJobClient = new JobClient(updateProductImageGroupName, GroupType.Topic, deleteThumbImageProductJobName, true, rabbitMQServer); WSS.DownloadImageByHand.DBTableAdapters.ProductTableAdapter productAdapter = new WSS.DownloadImageByHand.DBTableAdapters.ProductTableAdapter(); productAdapter.Connection.ConnectionString = connectionString; WSS.DownloadImageByHand.DB.ProductDataTable productTable = new WSS.DownloadImageByHand.DB.ProductDataTable(); int demgetproduct = 0; while (true) { try { if (reloadall) { productAdapter.FillBy_AllSPGoc(productTable, companyId); } else { //Lấy danh sách sản phẩm valid = 1 productAdapter.FillBy_SPGocAndValid(productTable, companyId); } break; } catch (Exception ex) { demgetproduct++; if (demgetproduct == 1) { Log.Error(string.Format("CompanyID = {0} Get Product of Company in SQL Error 1", companyId), ex); } else if (demgetproduct == numbererror) { Log.Error(string.Format("CompanyID = {0} Get Product of Company in SQL Error {1}", companyId, numbererror), ex); break; } Thread.Sleep(60000); } } try { if (productTable.Rows.Count > 0) { this.Invoke(new Action(() => { rbsuccess.AppendText(companyId + ".Get " + productTable.Rows.Count + " Product from SQL..." + System.Environment.NewLine); })); string path = pathImageSPGOC; int countsuccess = 0; int countproduct = productTable.Rows.Count; for (int i = 0; i < productTable.Rows.Count; i++) { long productId = Common.Obj2Int64(productTable.Rows[i]["ID"].ToString()); string nameProduct = productTable.Rows[i]["Name"].ToString(); string imageUrl = productTable.Rows[i]["ImageUrls"].ToString(); if (string.IsNullOrEmpty(nameProduct) || string.IsNullOrEmpty(imageUrl)) { Log.Error(string.Format("CompanyID = {0} ProductID = {1} Name or ImageUrl is null or empty.", companyId, productId)); continue; } string filename = Common.UnicodeToKoDauAndGach(nameProduct); string tempf = filename.Replace("-", ""); if (tempf.Length < 3) { continue; } string folder = tempf.Substring(0, 3); if (folder == "bin") { folder = "bin1"; } if (folder == "con") { folder = "con1"; } if (folder == "aux") { folder = "aux1"; } if (folder == "prn") { folder = "prn1"; } if (folder == "nul") { folder = "nul1"; } if (filename.Length > 100) { filename = filename.Substring(0, 99); } bool fileSaved = true; try { //fileSaved = Common.SaveFileDownloadImage(imageUrl, path + folder, filename + ".jpg", productId, companyId); } catch (Exception) { } if (fileSaved) { int w = 0, h = 0; string pathsave = "Store/images/" + folder + "/" + filename + ".jpg"; while (checkstop) { try { productAdapter.UpdateQuery(pathsave, w, h, productId); Log.InfoFormat("{0}/{1}. CompanyID = {2} ProductID = {3} Download Image success.", i, countproduct, companyId, productId); countsuccess++; #region Send Message Update solr and redis Job job = new Job(); job.Data = BitConverter.GetBytes(productId); job.Type = 2; updateProductJobClient.PublishJob(job, updateProductJobExpirationMS); //Log.InfoFormat("Send message To RabbitMq {0} with ID = {1}", rabbitMQServerName, productId); #endregion #region Xóa ảnh Thumb nếu có //push message lên service xóa ảnh thumb Job deletejob = new Job(); MqThumbImageInfo thumb = new MqThumbImageInfo(); thumb.ProductId = productId; thumb.FolderImage = folder; thumb.ImageName = filename; thumb.TypeProduct = 2; deletejob.Data = MqThumbImageInfo.GetMess(thumb); deleteThumbJobClient.PublishJob(deletejob); #endregion this.Invoke(new Action(() => { rbsuccess.AppendText(string.Format("{0}...{1}: {2}.{3} sucess.", 1, companyId, i, productId) + System.Environment.NewLine); })); break; } catch (Exception ex) { this.Invoke(new Action(() => { rbfail.AppendText(i + "." + productId + ": Update Solr + Push service Delete Thumb fails." + ex.ToString() + System.Environment.NewLine); })); break; //Thread.Sleep(10000); //dem++; //if (dem == 1) // Log.Error(string.Format("CompanyID = {0} Product = {1}, Update ImagePath Error 1", companyId, productId), ex); //else if (dem == numbererror) //{ // Log.Error(string.Format("CompanyID = {0} Product = {1}, Update ImagePath Error {2}", companyId, productId, numbererror), ex); // break; //} } } } else { this.Invoke(new Action(() => { //lbFails.Text = countfail.ToString(); rbsuccess.AppendText(string.Format("{0}...{1}: {2}.{3} fails.", 1, companyId, i, productId) + System.Environment.NewLine); rbfail.AppendText(i + "." + productId + ": Download fails." + imageUrl + " ...Name: " + nameProduct + System.Environment.NewLine); })); } } Log.InfoFormat("CompanyID = {0} Download Image Success {1}/{2}", companyId, countsuccess, productTable.Rows.Count); } else { Log.InfoFormat("CompanyID = {0} 0 product (valid = 1 and ImagePath is Null or empty)", companyId); } } catch (Exception ex) { Log.Error(string.Format("CompanyID = {0} ERROR~~~", companyId), ex); } }
static void Main(string[] args) { string jobId = "DHCMD" + Guid.NewGuid().ToString(); // The query condition can also be on a single device Id or on a list of device Ids. // https://docs.microsoft.com/en-us/azure/iot-hub/iot-hub-devguide-query-language covers // IoT Hub query language in additional detail. string query = $"DeviceId IN ['{deviceId}']"; Twin twin = new Twin(deviceId); twin.Tags = new TwinCollection(); twin.Tags[TestTagName] = TestTagValue; // *************************************** Start JobClient *************************************** Console.WriteLine($"Create JobClient from the connectionString..."); JobClient jobClient = JobClient.CreateFromConnectionString(connectionString); Console.WriteLine($"JobClient created with success"); Console.WriteLine(); // *************************************** Schedule twin job *************************************** Console.WriteLine($"Schedule twin job {jobId} for {deviceId}..."); JobResponse createJobResponse = jobClient.ScheduleTwinUpdateAsync( jobId, query, twin, DateTime.UtcNow, (long)TimeSpan.FromMinutes(2).TotalSeconds).Result; Console.WriteLine($"Schedule response"); Console.WriteLine(JsonConvert.SerializeObject(createJobResponse, Formatting.Indented)); Console.WriteLine(); // *************************************** Get all Jobs *************************************** IEnumerable <JobResponse> queryResults = jobClient.CreateQuery().GetNextAsJobResponseAsync().Result; var getJobs = queryResults.ToList(); Console.WriteLine($"getJobs return {getJobs.Count} result(s)"); foreach (JobResponse job in getJobs) { Console.WriteLine(JsonConvert.SerializeObject(job, Formatting.Indented)); if (job.Status != JobStatus.Completed) { Console.WriteLine($"Incorrect query jobs result"); return; } } Console.WriteLine(); // *************************************** Check completion *************************************** Console.WriteLine($"Monitoring jobClient for job completion..."); JobResponse jobResponse = jobClient.GetJobAsync(jobId).Result; Console.WriteLine($"First result"); Console.WriteLine(JsonConvert.SerializeObject(jobResponse, Formatting.Indented)); while (jobResponse.Status != JobStatus.Completed) { Task.Delay(TimeSpan.FromMilliseconds(100)); jobResponse = jobClient.GetJobAsync(jobId).Result; } Console.WriteLine($"Job ends with status {jobResponse.Status}"); }
public TransportationJobClient(string connectionString) { jobClient = JobClient.CreateFromConnectionString(connectionString); }
private void InitData() { _productAdapter = new ProductAdapter(new SqlDb(ConfigCrawler.ConnectProduct)); _jobClient = new JobClient(ConfigImages.ImboExchangeImages, GroupType.Topic, ConfigImages.ImboRoutingKeyDownloadImageProduct, true, RabbitMQManager.GetRabbitMQServer(ConfigImages.RabbitMqServerName)); }
public void Setup() { httpClientHelperMock = new Mock <IHttpClientHelper>(); jobClient = new JobClient(httpClientHelperMock.Object); }
private void SendMessageToServiceInsertHistoryDownload(LogHistoryImageProduct historyImageProduct, JobClient historyJobClient) { var job = new Job { Data = LogHistoryImageProduct.GetMessage(historyImageProduct) }; while (_isRunning) { try { historyJobClient.PublishJob(job); Log.Info(string.Format("Push message to services insert history download image productid = {0}", historyImageProduct.ProductId)); break; } catch (Exception ex) { Log.Error(string.Format("ProductId = {0} Push message to services insert history download image error.", historyImageProduct.ProductId), ex); Thread.Sleep(60000); } } }
public async Task Execute(long ciProjectId, long ciBuildId, long ciJobId, long serverId, bool serverIsExternal, int retries, CancellationToken cancellationToken) { // Includes are needed here to provide fully populated data for update notifications var job = await Database.CiJobs.Include(j => j.Build !).ThenInclude(b => b.CiProject) .FirstOrDefaultAsync( j => j.CiProjectId == ciProjectId && j.CiBuildId == ciBuildId && j.CiJobId == ciJobId, cancellationToken); BaseServer?server; if (serverIsExternal) { server = await Database.ExternalServers.FindAsync(new object[] { serverId }, cancellationToken); } else { server = await Database.ControlledServers.FindAsync(new object[] { serverId }, cancellationToken); } if (server == null) { throw new ArgumentException($"Could not find server ({serverId}, external: {serverIsExternal}) " + "to run build on"); } if (job == null) { Logger.LogWarning("Skipping CI job as it doesn't exist"); ReleaseServerReservation(server); return; } if (job.State != CIJobState.WaitingForServer) { Logger.LogWarning( "CI job is not in waiting for server status, refusing to start running it on server: {ServerId}", serverId); ReleaseServerReservation(server); return; } if (server.ReservedFor != job.CiJobId) { Logger.LogWarning( "CI job id doesn't match reservation on server, refusing to start it on server: {ServerId}", serverId); ReleaseServerReservation(server); return; } // Get the CI image for the job var imageFileName = job.GetImageFileName(); var serverSideImagePath = Path.Join("CI/Images", imageFileName); StorageItem?imageItem; try { imageItem = await StorageItem.FindByPath(Database, serverSideImagePath); } catch (Exception e) { // ReSharper disable once ExceptionPassedAsTemplateArgumentProblem Logger.LogError("Invalid image specified for CI job: {Image}, path parse exception: {@E}", job.Image, e); job.SetFinishSuccess(false); await job.CreateFailureSection(Database, "Invalid image specified for job (invalid path)"); await OnJobEnded(server, job); return; } if (string.IsNullOrEmpty(job.Image) || imageItem == null) { Logger.LogError("Invalid image specified for CI job: {Image}", job.Image); job.SetFinishSuccess(false); await job.CreateFailureSection(Database, "Invalid image specified for job (not found)"); await OnJobEnded(server, job); return; } // The CI system uses the first valid image version. For future updates a different file name is needed // For example bumping the ":v1" to a ":v2" suffix var version = await imageItem.GetLowestUploadedVersion(Database); if (version == null || version.StorageFile == null) { Logger.LogError("Image with no uploaded version specified for CI job: {Image}", job.Image); job.SetFinishSuccess(false); await job.CreateFailureSection(Database, "Invalid image specified for job (not uploaded version)"); await OnJobEnded(server, job); return; } // Queue a job to lock writing to the CI image if it isn't write protected yet if (imageItem.WriteAccess != FileAccess.Nobody) { Logger.LogInformation( "Storage item {Id} used as CI image is not write locked, queuing a job to lock it", imageItem.Id); // To ensure the upload time is expired, this is upload time + 5 minutes JobClient.Schedule <LockCIImageItemJob>(x => x.Execute(imageItem.Id, CancellationToken.None), AppInfo.RemoteStorageUploadExpireTime + TimeSpan.FromMinutes(5)); } Logger.LogInformation("Trying to start job {CIProjectId}-{CIBuildId}-{CIJobId} on reserved " + "server ({Id}, {ServerIsExternal})", ciProjectId, ciBuildId, ciJobId, server.Id, serverIsExternal); if (server.PublicAddress == null || server.PublicAddress.Equals(IPAddress.None)) { throw new Exception($"Server ({server.Id}, {serverIsExternal}) doesn't have a public address set"); } // Try to start running the job, this can fail if the server is not actually really up yet IBaseSSHAccess sshAccess; try { if (serverIsExternal) { externalSSHAccess.ConnectTo(server.PublicAddress.ToString(), ((ExternalServer)server).SSHKeyFileName); sshAccess = externalSSHAccess; } else { controlledSSHAccess.ConnectTo(server.PublicAddress.ToString()); sshAccess = controlledSSHAccess; } } catch (SocketException) { Logger.LogInformation("Connection failed (socket exception), server is probably not up (yet)"); await Requeue(job, retries - 1, server, serverIsExternal); return; } catch (SshOperationTimeoutException) { Logger.LogInformation("Connection failed (ssh timed out), server is probably not up (yet)"); await Requeue(job, retries - 1, server, serverIsExternal); return; } var imageDownloadUrl = remoteDownloadUrls.CreateDownloadFor(version.StorageFile, AppInfo.RemoteStorageDownloadExpireTime); // Connection success, so now we can run the job starting on the server job.RunningOnServerId = serverId; job.RunningOnServerIsExternal = server.IsExternal; // TODO: permanently store on which server this job was ran on and how long since creation it took to get // here if (job.Build == null) { throw new NotLoadedModelNavigationException(); } CISecretType jobSpecificSecretType = job.Build.IsSafe ? CISecretType.SafeOnly : CISecretType.UnsafeOnly; var secrets = await Database.CiSecrets.Where(s => s.CiProjectId == job.CiProjectId && (s.UsedForBuildTypes == jobSpecificSecretType || s.UsedForBuildTypes == CISecretType.All)) .ToListAsync(cancellationToken); await PerformServerCleanUpIfNeeded(server, sshAccess); // Then move on to the build starting, first thing is to download the CI executor script // TODO: is there a possibility that this is not secure? Someone would need to do HTTPS MItM attack... var executorDownload = GetUrlToDownloadCIExecutor(); var executorResourceDownload = GetUrlToDownloadCIExecutorResource(); var executorHash = await hashCalculator.Sha256(executorDownload, cancellationToken); var posixHelperHash = await hashCalculator.Sha256(executorResourceDownload, cancellationToken); // TODO: using async would be nice for the run commands when supported var result1 = sshAccess .RunCommand("set -e\n" + CreateDownloadCommand("~/CIExecutor", executorHash, executorDownload) + CreateDownloadCommand("~/libMonoPosixHelper.so", posixHelperHash, executorResourceDownload) + "chmod +x ~/CIExecutor"); if (!result1.Success) { throw new Exception($"Failed to run executor download step: {result1.Result}, error: {result1.Error}"); } // This save is done here as the build status might get reported back to us before we finish with the ssh // commands job.State = CIJobState.Running; job.RanOnServer = serverIsExternal ? $"External server {serverId}" : $"Controlled server {serverId}"; job.TimeWaitingForServer = DateTime.UtcNow - job.CreatedAt; await Database.SaveChangesAsync(cancellationToken); // and then run it with environment variables for this build // Remove all type secrets if there is one with the same name that is build specific var cleanedSecrets = secrets .Where(s => s.UsedForBuildTypes != CISecretType.All || !secrets.Any(s2 => s2.SecretName == s.SecretName && s2.UsedForBuildTypes != s.UsedForBuildTypes)) .Select(s => s.ToExecutorData()); if (job.Build.CiProject == null) { throw new NotLoadedModelNavigationException(); } var env = new StringBuilder(250); env.Append("export CI_REF=\""); env.Append(BashEscape.EscapeForBash(job.Build.RemoteRef)); env.Append("\"; export CI_COMMIT_HASH=\""); env.Append(BashEscape.EscapeForBash(job.Build.CommitHash)); env.Append("\"; export CI_EARLIER_COMMIT=\""); env.Append(BashEscape.EscapeForBash(job.Build.PreviousCommit ?? AppInfo.NoCommitHash)); env.Append("\"; export CI_BRANCH=\""); env.Append(BashEscape.EscapeForBash(job.Build.Branch ?? "unknown_branch")); env.Append("\"; export CI_DEFAULT_BRANCH=\""); env.Append(BashEscape.EscapeForBash(job.Build.CiProject.DefaultBranch)); env.Append("\"; export CI_TRUSTED=\""); env.Append(job.Build.IsSafe); env.Append("\"; export CI_ORIGIN=\""); env.Append(BashEscape.EscapeForBash(job.Build.CiProject.RepositoryCloneUrl)); env.Append("\"; export CI_IMAGE_DL_URL=\""); env.Append(BashEscape.EscapeForBash(imageDownloadUrl)); env.Append("\"; export CI_IMAGE_NAME=\""); env.Append(BashEscape.EscapeForBash(job.Image)); env.Append("\"; export CI_IMAGE_FILENAME=\""); env.Append(BashEscape.EscapeForBash(imageFileName)); env.Append("\"; export CI_CACHE_OPTIONS=\""); env.Append(BashEscape.EscapeForBash(job.CacheSettingsJson ?? "{}")); env.Append("\"; export CI_SECRETS=\""); env.Append(BashEscape.EscapeForBash(JsonSerializer.Serialize(cleanedSecrets))); env.Append("\"; export CI_JOB_NAME=\""); env.Append(BashEscape.EscapeForBash(job.JobName)); env.Append("\";"); var result2 = sshAccess.RunCommand($"{env} nohup ~/CIExecutor {GetConnectToUrl(job)} > " + "build_script_output.txt 2>&1 &"); if (!result2.Success) { throw new Exception($"Failed to start running CI executor: {result2.Result}, error: {result2.Error}"); } JobClient.Schedule <CheckCIJobOutputHasConnectedJob>( x => x.Execute(ciProjectId, ciBuildId, ciJobId, serverId, CancellationToken.None), TimeSpan.FromMinutes(5)); JobClient.Schedule <CancelCIBuildIfStuckJob>( x => x.Execute(ciProjectId, ciBuildId, ciJobId, serverId, server.IsExternal, CancellationToken.None), TimeSpan.FromMinutes(61)); Logger.LogInformation( "CI job startup succeeded, now it's up to the executor to contact us with updates"); }
/// <exception cref="System.Exception"/> public virtual int Run(string[] args) { if (args.Length < 4) { //input-dir should contain a huge file ( > 2GB) Usage(); } Path bigMapInput = null; Path outputPath = null; bool createInput = false; long fileSizeInMB = 3 * 1024; // default of 3GB (>2GB) for (int i = 0; i < args.Length; ++i) { if ("-input".Equals(args[i])) { bigMapInput = new Path(args[++i]); } else { if ("-output".Equals(args[i])) { outputPath = new Path(args[++i]); } else { if ("-create".Equals(args[i])) { createInput = true; fileSizeInMB = long.Parse(args[++i]); } else { Usage(); } } } } FileSystem fs = FileSystem.Get(GetConf()); JobConf jobConf = new JobConf(GetConf(), typeof(BigMapOutput)); jobConf.SetJobName("BigMapOutput"); jobConf.SetInputFormat(typeof(SortValidator.RecordStatsChecker.NonSplitableSequenceFileInputFormat )); jobConf.SetOutputFormat(typeof(SequenceFileOutputFormat)); FileInputFormat.SetInputPaths(jobConf, bigMapInput); if (fs.Exists(outputPath)) { fs.Delete(outputPath, true); } FileOutputFormat.SetOutputPath(jobConf, outputPath); jobConf.SetMapperClass(typeof(IdentityMapper)); jobConf.SetReducerClass(typeof(IdentityReducer)); jobConf.SetOutputKeyClass(typeof(BytesWritable)); jobConf.SetOutputValueClass(typeof(BytesWritable)); if (createInput) { CreateBigMapInputFile(jobConf, fs, bigMapInput, fileSizeInMB); } DateTime startTime = new DateTime(); System.Console.Out.WriteLine("Job started: " + startTime); JobClient.RunJob(jobConf); DateTime end_time = new DateTime(); System.Console.Out.WriteLine("Job ended: " + end_time); return(0); }
// The following methods are only used when LoadGenerator is run a MR job /// <summary>Based on args we submit the LoadGenerator as MR job.</summary> /// <remarks> /// Based on args we submit the LoadGenerator as MR job. /// Number of MapTasks is numMapTasks /// </remarks> /// <returns>exitCode for job submission</returns> private int SubmitAsMapReduce() { System.Console.Out.WriteLine("Running as a MapReduce job with " + numMapTasks + " mapTasks; Output to file " + mrOutDir); Configuration conf = new Configuration(GetConf()); // First set all the args of LoadGenerator as Conf vars to pass to MR tasks conf.Set(LgRoot, root.ToString()); conf.SetInt(LgMaxdelaybetweenops, maxDelayBetweenOps); conf.SetInt(LgNumofthreads, numOfThreads); conf.Set(LgReadpr, readProbs[0] + string.Empty); //Pass Double as string conf.Set(LgWritepr, writeProbs[0] + string.Empty); //Pass Double as string conf.SetLong(LgSeed, seed); //No idea what this is conf.SetInt(LgNummaptasks, numMapTasks); if (scriptFile == null && durations[0] <= 0) { System.Console.Error.WriteLine("When run as a MapReduce job, elapsed Time or ScriptFile must be specified" ); System.Environment.Exit(-1); } conf.SetLong(LgElapsedtime, durations[0]); conf.SetLong(LgStarttime, startTime); if (scriptFile != null) { conf.Set(LgScriptfile, scriptFile); } conf.Set(LgFlagfile, flagFile.ToString()); // Now set the necessary conf variables that apply to run MR itself. JobConf jobConf = new JobConf(conf, typeof(Org.Apache.Hadoop.FS.LoadGenerator.LoadGenerator )); jobConf.SetJobName("NNLoadGeneratorViaMR"); jobConf.SetNumMapTasks(numMapTasks); jobConf.SetNumReduceTasks(1); // 1 reducer to collect the results jobConf.SetOutputKeyClass(typeof(Text)); jobConf.SetOutputValueClass(typeof(IntWritable)); jobConf.SetMapperClass(typeof(LoadGeneratorMR.MapperThatRunsNNLoadGenerator)); jobConf.SetReducerClass(typeof(LoadGeneratorMR.ReducerThatCollectsLGdata)); jobConf.SetInputFormat(typeof(LoadGeneratorMR.DummyInputFormat)); jobConf.SetOutputFormat(typeof(TextOutputFormat)); // Explicitly set number of max map attempts to 1. jobConf.SetMaxMapAttempts(1); // Explicitly turn off speculative execution jobConf.SetSpeculativeExecution(false); // This mapReduce job has no input but has output FileOutputFormat.SetOutputPath(jobConf, new Path(mrOutDir)); try { JobClient.RunJob(jobConf); } catch (IOException e) { System.Console.Error.WriteLine("Failed to run job: " + e.Message); return(-1); } return(0); }
// set up heap options, target value for memory loader and the output // directory before running the job /// <exception cref="System.IO.IOException"/> private static RunningJob RunHeapUsageTestJob(JobConf conf, Path testRootDir, string heapOptions, long targetMapValue, long targetReduceValue, FileSystem fs, JobClient client, Path inDir) { // define a job JobConf jobConf = new JobConf(conf); // configure the jobs jobConf.SetNumMapTasks(1); jobConf.SetNumReduceTasks(1); jobConf.SetMapperClass(typeof(TestJobCounters.MemoryLoaderMapper)); jobConf.SetReducerClass(typeof(TestJobCounters.MemoryLoaderReducer)); jobConf.SetInputFormat(typeof(TextInputFormat)); jobConf.SetOutputKeyClass(typeof(LongWritable)); jobConf.SetOutputValueClass(typeof(Org.Apache.Hadoop.IO.Text)); jobConf.SetMaxMapAttempts(1); jobConf.SetMaxReduceAttempts(1); jobConf.Set(JobConf.MapredTaskJavaOpts, heapOptions); // set the targets jobConf.SetLong(TestJobCounters.MemoryLoaderMapper.TargetValue, targetMapValue); jobConf.SetLong(TestJobCounters.MemoryLoaderReducer.TargetValue, targetReduceValue ); // set the input directory for the job FileInputFormat.SetInputPaths(jobConf, inDir); // define job output folder Path outDir = new Path(testRootDir, "out"); fs.Delete(outDir, true); FileOutputFormat.SetOutputPath(jobConf, outDir); // run the job RunningJob job = client.SubmitJob(jobConf); job.WaitForCompletion(); JobID jobID = job.GetID(); NUnit.Framework.Assert.IsTrue("Job " + jobID + " failed!", job.IsSuccessful()); return(job); }
private void ResendToServiceDownloadImageProduct(ImageProductInfo imageProductInfo, JobClient downloadImageProductJobClient) { var job = new Job { Data = ImageProductInfo.GetMessage(imageProductInfo) }; while (_isRunning) { try { downloadImageProductJobClient.PublishJob(job); Log.Info(string.Format("Resend to services download image productid {0}", imageProductInfo.Id)); break; } catch (Exception ex) { Log.Error(string.Format("ProductId = {0} resend message to service downloadimageproduct.", imageProductInfo.Id), ex); Thread.Sleep(120000); } } }
// generates the input for the benchmark /// <summary>This is the main routine for launching the benchmark.</summary> /// <remarks> /// This is the main routine for launching the benchmark. It generates random /// input data. The input is non-splittable. Sort is used for benchmarking. /// This benchmark reports the effect of having multiple sort and spill /// cycles over a single sort and spill. /// </remarks> /// <exception cref="System.IO.IOException"></exception> /// <exception cref="System.Exception"/> public virtual int Run(string[] args) { Log.Info("Starting the benchmark for threaded spills"); string version = "ThreadedMapBenchmark.0.0.1"; System.Console.Out.WriteLine(version); string usage = "Usage: threadedmapbenchmark " + "[-dataSizePerMap <data size (in mb) per map, default is 128 mb>] " + "[-numSpillsPerMap <number of spills per map, default is 2>] " + "[-numMapsPerHost <number of maps per host, default is 1>]"; int dataSizePerMap = 128; // in mb int numSpillsPerMap = 2; int numMapsPerHost = 1; JobConf masterConf = new JobConf(GetConf()); for (int i = 0; i < args.Length; i++) { // parse command line if (args[i].Equals("-dataSizePerMap")) { dataSizePerMap = System.Convert.ToInt32(args[++i]); } else { if (args[i].Equals("-numSpillsPerMap")) { numSpillsPerMap = System.Convert.ToInt32(args[++i]); } else { if (args[i].Equals("-numMapsPerHost")) { numMapsPerHost = System.Convert.ToInt32(args[++i]); } else { System.Console.Error.WriteLine(usage); System.Environment.Exit(-1); } } } } if (dataSizePerMap < 1 || numSpillsPerMap < 1 || numMapsPerHost < 1) { // verify arguments System.Console.Error.WriteLine(usage); System.Environment.Exit(-1); } FileSystem fs = null; try { // using random-writer to generate the input data GenerateInputData(dataSizePerMap, numSpillsPerMap, numMapsPerHost, masterConf); // configure job for sorting JobConf job = new JobConf(masterConf, typeof(ThreadedMapBenchmark)); job.SetJobName("threaded-map-benchmark-unspilled"); job.SetJarByClass(typeof(ThreadedMapBenchmark)); job.SetInputFormat(typeof(SortValidator.RecordStatsChecker.NonSplitableSequenceFileInputFormat )); job.SetOutputFormat(typeof(SequenceFileOutputFormat)); job.SetOutputKeyClass(typeof(BytesWritable)); job.SetOutputValueClass(typeof(BytesWritable)); job.SetMapperClass(typeof(IdentityMapper)); job.SetReducerClass(typeof(IdentityReducer)); FileInputFormat.AddInputPath(job, InputDir); FileOutputFormat.SetOutputPath(job, OutputDir); JobClient client = new JobClient(job); ClusterStatus cluster = client.GetClusterStatus(); job.SetNumMapTasks(numMapsPerHost * cluster.GetTaskTrackers()); job.SetNumReduceTasks(1); // set mapreduce.task.io.sort.mb to avoid spill int ioSortMb = (int)Math.Ceil(Factor * dataSizePerMap); job.Set(JobContext.IoSortMb, ioSortMb.ToString()); fs = FileSystem.Get(job); Log.Info("Running sort with 1 spill per map"); long startTime = Runtime.CurrentTimeMillis(); JobClient.RunJob(job); long endTime = Runtime.CurrentTimeMillis(); Log.Info("Total time taken : " + (endTime - startTime).ToString() + " millisec"); fs.Delete(OutputDir, true); // set mapreduce.task.io.sort.mb to have multiple spills JobConf spilledJob = new JobConf(job, typeof(ThreadedMapBenchmark)); ioSortMb = (int)Math.Ceil(Factor * Math.Ceil((double)dataSizePerMap / numSpillsPerMap )); spilledJob.Set(JobContext.IoSortMb, ioSortMb.ToString()); spilledJob.SetJobName("threaded-map-benchmark-spilled"); spilledJob.SetJarByClass(typeof(ThreadedMapBenchmark)); Log.Info("Running sort with " + numSpillsPerMap + " spills per map"); startTime = Runtime.CurrentTimeMillis(); JobClient.RunJob(spilledJob); endTime = Runtime.CurrentTimeMillis(); Log.Info("Total time taken : " + (endTime - startTime).ToString() + " millisec"); } finally { if (fs != null) { fs.Delete(BaseDir, true); } } return(0); }
private void CheckErrorDownloadImageProduct(ImageProductInfo imageProductInfo, JobClient downloadImageProductJobClient, JobClient historyJobClient) { //Check số lần download error trên Redis var errordownload = RedisErrorDownloadImageProductAdapter.GetErrorDownloadImage(imageProductInfo.Id); //errordownload < 5 resend message to service downloadimageproduct if (errordownload <= 5) { errordownload++; RedisErrorDownloadImageProductAdapter.SetErrorDownloadImage(imageProductInfo.Id, errordownload); ResendToServiceDownloadImageProduct(imageProductInfo, downloadImageProductJobClient); } else // send to service insert log history download image { SendMessageToServiceInsertHistoryDownload(imageProductInfo, historyJobClient); } }
/// <summary>This is the main routine for launching a distributed random write job.</summary> /// <remarks> /// This is the main routine for launching a distributed random write job. /// It runs 10 maps/node and each node writes 1 gig of data to a DFS file. /// The reduce doesn't do anything. /// </remarks> /// <exception cref="System.IO.IOException"></exception> /// <exception cref="System.Exception"/> public virtual int Run(string[] args) { if (args.Length == 0) { return(PrintUsage()); } Configuration conf = GetConf(); JobClient client = new JobClient(conf); ClusterStatus cluster = client.GetClusterStatus(); int numMapsPerHost = conf.GetInt(MapsPerHost, 10); long numBytesToWritePerMap = conf.GetLong(BytesPerMap, 1 * 1024 * 1024 * 1024); if (numBytesToWritePerMap == 0) { System.Console.Error.WriteLine("Cannot have " + BytesPerMap + " set to 0"); return(-2); } long totalBytesToWrite = conf.GetLong(TotalBytes, numMapsPerHost * numBytesToWritePerMap * cluster.GetTaskTrackers()); int numMaps = (int)(totalBytesToWrite / numBytesToWritePerMap); if (numMaps == 0 && totalBytesToWrite > 0) { numMaps = 1; conf.SetLong(BytesPerMap, totalBytesToWrite); } conf.SetInt(MRJobConfig.NumMaps, numMaps); Job job = Job.GetInstance(conf); job.SetJarByClass(typeof(RandomTextWriter)); job.SetJobName("random-text-writer"); job.SetOutputKeyClass(typeof(Org.Apache.Hadoop.IO.Text)); job.SetOutputValueClass(typeof(Org.Apache.Hadoop.IO.Text)); job.SetInputFormatClass(typeof(RandomWriter.RandomInputFormat)); job.SetMapperClass(typeof(RandomTextWriter.RandomTextMapper)); Type outputFormatClass = typeof(SequenceFileOutputFormat); IList <string> otherArgs = new AList <string>(); for (int i = 0; i < args.Length; ++i) { try { if ("-outFormat".Equals(args[i])) { outputFormatClass = Sharpen.Runtime.GetType(args[++i]).AsSubclass <OutputFormat>(); } else { otherArgs.AddItem(args[i]); } } catch (IndexOutOfRangeException) { System.Console.Out.WriteLine("ERROR: Required parameter missing from " + args[i - 1]); return(PrintUsage()); } } // exits job.SetOutputFormatClass(outputFormatClass); FileOutputFormat.SetOutputPath(job, new Path(otherArgs[0])); System.Console.Out.WriteLine("Running " + numMaps + " maps."); // reducer NONE job.SetNumReduceTasks(0); DateTime startTime = new DateTime(); System.Console.Out.WriteLine("Job started: " + startTime); int ret = job.WaitForCompletion(true) ? 0 : 1; DateTime endTime = new DateTime(); System.Console.Out.WriteLine("Job ended: " + endTime); System.Console.Out.WriteLine("The job took " + (endTime.GetTime() - startTime.GetTime ()) / 1000 + " seconds."); return(ret); }
private void SendMessageToServiceInsertHistoryDownload(ImageProductInfo imageProductInfo, JobClient historyJobClient) { var job = new Job { Data = LogHistoryImageProduct.GetMessage(new LogHistoryImageProduct() { ProductId = imageProductInfo.Id, DateLog = DateTime.Now, ErrorName = imageProductInfo.ErrorMessage, IsDownloaded = false, NewsToValid = false }) }; while (_isRunning) { try { historyJobClient.PublishJob(job); Log.Info(string.Format("Push message to services insert history download image productid = {0}", imageProductInfo.Id)); break; } catch (Exception ex) { Log.Error(string.Format("ProductId = {0} Push message to services insert history download image.", imageProductInfo.Id), ex); Thread.Sleep(60000); } } }