public void OpenCommandOpensSequence() { var sequenceEditor = A.Fake <ISequenceEditorService>(); var sequencePersistence = A.Fake <ISequencePersistence>(); var path = "Test Path"; var file = new SequenceFile(path, new SequenceViewModel(new Models.Sequence())); var viewModel = new LiveViewModel(A.Fake <ISequenceRunner>(), sequencePersistence, sequenceEditor, A.Fake <IDataAquisition>()); var openRequestRaised = false; viewModel.OpenRequest.Raised += (o, e) => { openRequestRaised = true; var notification = (FileInteractionNotification)e.Context; notification.Confirmed = true; notification.Path = path; e.Callback(); }; viewModel.OpenSequenceCommand.Execute(null); Assert.True(openRequestRaised); A.CallTo(() => sequencePersistence.OpenSequence(path)).MustHaveHappened(); A.CallTo(() => sequenceEditor.OpenSequenceInRegion(A <string> .Ignored, path, A <SequenceViewModel> .Ignored)).MustHaveHappened(); }
public virtual void Configure() { Path testdir = new Path(TestDir.GetAbsolutePath()); Path inDir = new Path(testdir, "in"); Path outDir = new Path(testdir, "out"); FileSystem fs = FileSystem.Get(conf); fs.Delete(testdir, true); conf.SetInt(JobContext.IoSortMb, 1); conf.SetInputFormat(typeof(SequenceFileInputFormat)); FileInputFormat.SetInputPaths(conf, inDir); FileOutputFormat.SetOutputPath(conf, outDir); conf.SetMapperClass(typeof(TestMapOutputType.TextGen)); conf.SetReducerClass(typeof(TestMapOutputType.TextReduce)); conf.SetOutputKeyClass(typeof(Text)); conf.SetOutputValueClass(typeof(Text)); conf.Set(MRConfig.FrameworkName, MRConfig.LocalFrameworkName); conf.SetOutputFormat(typeof(SequenceFileOutputFormat)); if (!fs.Mkdirs(testdir)) { throw new IOException("Mkdirs failed to create " + testdir.ToString()); } if (!fs.Mkdirs(inDir)) { throw new IOException("Mkdirs failed to create " + inDir.ToString()); } Path inFile = new Path(inDir, "part0"); SequenceFile.Writer writer = SequenceFile.CreateWriter(fs, conf, inFile, typeof(Text ), typeof(Text)); writer.Append(new Text("rec: 1"), new Text("Hello")); writer.Close(); jc = new JobClient(conf); }
/// <exception cref="System.IO.IOException"/> private static Path WritePartitionFile <T>(string testname, Configuration conf, T[] splits) where T : WritableComparable <object> { FileSystem fs = FileSystem.GetLocal(conf); Path testdir = new Path(Runtime.GetProperty("test.build.data", "/tmp")).MakeQualified (fs); Path p = new Path(testdir, testname + "/_partition.lst"); TotalOrderPartitioner.SetPartitionFile(conf, p); conf.SetInt(MRJobConfig.NumReduces, splits.Length + 1); SequenceFile.Writer w = null; try { w = SequenceFile.CreateWriter(fs, conf, p, splits[0].GetType(), typeof(NullWritable ), SequenceFile.CompressionType.None); for (int i = 0; i < splits.Length; ++i) { w.Append(splits[i], NullWritable.Get()); } } finally { if (null != w) { w.Close(); } } return(p); }
// public string dataGridView_ToolTipText = //@"Lists the File Properties written to the TDMS log file. // //For each property, the following information is stored: //Name (string) //Data type (tdsDataType) //Value (strings are encoded in UTF-8 Unicode). // //Strings in TDMS files can be null-terminated, but since the length //information is stored, the null terminator will be ignored when //you read from the file."; public AdvancedLoggingProperties(SequenceContext _seqContext, ConfigureLoggingFile CallingForm) { InitializeComponent(); CallingFormGlobal = CallingForm; //Set up Teststand objects seqContext = _seqContext; seqContextPO = seqContext.AsPropertyObject(); selectedTSSequence = seqContext.SelectedSequences[0]; selectedTSStep = seqContext.SelectedSteps[0]; stepID = selectedTSStep.UniqueStepId; seqFile = selectedTSSequence.SequenceFile; permSeqContext = selectedTSSequence.Locals; //Must get sequence context this way for variables to save if teststand is restarted propObjectFile = seqFile.AsPropertyObjectFile(); EvaluationTypes eval = seqContext.Engine.NewEvaluationTypes(); eval.PropertyValueTypeFlags = 0x4; //set string as the valid evaluation type EvaluationTypes evalNumeric = seqContext.Engine.NewEvaluationTypes(); stepPropertyObject = selectedTSSequence.GetStepByUniqueId(stepID).AsPropertyObject(); FileDescription_text.Text = CallingForm.fileDescription; filePropertyNames = CallingFormGlobal.filePropertyNames; filePropertyValues = CallingFormGlobal.filePropertyValues; InitializeFilePropertiesDataGridView(); VSDialogs vsdiag = new VSDialogs(); //this.OK.Click +=new EventHandler(OK_Click,new EventArgs(_fileAuthor)); }
public ConfigureRTSequence(SequenceContext _seqContext, ConcurrentQueue <SequenceCallInfo[]> _cq) { InitializeComponent(); seqContext = _seqContext; seqContextPO = seqContext.AsPropertyObject(); selectedTSSequence = seqContext.SelectedSequences[0]; selectedTSStep = seqContext.SelectedSteps[0]; stepID = selectedTSStep.UniqueStepId; seqFile = selectedTSSequence.SequenceFile; permSeqContext = selectedTSSequence.Locals; propObjectFile = seqFile.AsPropertyObjectFile(); VSDialogs vsdiag = new VSDialogs(); cq = _cq; selectedSequence = 0; stepPropertyObject = selectedTSSequence.GetStepByUniqueId(stepID).AsPropertyObject(); //Get old values if they exist try { autostart = stepPropertyObject.GetValBoolean("Veristand.StimProfAutoStart", 0); this.auto_start.Checked = autostart; timeout = (uint)stepPropertyObject.GetValNumber("Veristand.StimProfTimeout", 0); this.rtseq_timeout.Value = timeout; this.sessionName_Control.Text = stepPropertyObject.GetValString("Veristand.RTSessionName", 0); sequenceNames = Array.ConvertAll((object[])stepPropertyObject.GetValVariant("Veristand.SequenceNames", 0), o => o.ToString()); parameterValues = Array.ConvertAll((object[])stepPropertyObject.GetValVariant("Veristand.ParamValues", 0), o => o.ToString()); parameterTypes = Array.ConvertAll((object[])stepPropertyObject.GetValVariant("Veristand.ParamTypes", 0), o => o.ToString()); parameterNames = Array.ConvertAll((object[])stepPropertyObject.GetValVariant("Veristand.ParamNames", 0), o => o.ToString()); numSequences = (int)stepPropertyObject.GetValNumber("Veristand.RTNumSequences", 0); try { InitializeDataGridView(sequenceNames[0], parameterValues, parameterTypes, numSequences, 0); //Initialize the data grid with the previously selected data this.FilePath.Text = sequenceNames[0]; } catch (System.ArgumentException ex) { //Intentially do nothing } catch (IndexOutOfRangeException ex) { //no sequences, don't need to initialize the data grid } } catch (System.Runtime.InteropServices.COMException ex) { //Variables are not already created in TestStand. They will be created later vsdiag.ShowWarningDialog(ex.Message + "------" + ex.StackTrace); autostart = false; StimProfilePathStr = ""; this.auto_start.Checked = autostart; this.FilePath.Text = StimProfilePathStr; timeout = 1000; this.rtseq_timeout.Value = timeout; this.sessionName_Control.Text = "Session 1"; } }
public virtual void TestSequenceFileSync() { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build(); FileSystem fs = cluster.GetFileSystem(); Path p = new Path("/testSequenceFileSync/foo"); int len = 1 << 16; FSDataOutputStream @out = fs.Create(p, FsPermission.GetDefault(), EnumSet.Of(CreateFlag .Create, CreateFlag.Overwrite, CreateFlag.SyncBlock), 4096, (short)1, len, null); SequenceFile.Writer w = SequenceFile.CreateWriter(new Configuration(), SequenceFile.Writer .Stream(@out), SequenceFile.Writer.KeyClass(typeof(RandomDatum)), SequenceFile.Writer .ValueClass(typeof(RandomDatum)), SequenceFile.Writer.Compression(SequenceFile.CompressionType .None, new DefaultCodec())); w.Hflush(); CheckSyncMetric(cluster, 0); w.Hsync(); CheckSyncMetric(cluster, 1); int seed = new Random().Next(); RandomDatum.Generator generator = new RandomDatum.Generator(seed); generator.Next(); w.Append(generator.GetKey(), generator.GetValue()); w.Hsync(); CheckSyncMetric(cluster, 2); w.Close(); CheckSyncMetric(cluster, 2); @out.Close(); CheckSyncMetric(cluster, 3); cluster.Shutdown(); }
/// <exception cref="System.IO.IOException"/> private static void CreateControlFile(FileSystem fs, int fileSize, int nrFiles) { // in MB Log.Info("creating control file: " + fileSize + " mega bytes, " + nrFiles + " files" ); fs.Delete(ControlDir, true); for (int i = 0; i < nrFiles; i++) { string name = GetFileName(i); Path controlFile = new Path(ControlDir, "in_file_" + name); SequenceFile.Writer writer = null; try { writer = SequenceFile.CreateWriter(fs, fsConfig, controlFile, typeof(Text), typeof( LongWritable), SequenceFile.CompressionType.None); writer.Append(new Text(name), new LongWritable(fileSize)); } catch (Exception e) { throw new IOException(e.GetLocalizedMessage()); } finally { if (writer != null) { writer.Close(); } writer = null; } } Log.Info("created control files for: " + nrFiles + " files"); }
/// <exception cref="System.IO.IOException"/> private static void CreateFiles(int length, int numFiles, Random random, Job job) { TestCombineSequenceFileInputFormat.Range[] ranges = CreateRanges(length, numFiles , random); for (int i = 0; i < numFiles; i++) { Path file = new Path(workDir, "test_" + i + ".seq"); // create a file with length entries SequenceFile.Writer writer = SequenceFile.CreateWriter(localFs, job.GetConfiguration (), file, typeof(IntWritable), typeof(BytesWritable)); TestCombineSequenceFileInputFormat.Range range = ranges[i]; try { for (int j = range.start; j < range.end; j++) { IntWritable key = new IntWritable(j); byte[] data = new byte[random.Next(10)]; random.NextBytes(data); BytesWritable value = new BytesWritable(data); writer.Append(key, value); } } finally { writer.Close(); } } }
public void SaveAsUpdatesViewName() { var sequenceEditor = A.Fake <ISequenceEditorService>(); var path = "Test Path"; var file = new SequenceFile(path, new SequenceViewModel(new Models.Sequence())); var viewModel = new SequenceManagerViewModel(sequenceEditor, A.Fake <ISequencePersistence>()) { CurrentSequence = file }; var saveRequestRaised = false; viewModel.SaveRequest.Raised += (o, e) => { saveRequestRaised = true; ((IConfirmation)e.Context).Confirmed = true; e.Callback(); }; viewModel.SaveCommand.Execute(true.ToString()); Assert.True(saveRequestRaised); A.CallTo(() => sequenceEditor.UpdateViewNameForSequence(A <string> .Ignored, null, file)).MustHaveHappened(); }
/// <exception cref="System.Exception"/> public virtual void TestAppendSort() { GenericTestUtils.AssumeInNativeProfile(); Path file = new Path(RootPath, "testseqappendSort.seq"); fs.Delete(file, true); Path sortedFile = new Path(RootPath, "testseqappendSort.seq.sort"); fs.Delete(sortedFile, true); SequenceFile.Sorter sorter = new SequenceFile.Sorter(fs, new JavaSerializationComparator <long>(), typeof(long), typeof(string), conf); SequenceFile.Writer.Option compressOption = SequenceFile.Writer.Compression(SequenceFile.CompressionType .Block, new GzipCodec()); SequenceFile.Writer writer = SequenceFile.CreateWriter(conf, SequenceFile.Writer. File(file), SequenceFile.Writer.KeyClass(typeof(long)), SequenceFile.Writer.ValueClass (typeof(string)), compressOption); writer.Append(2L, "two"); writer.Append(1L, "one"); writer.Close(); writer = SequenceFile.CreateWriter(conf, SequenceFile.Writer.File(file), SequenceFile.Writer .KeyClass(typeof(long)), SequenceFile.Writer.ValueClass(typeof(string)), SequenceFile.Writer .AppendIfExists(true), compressOption); writer.Append(4L, "four"); writer.Append(3L, "three"); writer.Close(); // Sort file after append sorter.Sort(file, sortedFile); VerifyAll4Values(sortedFile); fs.DeleteOnExit(file); fs.DeleteOnExit(sortedFile); }
/// <summary>Create control files before a test run.</summary> /// <remarks> /// Create control files before a test run. /// Number of files created is equal to the number of maps specified /// </remarks> /// <exception cref="System.IO.IOException">on error</exception> private static void CreateControlFiles() { FileSystem tempFS = FileSystem.Get(config); Log.Info("Creating " + numberOfMaps + " control files"); for (int i = 0; i < numberOfMaps; i++) { string strFileName = "NNBench_Controlfile_" + i; Path filePath = new Path(new Path(baseDir, ControlDirName), strFileName); SequenceFile.Writer writer = null; try { writer = SequenceFile.CreateWriter(tempFS, config, filePath, typeof(Text), typeof( LongWritable), SequenceFile.CompressionType.None); writer.Append(new Text(strFileName), new LongWritable(0l)); } finally { if (writer != null) { writer.Close(); } } } }
public GetChannelsDialog(SequenceContext _seqContext, ChannelType _channelType) { InitializeComponent(); seqContext = _seqContext; seqContextPO = seqContext.AsPropertyObject(); selectedTSSequence = seqContext.SelectedSequences[0]; selectedTSStep = seqContext.SelectedSteps[0]; stepID = selectedTSStep.UniqueStepId; seqFile = selectedTSSequence.SequenceFile; permSeqContext = selectedTSSequence.Locals; //Must get sequence context this way for variables to save if teststand is restarted propObjectFile = seqFile.AsPropertyObjectFile(); stepPropertyObject = selectedTSSequence.GetStepByUniqueId(stepID).AsPropertyObject(); //Set up dialog objects sysDefPath = seqContext.SequenceFile.FileGlobalsDefaultValues.GetValString("Veristand.SystemDefinitionPath", 1); //Get the System Definition path for the Client Sequence File. Create it if it does not exist. seqContext.SequenceFile.FileGlobalsDefaultValues.SetFlags("Veristand.SystemDefinitionPath", 0, 0x4400000); channelNamesList.AddRange(stepPropertyObject.GetValVariant("VeriStand.ChannelNames", 0)); //Get ChannelNames array of strings. channelType = _channelType; VSDialogs vsdiag = new VSDialogs(); this.loggingChannelSelection.ShowCheckBox = true; //If the file at path FileGlobals.Veristand.SystemDefinitionPath exists and the extension is ".nivssdf" use that System Definition file to initialize the TreeAliasBrowserWF. if (System.IO.File.Exists(StringUtilities.unparseFilePathString(sysDefPath)) && System.IO.Path.GetExtension(StringUtilities.unparseFilePathString(sysDefPath)) == ".nivssdf") { //File exists with correct extension so try and populate the tree InitializeListBox(sysDefPath); } //If FileGlobals.Veristand.SystemDefinitionPath is empty or the file does not exist at path FileGlobals.Veristand.SystemDefinitionPath. else //(sysDefPath == "" || !System.IO.File.Exists(StringUtilities.unparseFilePathString(sysDefPath))) { //Do nothing } }
public NativeChannelSelectionDialog(SequenceContext _seqContext, ChannelType _channelType) { InitializeComponent(); channelType = _channelType; //Set up Teststand objects seqContext = _seqContext; seqContextPO = seqContext.AsPropertyObject(); selectedTSSequence = seqContext.SelectedSequences[0]; selectedTSStep = seqContext.SelectedSteps[0]; stepID = selectedTSStep.UniqueStepId; seqFile = selectedTSSequence.SequenceFile; permSeqContext = selectedTSSequence.Locals; //Must get sequence context this way for variables to save if teststand is restarted propObjectFile = seqFile.AsPropertyObjectFile(); EvaluationTypes eval = seqContext.Engine.NewEvaluationTypes(); eval.PropertyValueTypeFlags = 0x4; //set string as the valid evaluation type EvaluationTypes evalNumeric = seqContext.Engine.NewEvaluationTypes(); evalNumeric.PropertyValueTypeFlags = 0x2; //Set valid evaluation types to number value_exp.SetValidEvaluationTypes(evalNumeric); value_exp.Context = seqContextPO; stepPropertyObject = selectedTSSequence.GetStepByUniqueId(stepID).AsPropertyObject(); VSDialogs vsdiag = new VSDialogs(); try { value_exp.Text = stepPropertyObject.GetValString("Veristand.ValueToSet", 0); selectedChannelName = stepPropertyObject.GetValString("Veristand.ChannelName", 0); sysDefPath = seqContext.SequenceFile.FileGlobalsDefaultValues.GetValString("Veristand.SystemDefinitionPath", 1);//Try to get the specific system definition path associated with this step seqContext.SequenceFile.FileGlobalsDefaultValues.SetFlags("Veristand.SystemDefinitionPath", 0, 0x4400000); if (sysDefPath == null) { sysDefPath = ""; } if (channelType == ChannelType.paramChannel) { ModelListFullPath = stepPropertyObject.GetValString("Veristand.FullChannelPath", 0); } } catch (System.Runtime.InteropServices.COMException ex) { selectedChannelName = ""; } catch (System.NullReferenceException ex) { //Do Nothing } if (sysDefPath != null) { if (System.IO.File.Exists(StringUtilities.unparseFilePathString(sysDefPath)) && System.IO.Path.GetExtension(StringUtilities.unparseFilePathString(sysDefPath)) == ".nivssdf") { //File exists with correct extension so try and populate the tree InitializeListBox(sysDefPath); } } }
public ConfigureSequenceName(SequenceContext _seqContext) { InitializeComponent(); seqContext = _seqContext; selectedTSSequence = seqContext.SelectedSequences[0]; selectedTSStep = seqContext.SelectedSteps[0]; stepID = selectedTSStep.UniqueStepId; seqFile = selectedTSSequence.SequenceFile; seqContextPO = selectedTSSequence.Locals; int thisStepIndex = selectedTSStep.StepIndex; propObjectFile = seqFile.AsPropertyObjectFile(); try { this.sequenceName_Control.Text = seqContextPO.GetValString("Veristand." + stepID + "RTSequenceName", 0); } catch (System.Runtime.InteropServices.COMException) { //If variables don't exist set default values this.sequenceName_Control.Text = ""; } int count = thisStepIndex; bool found = false; while (count >= 0 && found == false) { Step currentStep = selectedTSSequence.GetStep(count, StepGroups.StepGroup_Main); if (currentStep.Name == "Open and Deploy RT Sequence") //This will need to change if the name of the Open RT Sequence Name Step Changes { //this is the open RTSequence step that is the closes previous to this step name configuration string openRTSequenceStepID = currentStep.UniqueStepId; VSDialogs vsdiags = new VSDialogs(); try //When this does and does not exist is different from the RTSequenceName so it needs its own Try/Catch block { seqCallInfoArray = vsdiags.ReinitializeSequenceCallInfo(seqContext, openRTSequenceStepID); //Get the sequence info created by the Open RT Sequence Step allSequenceNames = ParseNamesFromSequenceCallInfo(seqCallInfoArray); //Populate the parsed list of sequence names if (allSequenceNames.Length > 0) { if (allSequenceNames[0] != null) { //Set the sequence name control to the first name we parsed this.sequenceName_Control.Text = allSequenceNames[0]; } else { this.sequenceName_Control.Text = ""; } } } catch (System.Runtime.InteropServices.COMException) { seqCallInfoArray = null; } } count--; } }
void OnCreateSeqFile() { //ISequenceFile seqFile = SequenceFileFactory.CreateSequenceFile(); // for test SequenceFile seqFile = GenerateTestSequenceFile(); SeqFileVM = new SequenceFileVM(seqFile); }
public void SaveSequenceFileTest() { SequenceFile seqFile = GenerateTestSequenceFile(); string filePath = "TestSequenceFile.xml"; bool result = SequenceFileFactory.SaveSequenceFile(seqFile, filePath); Assert.AreEqual(true, result); Assert.AreEqual(true, File.Exists(filePath)); }
/// <summary> /// Test that makes sure createWriter succeeds on a file that was /// already created /// </summary> /// <exception cref="System.IO.IOException"/> public virtual void TestCreateWriterOnExistingFile() { Configuration conf = new Configuration(); FileSystem fs = FileSystem.GetLocal(conf); Path name = new Path(new Path(Runtime.GetProperty("test.build.data", "."), "createWriterOnExistingFile" ), "file"); fs.Create(name); SequenceFile.CreateWriter(fs, conf, name, typeof(RandomDatum), typeof(RandomDatum ), 512, (short)1, 4096, false, SequenceFile.CompressionType.None, null, new SequenceFile.Metadata ()); }
/// <exception cref="System.IO.IOException"/> private void WriteSkippedRec(KEY key, VALUE value) { if (this.skipWriter == null) { Path skipDir = SkipBadRecords.GetSkipOutputPath(this._enclosing.conf); Path skipFile = new Path(skipDir, this._enclosing.GetTaskID().ToString()); this.skipWriter = SequenceFile.CreateWriter(skipFile.GetFileSystem(this._enclosing .conf), this._enclosing.conf, skipFile, this.keyClass, this.valClass, SequenceFile.CompressionType .Block, this.reporter); } this.skipWriter.Append(key, value); }
public ChannelSelectionDialog(SequenceContext _seqContext, bool _ModelList) { ModelList = _ModelList; InitializeComponent(); //Set up Teststand objects seqContext = _seqContext; seqContextPO = seqContext.AsPropertyObject(); selectedTSSequence = seqContext.SelectedSequences[0]; selectedTSStep = seqContext.SelectedSteps[0]; stepID = selectedTSStep.UniqueStepId; seqFile = selectedTSSequence.SequenceFile; permSeqContext = selectedTSSequence.Locals; //Must get sequence context this way for variables to save if teststand is restarted propObjectFile = seqFile.AsPropertyObjectFile(); stepPropertyObject = selectedTSSequence.GetStepByUniqueId(stepID).AsPropertyObject(); try { channelValue = stepPropertyObject.GetValNumber("Veristand.ValueToSet", 0); selectedChannelName = stepPropertyObject.GetValString("Veristand.ChannelName", 0); SystemDefinitionPath = seqContext.SequenceFile.FileGlobalsDefaultValues.GetValString("Veristand.SystemDefinitionPath", 1);//Try to get the specific system definition path associated with this step seqContext.SequenceFile.FileGlobalsDefaultValues.SetFlags("Veristand.SystemDefinitionPath", 0, 0x4400000); this.SystemDefinitionPathControl.Text = SystemDefinitionPath; this.set_value_control.Value = (decimal)channelValue; } catch (System.Runtime.InteropServices.COMException) { selectedChannelName = ""; SystemDefinitionPath = ""; //If there is no system definition path default to the empty string this.SystemDefinitionPathControl.Text = SystemDefinitionPath; } if (System.IO.File.Exists(SystemDefinitionPath) && System.IO.Path.GetExtension(SystemDefinitionPath) == ".nivssdf") { //File exists with correct extension so try and populate the grid TreeNodeCollection tempTreeNodeCollection = InitializeListBox(SystemDefinitionPath); treeNodes = new TreeNode[tempTreeNodeCollection.Count]; tempTreeNodeCollection.CopyTo(treeNodes, 0); } if (selectedChannelName != "") { TreeNode[] foundNodes = this.ChannelSelection.Nodes.Find(selectedChannelName, true); if (foundNodes.Length > 0) { this.ChannelSelection.SelectedNode = foundNodes[0];//Only one node should match exactly this.ChannelSelection.HideSelection = false; this.ChannelSelection.SelectedNode.BackColor = Color.Violet; this.ChannelSelection.Focus();//This does not seem to be working } } }
/// <summary>Test that makes sure the FileSystem passed to createWriter</summary> /// <exception cref="System.Exception"/> public virtual void TestCreateUsesFsArg() { FileSystem fs = FileSystem.GetLocal(conf); FileSystem spyFs = Org.Mockito.Mockito.Spy(fs); Path p = new Path(Runtime.GetProperty("test.build.data", ".") + "/testCreateUsesFSArg.seq" ); SequenceFile.Writer writer = SequenceFile.CreateWriter(spyFs, conf, p, typeof(NullWritable ), typeof(NullWritable)); writer.Close(); Org.Mockito.Mockito.Verify(spyFs).GetDefaultReplication(p); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Sharpen.TimeoutException"/> /// <exception cref="System.Exception"/> internal static void WriteFile(NameNode namenode, Configuration conf, Path name, short replication) { FileSystem fileSys = FileSystem.Get(conf); SequenceFile.Writer writer = SequenceFile.CreateWriter(fileSys, conf, name, typeof( BytesWritable), typeof(BytesWritable), SequenceFile.CompressionType.None); writer.Append(new BytesWritable(), new BytesWritable()); writer.Close(); fileSys.SetReplication(name, replication); DFSTestUtil.WaitReplication(fileSys, name, replication); }
// for Test SequenceFile GenerateTestSequenceFile() { SequenceFile seqFile = new SequenceFile(); seqFile.Name = "TestSequenceFile"; seqFile.Description = "This is a test sequenceFile."; seqFile.Comment = "Nothing"; seqFile.Version.MarjorVersion = "1"; ISequence mainSequence = SequenceFactory.CreateSequence(SequenceTypes.Normal); mainSequence.Name = "MainSequence1"; mainSequence.Description = "Main sequence for test."; mainSequence.EnableLogging = false; mainSequence.TestTimeout = 3000; mainSequence.BreakPoint = true; IStep actionStep = StepFactory.CreateStep(StepTypes.Action); actionStep.Name = "Action step test"; actionStep.Description = "this is a test action step"; IAdaptor adaptor = AdaptorFactory.CreateAdaptor(AdaptorTypes.DotnetAdaptor); adaptor.MethodName = "Test"; adaptor.TestModuleName = "DotNetTest.dll"; adaptor.Parameters.Add(new DotNetParameter()); adaptor.Parameters.Add(new DotNetParameter("parameter1")); adaptor.Parameters.Add(new DotNetParameter("parameter2")); actionStep.Adaptor = adaptor; IStep subActionStep = StepFactory.CreateStep(StepTypes.Action); subActionStep.Name = "SubAction step test"; subActionStep.Description = "this is a sub test action step"; actionStep.Children.Add(subActionStep); mainSequence.Children.Add(actionStep); mainSequence.Children.Add(subActionStep); mainSequence.Children.Add(subActionStep); ISequence mainSequence2 = SequenceFactory.CreateSequence(SequenceTypes.Normal); mainSequence2.Name = "MainSequence2"; mainSequence2.Description = "Main sequence 2 for test."; mainSequence2.EnableLogging = false; mainSequence2.TestTimeout = 3000; mainSequence2.BreakPoint = true; seqFile.Sequences.Add(mainSequence); seqFile.Sequences.Add(mainSequence2); return(seqFile); }
public ConfigureSystemDefinition(SequenceContext _seqContext, bool _deploying) { InitializeComponent(); //Set up Teststand objects deploying = _deploying; seqContext = _seqContext; seqContextPO = seqContext.AsPropertyObject(); selectedTSSequence = seqContext.SelectedSequences[0]; selectedTSStep = seqContext.SelectedSteps[0]; stepID = selectedTSStep.UniqueStepId; seqFile = selectedTSSequence.SequenceFile; permSeqContext = selectedTSSequence.Locals; //Must get sequence context this way for variables to save if teststand is restarted propObjectFile = seqFile.AsPropertyObjectFile(); EvaluationTypes eval = seqContext.Engine.NewEvaluationTypes(); eval.PropertyValueTypeFlags = 0x4; EvaluationTypes evalBool = seqContext.Engine.NewEvaluationTypes(); evalBool.PropertyValueTypeFlags = 0x1;//Booleans are valid deploysysdef_exp.SetValidEvaluationTypes(evalBool); systemDefinitionPath_exp.SetValidEvaluationTypes(eval); deploysysdef_exp.Context = seqContextPO; systemDefinitionPath_exp.Context = seqContextPO; stepPropertyObject = selectedTSSequence.GetStepByUniqueId(stepID).AsPropertyObject(); if (!deploying) { this.systemDefinitionPath_exp.Visible = false; this.SystemDefinitionPathLabel.Visible = false; this.Browse.Visible = false; } //Get old variable values if they exist try { deploysysdef_exp.Text = stepPropertyObject.GetValString("Veristand.DeploySystemDefinition", 0); if (deploying) { SystemDefinitionPath = seqContext.SequenceFile.FileGlobalsDefaultValues.GetValString("Veristand.SystemDefinitionPath", 1); seqContext.SequenceFile.FileGlobalsDefaultValues.SetFlags("Veristand.SystemDefinitionPath", 0, 0x4400000); this.systemDefinitionPath_exp.Text = SystemDefinitionPath; } } catch (System.Runtime.InteropServices.COMException) { //Variables are not already created. They will be created later deploysysdef_exp.Text = "True"; SystemDefinitionPath = ""; this.systemDefinitionPath_exp.Text = SystemDefinitionPath; } }
public virtual void TestNullKeys() { JobConf conf = new JobConf(typeof(TestMapRed)); FileSystem fs = FileSystem.GetLocal(conf); HashSet <string> values = new HashSet <string>(); string m = "AAAAAAAAAAAAAA"; for (int i = 1; i < 11; ++i) { values.AddItem(m); m = m.Replace((char)('A' + i - 1), (char)('A' + i)); } Path testdir = new Path(Runtime.GetProperty("test.build.data", "/tmp")).MakeQualified (fs); fs.Delete(testdir, true); Path inFile = new Path(testdir, "nullin/blah"); SequenceFile.Writer w = SequenceFile.CreateWriter(fs, conf, inFile, typeof(NullWritable ), typeof(Text), SequenceFile.CompressionType.None); Text t = new Text(); foreach (string s in values) { t.Set(s); w.Append(NullWritable.Get(), t); } w.Close(); FileInputFormat.SetInputPaths(conf, inFile); FileOutputFormat.SetOutputPath(conf, new Path(testdir, "nullout")); conf.SetMapperClass(typeof(TestMapRed.NullMapper)); conf.SetReducerClass(typeof(IdentityReducer)); conf.SetOutputKeyClass(typeof(NullWritable)); conf.SetOutputValueClass(typeof(Text)); conf.SetInputFormat(typeof(SequenceFileInputFormat)); conf.SetOutputFormat(typeof(SequenceFileOutputFormat)); conf.SetNumReduceTasks(1); conf.Set(MRConfig.FrameworkName, MRConfig.LocalFrameworkName); JobClient.RunJob(conf); // Since null keys all equal, allow any ordering SequenceFile.Reader r = new SequenceFile.Reader(fs, new Path(testdir, "nullout/part-00000" ), conf); m = "AAAAAAAAAAAAAA"; for (int i_1 = 1; r.Next(NullWritable.Get(), t); ++i_1) { NUnit.Framework.Assert.IsTrue("Unexpected value: " + t, values.Remove(t.ToString( ))); m = m.Replace((char)('A' + i_1 - 1), (char)('A' + i_1)); } NUnit.Framework.Assert.IsTrue("Missing values: " + values.ToString(), values.IsEmpty ()); }
/// <summary>Reduce task done, write output to a file.</summary> /// <exception cref="System.IO.IOException"/> protected override void Cleanup(Reducer.Context context) { //write output to a file Configuration conf = context.GetConfiguration(); Path outDir = new Path(conf.Get(FileOutputFormat.Outdir)); Path outFile = new Path(outDir, "reduce-out"); FileSystem fileSys = FileSystem.Get(conf); SequenceFile.Writer writer = SequenceFile.CreateWriter(fileSys, conf, outFile, typeof( LongWritable), typeof(LongWritable), SequenceFile.CompressionType.None); writer.Append(new LongWritable(numInside), new LongWritable(numOutside)); writer.Close(); }
// should succeed, fails if exception thrown /// <exception cref="System.IO.IOException"/> public virtual void TestSerializationAvailability() { Configuration conf = new Configuration(); Path path = new Path(Runtime.GetProperty("test.build.data", "."), "serializationAvailability" ); // Check if any serializers aren't found. try { SequenceFile.CreateWriter(conf, SequenceFile.Writer.File(path), SequenceFile.Writer .KeyClass(typeof(string)), SequenceFile.Writer.ValueClass(typeof(NullWritable))); // Note: This may also fail someday if JavaSerialization // is activated by default. Fail("Must throw IOException for missing serializer for the Key class"); } catch (IOException e) { Assert.True(e.Message.StartsWith("Could not find a serializer for the Key class: '" + typeof(string).FullName + "'.")); } try { SequenceFile.CreateWriter(conf, SequenceFile.Writer.File(path), SequenceFile.Writer .KeyClass(typeof(NullWritable)), SequenceFile.Writer.ValueClass(typeof(string))); // Note: This may also fail someday if JavaSerialization // is activated by default. Fail("Must throw IOException for missing serializer for the Value class"); } catch (IOException e) { Assert.True(e.Message.StartsWith("Could not find a serializer for the Value class: '" + typeof(string).FullName + "'.")); } // Write a simple file to test deserialization failures with WriteTest(FileSystem.Get(conf), 1, 1, path, SequenceFile.CompressionType.None, null ); // Remove Writable serializations, to enforce error. conf.SetStrings(CommonConfigurationKeys.IoSerializationsKey, typeof(AvroReflectSerialization ).FullName); // Now check if any deserializers aren't found. try { new SequenceFile.Reader(conf, SequenceFile.Reader.File(path)); Fail("Must throw IOException for missing deserializer for the Key class"); } catch (IOException e) { Assert.True(e.Message.StartsWith("Could not find a deserializer for the Key class: '" + typeof(RandomDatum).FullName + "'.")); } }
public virtual void RunJob(int items) { try { JobConf conf = new JobConf(typeof(TestMapRed)); Path testdir = new Path(TestDir.GetAbsolutePath()); Path inDir = new Path(testdir, "in"); Path outDir = new Path(testdir, "out"); FileSystem fs = FileSystem.Get(conf); fs.Delete(testdir, true); conf.SetInt(JobContext.IoSortMb, 1); conf.SetInputFormat(typeof(SequenceFileInputFormat)); FileInputFormat.SetInputPaths(conf, inDir); FileOutputFormat.SetOutputPath(conf, outDir); conf.SetMapperClass(typeof(IdentityMapper)); conf.SetReducerClass(typeof(IdentityReducer)); conf.SetOutputKeyClass(typeof(Text)); conf.SetOutputValueClass(typeof(Text)); conf.SetOutputFormat(typeof(SequenceFileOutputFormat)); conf.Set(MRConfig.FrameworkName, MRConfig.LocalFrameworkName); if (!fs.Mkdirs(testdir)) { throw new IOException("Mkdirs failed to create " + testdir.ToString()); } if (!fs.Mkdirs(inDir)) { throw new IOException("Mkdirs failed to create " + inDir.ToString()); } Path inFile = new Path(inDir, "part0"); SequenceFile.Writer writer = SequenceFile.CreateWriter(fs, conf, inFile, typeof(Text ), typeof(Text)); StringBuilder content = new StringBuilder(); for (int i = 0; i < 1000; i++) { content.Append(i).Append(": This is one more line of content\n"); } Org.Apache.Hadoop.IO.Text text = new Org.Apache.Hadoop.IO.Text(content.ToString() ); for (int i_1 = 0; i_1 < items; i_1++) { writer.Append(new Org.Apache.Hadoop.IO.Text("rec:" + i_1), text); } writer.Close(); JobClient.RunJob(conf); } catch (Exception e) { NUnit.Framework.Assert.IsTrue("Threw exception:" + e, false); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.TypeLoadException"/> /// <exception cref="InstantiationException"/> /// <exception cref="System.MemberAccessException"/> private static void SequenceFileCodecTest(Configuration conf, int lines, string codecClass , int blockSize) { Path filePath = new Path("SequenceFileCodecTest." + codecClass); // Configuration conf.SetInt("io.seqfile.compress.blocksize", blockSize); // Create the SequenceFile FileSystem fs = FileSystem.Get(conf); Log.Info("Creating SequenceFile with codec \"" + codecClass + "\""); SequenceFile.Writer writer = SequenceFile.CreateWriter(fs, conf, filePath, typeof( Text), typeof(Text), SequenceFile.CompressionType.Block, (CompressionCodec)System.Activator.CreateInstance (Runtime.GetType(codecClass))); // Write some data Log.Info("Writing to SequenceFile..."); for (int i = 0; i < lines; i++) { Text key = new Text("key" + i); Text value = new Text("value" + i); writer.Append(key, value); } writer.Close(); // Read the data back and check Log.Info("Reading from the SequenceFile..."); SequenceFile.Reader reader = new SequenceFile.Reader(fs, filePath, conf); Writable key_1 = (Writable)System.Activator.CreateInstance(reader.GetKeyClass()); Writable value_1 = (Writable)System.Activator.CreateInstance(reader.GetValueClass ()); int lc = 0; try { while (reader.Next(key_1, value_1)) { Assert.Equal("key" + lc, key_1.ToString()); Assert.Equal("value" + lc, value_1.ToString()); lc++; } } finally { reader.Close(); } Assert.Equal(lines, lc); // Delete temporary files fs.Delete(filePath, false); Log.Info("SUCCESS! Completed SequenceFileCodecTest with codec \"" + codecClass + "\""); }
/// <exception cref="System.IO.IOException"/> public virtual void TestClose() { Configuration conf = new Configuration(); LocalFileSystem fs = FileSystem.GetLocal(conf); // create a sequence file 1 Path path1 = new Path(Runtime.GetProperty("test.build.data", ".") + "/test1.seq"); SequenceFile.Writer writer = SequenceFile.CreateWriter(fs, conf, path1, typeof(Text ), typeof(NullWritable), SequenceFile.CompressionType.Block); writer.Append(new Text("file1-1"), NullWritable.Get()); writer.Append(new Text("file1-2"), NullWritable.Get()); writer.Close(); Path path2 = new Path(Runtime.GetProperty("test.build.data", ".") + "/test2.seq"); writer = SequenceFile.CreateWriter(fs, conf, path2, typeof(Text), typeof(NullWritable ), SequenceFile.CompressionType.Block); writer.Append(new Text("file2-1"), NullWritable.Get()); writer.Append(new Text("file2-2"), NullWritable.Get()); writer.Close(); // Create a reader which uses 4 BuiltInZLibInflater instances SequenceFile.Reader reader = new SequenceFile.Reader(fs, path1, conf); // Returns the 4 BuiltInZLibInflater instances to the CodecPool reader.Close(); // The second close _could_ erroneously returns the same // 4 BuiltInZLibInflater instances to the CodecPool again reader.Close(); // The first reader gets 4 BuiltInZLibInflater instances from the CodecPool SequenceFile.Reader reader1 = new SequenceFile.Reader(fs, path1, conf); // read first value from reader1 Text text = new Text(); reader1.Next(text); Assert.Equal("file1-1", text.ToString()); // The second reader _could_ get the same 4 BuiltInZLibInflater // instances from the CodePool as reader1 SequenceFile.Reader reader2 = new SequenceFile.Reader(fs, path2, conf); // read first value from reader2 reader2.Next(text); Assert.Equal("file2-1", text.ToString()); // read second value from reader1 reader1.Next(text); Assert.Equal("file1-2", text.ToString()); // read second value from reader2 (this throws an exception) reader2.Next(text); Assert.Equal("file2-2", text.ToString()); NUnit.Framework.Assert.IsFalse(reader1.Next(text)); NUnit.Framework.Assert.IsFalse(reader2.Next(text)); }
private void OK_Click(object sender, EventArgs e) { try { BaseNodeType[] selections = this._aliasBrowser.GetCheckBoxSelections(false) .Concat(this._treeView.GetCheckBoxSelections(false)) .ToArray(); channelNamesList.Clear(); if (selections.Length > 0) { channelNamesList.AddRange(from selection in selections where selection is NationalInstruments.VeriStand.SystemStorage.ChannelType || selection is AliasType select selection.NodePath); stepPropertyObject.SetValVariant("VeriStand.ChannelNames", 0, channelNamesList.ToArray()); baseNodeArray = selections; stepPropertyObject.SetValInterface("VeriStand.BaseNodeArray", 0, baseNodeArray); } //sysDefPath; } catch (System.NullReferenceException ex) { //do nothing } catch (System.IndexOutOfRangeException ex) { //do nothing } seqContext.SequenceFile.FileGlobalsDefaultValues.SetValString("Veristand.SystemDefinitionPath", 1, sysDefPath); seqContext.SequenceFile.FileGlobalsDefaultValues.SetFlags("Veristand.SystemDefinitionPath", 0, 0x4400000); propObjectFile.IncChangeCount(); //Sets the flag that means the sequence has changes to save (dirty dot*) System.Runtime.InteropServices.Marshal.ReleaseComObject(seqContext); System.Runtime.InteropServices.Marshal.ReleaseComObject(seqContextPO); System.Runtime.InteropServices.Marshal.ReleaseComObject(selectedTSSequence); System.Runtime.InteropServices.Marshal.ReleaseComObject(seqFile); System.Runtime.InteropServices.Marshal.ReleaseComObject(permSeqContext); System.Runtime.InteropServices.Marshal.ReleaseComObject(selectedTSStep); System.Runtime.InteropServices.Marshal.ReleaseComObject(propObjectFile); System.Runtime.InteropServices.Marshal.ReleaseComObject(stepPropertyObject); seqContext = null; seqContextPO = null; selectedTSSequence = null; seqFile = null; permSeqContext = null; selectedTSStep = null; propObjectFile = null; stepPropertyObject = null; this.Close(); //Close the form }