public static InputFormat newPresegmentedTokenizer(InputFormat DISOWN) { global::System.IntPtr cPtr = udpipe_csharpPINVOKE.InputFormat_newPresegmentedTokenizer(InputFormat.getCPtrAndDisown(DISOWN)); InputFormat ret = (cPtr == global::System.IntPtr.Zero) ? null : new InputFormat(cPtr, true); return(ret); }
/// <summary>From each split sampled, take the first numSamples / numSplits records.</summary> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public virtual K[] GetSample(InputFormat <K, V> inf, Job job) { // ArrayList::toArray doesn't preserve type IList <InputSplit> splits = inf.GetSplits(job); AList <K> samples = new AList <K>(numSamples); int splitsToSample = Math.Min(maxSplitsSampled, splits.Count); int samplesPerSplit = numSamples / splitsToSample; long records = 0; for (int i = 0; i < splitsToSample; ++i) { TaskAttemptContext samplingContext = new TaskAttemptContextImpl(job.GetConfiguration (), new TaskAttemptID()); RecordReader <K, V> reader = inf.CreateRecordReader(splits[i], samplingContext); reader.Initialize(splits[i], samplingContext); while (reader.NextKeyValue()) { samples.AddItem(ReflectionUtils.Copy(job.GetConfiguration(), reader.GetCurrentKey (), null)); ++records; if ((i + 1) * samplesPerSplit <= records) { break; } } reader.Close(); } return((K[])Sharpen.Collections.ToArray(samples)); }
/// <summary> /// Process the file and output to the target file. /// </summary> /// <param name="target">The target file to write to.</param> public void Process(string target) { var csv = new ReadCSV(InputFilename.ToString(), ExpectInputHeaders, InputFormat); TextWriter tw = new StreamWriter(target); ResetStatus(); while (csv.Next()) { var line = new StringBuilder(); UpdateStatus(false); line.Append(GetColumnData(FileData.Date, csv)); line.Append(" "); line.Append(GetColumnData(FileData.Time, csv)); line.Append(";"); line.Append(InputFormat.Format(double.Parse(GetColumnData(FileData.Open, csv)), Precision)); line.Append(";"); line.Append(InputFormat.Format(double.Parse(GetColumnData(FileData.High, csv)), Precision)); line.Append(";"); line.Append(InputFormat.Format(double.Parse(GetColumnData(FileData.Low, csv)), Precision)); line.Append(";"); line.Append(InputFormat.Format(double.Parse(GetColumnData(FileData.Close, csv)), Precision)); line.Append(";"); line.Append(InputFormat.Format(double.Parse(GetColumnData(FileData.Volume, csv)), Precision)); tw.WriteLine(line.ToString()); } ReportDone(false); csv.Close(); tw.Close(); }
private static void Process(InputFormat inputFormat, OutputFormat outputFormat, string fromCode, string fromList, VjudgeOverflowBehavior overflowBehavior, int penaltyTime) { if (!string.IsNullOrEmpty(fromCode) && !string.IsNullOrEmpty(fromList)) { Console.WriteLine("'--fromCode' and '--fromList' can not be simultaneously set!"); return; } if (string.IsNullOrEmpty(fromCode) && string.IsNullOrEmpty(fromList)) { Console.WriteLine("At least one of '--fromCode' and '--fromList' must be set!"); return; } if (!string.IsNullOrEmpty(fromCode)) { Process(inputFormat, outputFormat, fromCode, overflowBehavior, penaltyTime); } else { var lines = File.ReadAllLines(fromList); foreach (var line in lines) { Process(inputFormat, outputFormat, line, overflowBehavior, penaltyTime); } } }
/// <summary>From each split sampled, take the first numSamples / numSplits records.</summary> /// <exception cref="System.IO.IOException"/> public virtual K[] GetSample(InputFormat <K, V> inf, JobConf job) { // ArrayList::toArray doesn't preserve type InputSplit[] splits = inf.GetSplits(job, job.GetNumMapTasks()); AList <K> samples = new AList <K>(numSamples); int splitsToSample = Math.Min(maxSplitsSampled, splits.Length); int splitStep = splits.Length / splitsToSample; int samplesPerSplit = numSamples / splitsToSample; long records = 0; for (int i = 0; i < splitsToSample; ++i) { RecordReader <K, V> reader = inf.GetRecordReader(splits[i * splitStep], job, Reporter .Null); K key = reader.CreateKey(); V value = reader.CreateValue(); while (reader.Next(key, value)) { samples.AddItem(key); key = reader.CreateKey(); ++records; if ((i + 1) * samplesPerSplit <= records) { break; } } reader.Close(); } return((K[])Sharpen.Collections.ToArray(samples)); }
public Dominomics(MultipleDirectorySelector mds, InputFormat inputF) { Progress = 0; MyInputFormat = inputF; //Read all SEPro Files to obtain a dictionary of ID (key) and fasta (value) and peptide (key) SeproFiles (values) locusFastaDict = new Dictionary <string, string>(); myResultPackages = new List <FileInfoResultPackage>(); matrixClassDescriptionDictionary = new Dictionary <int, string>(); foreach (DirectoryClassDescription myDir in mds.MyDirectoryDescriptionDictionary) { if (inputF == InputFormat.SEPro) { FileInfo[] resultFilteredFiles = new DirectoryInfo(myDir.MyDirectoryFullName).GetFiles("*.sepr", SearchOption.AllDirectories); FeedLocusFastaDictSEPro(myDir, resultFilteredFiles); } else if (inputF == InputFormat.MPex) { FileInfo[] resultFilteredFiles = new DirectoryInfo(myDir.MyDirectoryFullName).GetFiles("*.mpex", SearchOption.AllDirectories); FeedLocusFastaDictPex(myDir, resultFilteredFiles); } matrixClassDescriptionDictionary.Add(myDir.ClassLabel, myDir.Description); } bdf = new HMMVerifier.BatchDomainFetcher(); }
/// <summary> /// Prints keys and values from <see cref="RocksDbSharp.RocksDb"/> located at <paramref name="rocksdbPath"/>. /// </summary> /// <param name="prefix">The prefix of keys to filter.</param> /// <param name="inputFormat">An format of <paramref name="prefix"/>. /// For instance, if <paramref name="inputFormat"/> is <see cref="InputFormat.Base64"/>, <paramref name="prefix"/> /// should be base64 encoded value. If <paramref name="inputFormat"/> is <see cref="InputFormat.String"/>, /// it treats <paramref name="prefix"/> as <see cref="Encoding.UTF8"/> encoded string.</param> /// <param name="outputFormat">A format of output (i.e. keys and values).</param> /// <param name="rocksdbPath">The path of <see cref="RocksDbSharp.RocksDb"/> to load.</param> public void List( [Option] string prefix = "", [Option] InputFormat inputFormat = InputFormat.String, [Option] OutputFormat outputFormat = OutputFormat.Hex, [Option] string?rocksdbPath = null) { rocksdbPath ??= Directory.GetCurrentDirectory(); try { using var db = _rocksDbService.Load(rocksdbPath); byte[] prefixBytes = inputFormat.Decode(prefix); _inputOutputErrorContainer.Error.WriteLine("Key\tValue"); using var iterator = db.NewIterator(); for (iterator.Seek(prefixBytes); iterator.Valid() && iterator.Key().Take(prefixBytes.Length).SequenceEqual(prefixBytes); iterator.Next()) { _inputOutputErrorContainer.Out.Write(outputFormat.Encode(iterator.Key())); _inputOutputErrorContainer.Out.Write('\t'); _inputOutputErrorContainer.Out.WriteLine(outputFormat.Encode(iterator.Value())); } } catch (Exception e) { _inputOutputErrorContainer.Error.WriteLine(e.Message); } }
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: //ORIGINAL LINE: @Test(expected = MediaTypeNotSupportedException.class) public void canNotGetInputFormatBasedOnWildcardMediaType() throws Exception //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: public virtual void CanNotGetInputFormatBasedOnWildcardMediaType() { InputFormat format = _repository.inputFormat(MediaType.WILDCARD_TYPE); format.ReadValue("foo"); fail("Got InputFormat based on wild card type: " + format); }
public static BomFormat InputFormatHelper(string inputFile, InputFormat inputFormat) { if (inputFormat == InputFormat.autodetect) { if (string.IsNullOrEmpty(inputFile)) { Console.Error.WriteLine("Unable to auto-detect input stream format, please specify a value for --input-format"); } var inputBomFormat = CLIUtils.DetectFileFormat(inputFile); if (inputBomFormat == BomFormat.Unsupported) { Console.Error.WriteLine("Unable to auto-detect input format from input filename"); } return(inputBomFormat); } else { if (inputFormat == InputFormat.json) { return(BomFormat.Json); } else if (inputFormat == InputFormat.xml) { return(BomFormat.Xml); } else if (inputFormat == InputFormat.csv) { return(BomFormat.Csv); } } return(BomFormat.Unsupported); }
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: //ORIGINAL LINE: public ExtensionService(@Context InputFormat input, @Context OutputFormat output, @Context PluginInvocator extensions, @Context Database database) public ExtensionService(InputFormat input, OutputFormat output, PluginInvocator extensions, Database database) { this._input = input; this._output = output; this._extensions = extensions; this._graphDb = database.Graph; }
public static InputFormat newVerticalInputFormat() { global::System.IntPtr cPtr = udpipe_csharpPINVOKE.InputFormat_newVerticalInputFormat__SWIG_1(); InputFormat ret = (cPtr == global::System.IntPtr.Zero) ? null : new InputFormat(cPtr, true); return(ret); }
/// <exception cref="System.IO.IOException"/> internal static long ReadBench(JobConf conf) { // InputFormat instantiation InputFormat inf = conf.GetInputFormat(); string fn = conf.Get("test.filebench.name", string.Empty); Path pin = new Path(FileInputFormat.GetInputPaths(conf)[0], fn); FileStatus @in = pin.GetFileSystem(conf).GetFileStatus(pin); RecordReader rr = inf.GetRecordReader(new FileSplit(pin, 0, @in.GetLen(), (string [])null), conf, Reporter.Null); try { object key = rr.CreateKey(); object val = rr.CreateValue(); DateTime start = new DateTime(); while (rr.Next(key, val)) { } DateTime end = new DateTime(); return(end.GetTime() - start.GetTime()); } finally { rr.Close(); } }
/// <summary> /// Randomize the split order, then take the specified number of keys from /// each split sampled, where each key is selected with the specified /// probability and possibly replaced by a subsequently selected key when /// the quota of keys from that split is satisfied. /// </summary> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public virtual K[] GetSample(InputFormat <K, V> inf, Job job) { // ArrayList::toArray doesn't preserve type IList <InputSplit> splits = inf.GetSplits(job); AList <K> samples = new AList <K>(numSamples); int splitsToSample = Math.Min(maxSplitsSampled, splits.Count); Random r = new Random(); long seed = r.NextLong(); r.SetSeed(seed); Log.Debug("seed: " + seed); // shuffle splits for (int i = 0; i < splits.Count; ++i) { InputSplit tmp = splits[i]; int j = r.Next(splits.Count); splits.Set(i, splits[j]); splits.Set(j, tmp); } // our target rate is in terms of the maximum number of sample splits, // but we accept the possibility of sampling additional splits to hit // the target sample keyset for (int i_1 = 0; i_1 < splitsToSample || (i_1 < splits.Count && samples.Count < numSamples); ++i_1) { TaskAttemptContext samplingContext = new TaskAttemptContextImpl(job.GetConfiguration (), new TaskAttemptID()); RecordReader <K, V> reader = inf.CreateRecordReader(splits[i_1], samplingContext); reader.Initialize(splits[i_1], samplingContext); while (reader.NextKeyValue()) { if (r.NextDouble() <= freq) { if (samples.Count < numSamples) { samples.AddItem(ReflectionUtils.Copy(job.GetConfiguration(), reader.GetCurrentKey (), null)); } else { // When exceeding the maximum number of samples, replace a // random element with this one, then adjust the frequency // to reflect the possibility of existing elements being // pushed out int ind = r.Next(numSamples); if (ind != numSamples) { samples.Set(ind, ReflectionUtils.Copy(job.GetConfiguration(), reader.GetCurrentKey (), null)); } freq *= (numSamples - 1) / (double)numSamples; } } } reader.Close(); } return((K[])Sharpen.Collections.ToArray(samples)); }
/// <exception cref="System.IO.IOException"/> public virtual InputSplit[] GetSplits(JobConf conf, int numSplits) { JobConf confCopy = new JobConf(conf); IList <InputSplit> splits = new AList <InputSplit>(); IDictionary <Path, InputFormat> formatMap = MultipleInputs.GetInputFormatMap(conf); IDictionary <Path, Type> mapperMap = MultipleInputs.GetMapperTypeMap(conf); IDictionary <Type, IList <Path> > formatPaths = new Dictionary <Type, IList <Path> >(); // First, build a map of InputFormats to Paths foreach (KeyValuePair <Path, InputFormat> entry in formatMap) { if (!formatPaths.Contains(entry.Value.GetType())) { formatPaths[entry.Value.GetType()] = new List <Path>(); } formatPaths[entry.Value.GetType()].AddItem(entry.Key); } foreach (KeyValuePair <Type, IList <Path> > formatEntry in formatPaths) { Type formatClass = formatEntry.Key; InputFormat format = (InputFormat)ReflectionUtils.NewInstance(formatClass, conf); IList <Path> paths = formatEntry.Value; IDictionary <Type, IList <Path> > mapperPaths = new Dictionary <Type, IList <Path> >(); // Now, for each set of paths that have a common InputFormat, build // a map of Mappers to the paths they're used for foreach (Path path in paths) { Type mapperClass = mapperMap[path]; if (!mapperPaths.Contains(mapperClass)) { mapperPaths[mapperClass] = new List <Path>(); } mapperPaths[mapperClass].AddItem(path); } // Now each set of paths that has a common InputFormat and Mapper can // be added to the same job, and split together. foreach (KeyValuePair <Type, IList <Path> > mapEntry in mapperPaths) { paths = mapEntry.Value; Type mapperClass = mapEntry.Key; if (mapperClass == null) { mapperClass = conf.GetMapperClass(); } FileInputFormat.SetInputPaths(confCopy, Sharpen.Collections.ToArray(paths, new Path [paths.Count])); // Get splits for each input path and tag with InputFormat // and Mapper types by wrapping in a TaggedInputSplit. InputSplit[] pathSplits = format.GetSplits(confCopy, numSplits); foreach (InputSplit pathSplit in pathSplits) { splits.AddItem(new TaggedInputSplit(pathSplit, conf, format.GetType(), mapperClass )); } } } return(Sharpen.Collections.ToArray(splits, new InputSplit[splits.Count])); }
[InlineData(InputFormat.String, "")] // empty public void Remove(InputFormat inputFormat, string key) { _command.Remove( key: key, inputFormat: inputFormat, rocksdbPath: _temporaryDirectory); using var db = _rocksDbService.Load(_temporaryDirectory); Assert.Null(db.Get(inputFormat.Decode(key))); }
/// <summary> /// Initializes a new instance of the FormatInfo class. /// </summary> /// <param name="inputFormat">Possible values include: /// 'org.apache.hadoop.mapred.SequenceFileInputFormat', /// 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'</param> /// <param name="outputFormat">Possible values include: /// 'org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat', /// 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'</param> /// <param name="formatType">Possible values include: 'csv', /// 'parquet'</param> /// <param name="serializeLib">Possible values include: /// 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe', /// 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'</param> /// <param name="properties">Property bag.</param> /// <param name="serDeName">SerDe name.</param> public FormatInfo(InputFormat inputFormat, OutputFormat outputFormat, FormatType formatType, SerializeLib serializeLib, System.Collections.Generic.IDictionary <string, JToken> properties = default(System.Collections.Generic.IDictionary <string, JToken>), string serDeName = default(string)) { InputFormat = inputFormat; OutputFormat = outputFormat; FormatType = formatType; Properties = properties; SerDeName = serDeName; SerializeLib = serializeLib; }
private static InputFormat SetDefaultsBasedOnInputFormat(InputFormat inFormat, PublicationInformation projectInfo, string lcName, string inputType) { if (string.IsNullOrEmpty(projectInfo.ProjectPath)) { projectInfo.ProjectPath = Path.GetDirectoryName(lcName); } projectInfo.ProjectInputType = inputType; return(inFormat); }
public void ConvertMethod_ThrowsException_InCaseOfInvalidInput() { // arrange var logFactory = new NullLoggerFactory(); var invalidInput = new InputFormat[1] { new InputFormat("test13;4803;1835;zzz") }; // Note zzzs at the end - should be integers var actual = Transformations.ConvertCSVToFormatA(invalidInput, logFactory.CreateLogger("mock")); }
/// <summary> /// Randomize the split order, then take the specified number of keys from /// each split sampled, where each key is selected with the specified /// probability and possibly replaced by a subsequently selected key when /// the quota of keys from that split is satisfied. /// </summary> /// <exception cref="System.IO.IOException"/> public virtual K[] GetSample(InputFormat <K, V> inf, JobConf job) { // ArrayList::toArray doesn't preserve type InputSplit[] splits = inf.GetSplits(job, job.GetNumMapTasks()); AList <K> samples = new AList <K>(numSamples); int splitsToSample = Math.Min(maxSplitsSampled, splits.Length); Random r = new Random(); long seed = r.NextLong(); r.SetSeed(seed); Log.Debug("seed: " + seed); // shuffle splits for (int i = 0; i < splits.Length; ++i) { InputSplit tmp = splits[i]; int j = r.Next(splits.Length); splits[i] = splits[j]; splits[j] = tmp; } // our target rate is in terms of the maximum number of sample splits, // but we accept the possibility of sampling additional splits to hit // the target sample keyset for (int i_1 = 0; i_1 < splitsToSample || (i_1 < splits.Length && samples.Count < numSamples); ++i_1) { RecordReader <K, V> reader = inf.GetRecordReader(splits[i_1], job, Reporter.Null); K key = reader.CreateKey(); V value = reader.CreateValue(); while (reader.Next(key, value)) { if (r.NextDouble() <= freq) { if (samples.Count < numSamples) { samples.AddItem(key); } else { // When exceeding the maximum number of samples, replace a // random element with this one, then adjust the frequency // to reflect the possibility of existing elements being // pushed out int ind = r.Next(numSamples); if (ind != numSamples) { samples.Set(ind, key); } freq *= (numSamples - 1) / (double)numSamples; } key = reader.CreateKey(); } } reader.Close(); } return((K[])Sharpen.Collections.ToArray(samples)); }
public static async Task <int> Convert(string inputFile, string outputFile, InputFormat inputFormat, ConvertOutputFormat outputFormat) { var inputBomFormat = InputFormatHelper(inputFile, inputFormat); if (inputBomFormat == BomFormat.Unsupported) { return((int)ExitCode.ParameterValidationError); } BomFormat outputBomFormat = BomFormat.Unsupported; string inputBomString; CycloneDX.Models.v1_2.Bom inputBom; string outputBomString; if (outputFormat == ConvertOutputFormat.autodetect) { if (string.IsNullOrEmpty(outputFile)) { Console.Error.WriteLine("You must specify a value for --output-format when standard output is used"); return((int)ExitCode.ParameterValidationError); } outputBomFormat = Utils.DetectFileFormat(outputFile); if (outputBomFormat == BomFormat.Unsupported) { Console.Error.WriteLine("Unable to auto-detect output format from output filename"); return((int)ExitCode.ParameterValidationError); } } else { outputBomFormat = (BomFormat)outputFormat; } inputBomString = InputFileHelper(inputFile); if (inputBomString == null) { return((int)ExitCode.ParameterValidationError); } inputBom = Utils.BomDeserializer(inputBomString, inputBomFormat); outputBomString = Utils.BomSerializer(inputBom, outputBomFormat); if (string.IsNullOrEmpty(outputFile)) { Console.Write(outputBomString); } else { Console.WriteLine("Writing output file..."); await File.WriteAllTextAsync(outputFile, outputBomString); } return((int)ExitCode.Ok); }
private string BuildInputFormat() { if (InputFormat != null) { return($"-f {InputFormat.ToString()} "); } else { return(string.Empty); } }
/// <exception cref="System.IO.IOException"/> public virtual RecordReader GetRecordReader(InputSplit split, JobConf job, Reporter reporter) { InputFormat indirIF = (InputFormat)ReflectionUtils.NewInstance(job.GetClass(GenericMRLoadGenerator .IndirectInputFormat, typeof(SequenceFileInputFormat)), job); GenericMRLoadGenerator.IndirectInputFormat.IndirectSplit @is = ((GenericMRLoadGenerator.IndirectInputFormat.IndirectSplit )split); return(indirIF.GetRecordReader(new FileSplit(@is.GetPath(), 0, @is.GetLength(), ( string[])null), job, reporter)); }
public static InputFormat newInputFormat(string name) { global::System.IntPtr cPtr = udpipe_csharpPINVOKE.InputFormat_newInputFormat(name); InputFormat ret = (cPtr == global::System.IntPtr.Zero) ? null : new InputFormat(cPtr, true); if (udpipe_csharpPINVOKE.SWIGPendingException.Pending) { throw udpipe_csharpPINVOKE.SWIGPendingException.Retrieve(); } return(ret); }
/// <summary>Constructs the DelegatingRecordReader.</summary> /// <param name="split">TaggegInputSplit object</param> /// <param name="context">TaskAttemptContext object</param> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public DelegatingRecordReader(InputSplit split, TaskAttemptContext context) { // Find the InputFormat and then the RecordReader from the // TaggedInputSplit. TaggedInputSplit taggedInputSplit = (TaggedInputSplit)split; InputFormat <K, V> inputFormat = (InputFormat <K, V>)ReflectionUtils.NewInstance(taggedInputSplit .GetInputFormatClass(), context.GetConfiguration()); originalRR = inputFormat.CreateRecordReader(taggedInputSplit.GetInputSplit(), context ); }
public virtual InputFormat newTokenizer(string options) { global::System.IntPtr cPtr = udpipe_csharpPINVOKE.Model_newTokenizer(swigCPtr, options); InputFormat ret = (cPtr == global::System.IntPtr.Zero) ? null : new InputFormat(cPtr, true); if (udpipe_csharpPINVOKE.SWIGPendingException.Pending) { throw udpipe_csharpPINVOKE.SWIGPendingException.Retrieve(); } return(ret); }
private void CoreAction( BackgroundWorker bw, InputFormat format, string inputDir, string outputDir, string tempDir) { Converter converter = new Converter(); switch (format) { case InputFormat.MusicXml: case InputFormat.Xml: case InputFormat.Mxl: { File.Copy(inputDir, outputDir, true); return; } case InputFormat.Vsq3: converter.ImportVsq3(new List <string> { inputDir }); break; case InputFormat.Vsq4: converter.ImportVsq4(new List <string> { inputDir }); break; case InputFormat.Vpr: converter.ImportVpr(new List <string> { inputDir }, tempDir); break; case InputFormat.Ust: converter.ImportUst(new List <string> { inputDir }); break; case InputFormat.Ccs: converter.ImportCcs(new List <string> { inputDir }); break; default: throw new NeutrinoStudioFileConverterOperationException("Input format not supported."); } converter.ExportMusicXml(outputDir); }
/// <exception cref="System.IO.IOException"/> public virtual RecordReader <K, V> GetRecordReader(InputSplit split, JobConf conf, Reporter reporter) { // Find the InputFormat and then the RecordReader from the // TaggedInputSplit. TaggedInputSplit taggedInputSplit = (TaggedInputSplit)split; InputFormat <K, V> inputFormat = (InputFormat <K, V>)ReflectionUtils.NewInstance(taggedInputSplit .GetInputFormatClass(), conf); return(inputFormat.GetRecordReader(taggedInputSplit.GetInputSplit(), conf, reporter )); }
/// <summary> /// Process the file. /// </summary> /// /// <param name="outputFile">The output file.</param> /// <param name="method">The method to use.</param> public void Process(FileInfo outputFile, IMLRegression method) { var csv = new ReadCSV(InputFilename.ToString(), ExpectInputHeaders, InputFormat); if (method.InputCount > _inputCount) { throw new AnalystError("This machine learning method has " + method.InputCount + " inputs, however, the data has " + _inputCount + " inputs."); } IMLData input = new BasicMLData(method.InputCount); StreamWriter tw = AnalystPrepareOutputFile(outputFile); ResetStatus(); while (csv.Next()) { UpdateStatus(false); var row = new LoadedRow(csv, _idealCount); int dataIndex = 0; // load the input data for (int i = 0; i < _inputCount; i++) { String str = row.Data[i]; double d = InputFormat.Parse(str); input[i] = d; dataIndex++; } // do we need to skip the ideal values? dataIndex += _idealCount; // compute the result IMLData output = method.Compute(input); // display the computed result for (int i = 0; i < _outputCount; i++) { double d = output[i]; row.Data[dataIndex++] = InputFormat.Format(d, Precision); } WriteRow(tw, row); } ReportDone(false); tw.Close(); csv.Close(); }
public static char[] Parse(InputFormat format) { if (format.IsCustom) { var delimiters = format .GetDelimitersToParse() .ToCharArray(); return(delimiters); } return(DEFAULT_DELIMITERS); }
public Hex(string data, InputFormat inputFormat) { switch (inputFormat) { case InputFormat.String: this.FromString(data); break; case InputFormat.Hex: _data = data; break; } }
private void txt_FirstStationHeadPlace_KeyPress(object sender, KeyPressEventArgs e) { InputFormat ifobj = new InputFormat(); ifobj.HalfWidthFormat(e); }
private void txt_StaHeadAntennaB_KeyPress(object sender, KeyPressEventArgs e) { InputFormat ifobj = new InputFormat(); ifobj.HalfWidthFormat(e); }
private void txt_EndStationHeadAddress_KeyPress(object sender, KeyPressEventArgs e) { InputFormat ifobj = new InputFormat(); ifobj.HalfWidthFormat(e); }
public virtual HtmlValidatorResult Validate(string filename, OutputFormat outputFormat, string input, InputFormat inputFormat, IHtmlValidatorSettings settings) { return Validate(filename, outputFormat, input, inputFormat, settings, defaultValidatorAddress); }
private static InputFormat SetDefaultsBasedOnInputFormat(InputFormat inFormat, PublicationInformation projectInfo, string lcName, string inputType) { if (string.IsNullOrEmpty(projectInfo.ProjectPath)) { projectInfo.ProjectPath = Path.GetDirectoryName(lcName); } projectInfo.ProjectInputType = inputType; return inFormat; }
private void textBox_EmployeeCensusRegister_KeyPress(object sender, KeyPressEventArgs e) { InputFormat ifobj = new InputFormat(); ifobj.HalfWidthFormat(e); }
public virtual HtmlValidatorResult Validate(Stream output, OutputFormat outputFormat, string input, InputFormat inputFormat, string validatorAddress) { return Validate(output, outputFormat, input, inputFormat, new HtmlValidatorSettings(), validatorAddress); }
private static InputFormat SetDefaultsBasedOnInputFormat(InputFormat inFormat, PublicationInformation projectInfo, string lcName) { return SetDefaultsBasedOnInputFormat(inFormat, projectInfo, lcName, "Scripture"); }
public virtual HtmlValidatorResult Validate(Stream output, OutputFormat outputFormat, string input, InputFormat inputFormat, IHtmlValidatorSettings settings) { return Validate(output, outputFormat, input, inputFormat, settings, defaultValidatorAddress); }
public virtual HtmlValidatorResult Validate(string filename, OutputFormat outputFormat, string input, InputFormat inputFormat, IHtmlValidatorSettings settings, string validatorAddress) { HtmlValidatorResult result = null; // Create the directory automatically if it doesn't exist. string directory = Path.GetDirectoryName(filename); if (!Directory.Exists(directory)) { Directory.CreateDirectory(directory); } if (outputFormat == OutputFormat.Html) { // Copy the resources to the output directory if they don't already exist. // This is to ensure all dependencies of the HTML document are there. mResourceCopier.CopyResources(directory); } using (FileStream output = new FileStream(filename, FileMode.Create)) { result = Validate(output, outputFormat, input, inputFormat, settings, validatorAddress); } return result; }
private void textBox_EmpStateOfHealth_KeyPress(object sender, KeyPressEventArgs e) { InputFormat ifobj = new InputFormat(); ifobj.HalfWidthFormat(e); }
private void textBox_EmpWorkPlace_KeyPress(object sender, KeyPressEventArgs e) { InputFormat ifobj = new InputFormat(); ifobj.HalfWidthFormat(e); }
private void txtName_KeyPress(object sender, KeyPressEventArgs e) { InputFormat ifobj = new InputFormat(); ifobj.HalfWidthFormat(e); }
/// <summary> /// Runs PathwayB on the data and applies the back end /// </summary> private string RunPathwayB(InputFormat inFormat, string files, string project, string layout, string inputType, string backend, string message) { const bool overwrite = true; var arg = new StringBuilder(); var inPath = Common.PathCombine(_inputPath, project); if (project.Length < 1 || layout.Length < 1 || inputType.Length < 1 || backend.Length < 1) { // missing some args -- call usage on PathwayB arg.Append("-h "); } switch (inFormat) { case InputFormat.XHTML: // "-d .../TestFiles/Output" arg.Append("-d \""); arg.Append(_outputPath); // "-if xhtml" arg.Append("\" -if xhtml"); // "-c .../TestFiles/Output/<layout>.css" arg.Append(" -c \""); arg.Append(_outputPath); arg.Append(Path.DirectorySeparatorChar); arg.Append(layout); arg.Append(".css\""); arg.Append(" "); break; case InputFormat.USFM: // "-d .../TestFiles/Output" arg.Append("-d \""); arg.Append(_outputPath); // "-if usfm" arg.Append("\" -if usfm"); arg.Append(" "); break; case InputFormat.USX: // "-u .../TestFiles/Output" arg.Append("-d \""); arg.Append(_outputPath); // "-if usx" arg.Append("\" -if usx"); arg.Append(" "); break; default: break; } arg.Append("-f "); arg.Append(files); arg.Append(" -t \""); arg.Append(backend); arg.Append("\" -i \""); arg.Append(inputType); arg.Append("\" -n \""); arg.Append(project); arg.Append("\""); Debug.WriteLine("Calling PathwayB.exe " + arg.ToString()); var sb = new StringBuilder(); string errorMessage = "", outputMessage = ""; const int timeout = 60; try { var proc = new Process { StartInfo = { FileName = Common.PathCombine(PathwayPath.GetPathwayDir(), "PathwayB.exe"), Arguments = arg.ToString(), RedirectStandardError = true, RedirectStandardOutput = true, WindowStyle = ProcessWindowStyle.Hidden, UseShellExecute = false } }; proc.Start(); proc.WaitForExit ( timeout * 100 * 60 ); errorMessage = proc.StandardError.ReadToEnd(); proc.WaitForExit(); outputMessage = proc.StandardOutput.ReadToEnd(); proc.WaitForExit(); } catch (Exception e) { sb.AppendLine("Exception encountered in RunPathwayB:"); sb.AppendLine(); sb.AppendLine(e.Message); if (e.InnerException != null) { sb.AppendLine("Inner Exception:"); sb.AppendLine(e.InnerException.ToString()); } errorMessage = sb.ToString(); } if (errorMessage.Length > 0) Debug.WriteLine(errorMessage); if (outputMessage.Length > 0) Debug.WriteLine(outputMessage); // check the results switch (backend) { case "E-Book (Epub2 and Epub3)": epubCheck(layout, message); break; case "OpenOffice/LibreOffice": OdtCheck(project, message); break; case "InDesign": IdmlCheck(project, message); break; default: break; } if (errorMessage.Length > 0) return errorMessage; if (outputMessage.Length > 0) return outputMessage; return null; }
private void textBox_EmployeeArchives_KeyPress(object sender, KeyPressEventArgs e) { InputFormat ifobj = new InputFormat(); ifobj.HalfWidthFormat(e); }
public virtual HtmlValidatorResult Validate(Stream output, OutputFormat outputFormat, string input, InputFormat inputFormat, IHtmlValidatorSettings settings, string validatorAddress) { if (string.IsNullOrEmpty(input)) { throw new ArgumentNullException("input"); } if (settings == null) { throw new ArgumentNullException("settings"); } if (string.IsNullOrEmpty(validatorAddress)) validatorAddress = defaultValidatorAddress; NameValueCollection headers; string data = GetFormData(input, inputFormat, outputFormat, settings); if (inputFormat == InputFormat.Fragment) { headers = this.mHttpClient.Post(output, validatorAddress, data); } else { headers = this.mHttpClient.Get(output, validatorAddress + "?" + data); } HtmlValidatorResult result = ParseResult(headers); // W3C API BUG: for some reason it doesn't return headers every time it is accessed. // Check if the W3C headers were returned if (string.IsNullOrEmpty(result.Status)) { result = FixBrokenHeaders(output, outputFormat, input, inputFormat, settings, validatorAddress); } return result; }
public virtual HtmlValidatorResult Validate(string input, InputFormat inputFormat, IHtmlValidatorSettings settings, string validatorAddress) { return Validate((Stream)null, OutputFormat.Soap12, input, inputFormat, settings, validatorAddress); }
private static int ProcessExportType(string[] args, int i, PublicationInformation projectInfo, List<string> files, ref InputFormat inFormat, ref bool bShowDialog, ref string exportType, ref bool bOutputSpecified) { switch (args[i++]) { case "--directory": case "-d": projectInfo.ProjectPath = args[i++]; break; case "--files": case "-f": i = CaptureFileList(args, i, files); break; case "--inputformat": case "-if": var format = args[i++]; switch (format) { case "xhtml": inFormat = InputFormat.XHTML; break; case "usfm": inFormat = InputFormat.USFM; break; case "usx": inFormat = InputFormat.USX; break; case "ptb": inFormat = InputFormat.PTBUNDLE; break; } break; case "--xhtml": case "-x": // retained for backwards compatibility (not documented) inFormat = InputFormat.XHTML; projectInfo.DefaultXhtmlFileWithPath = args[i++]; break; case "--showdialog": case "-s": bShowDialog = true; break; case "--css": case "-c": projectInfo.DefaultCssFileWithPath = args[i++]; break; case "--target": case "-t": //Note: If export type is more than one word, quotes must be used exportType = args[i++]; bOutputSpecified = true; break; case "--input": case "-i": projectInfo.ProjectInputType = args[i++]; break; case "--launch": case "-l": projectInfo.IsOpenOutput = true; break; case "--name": case "-n": projectInfo.ProjectName = args[i++]; break; case "-?": case "-h": Usage(); Environment.Exit(0); break; default: i = CaptureFileList(args, --i, files); if (files.Count > 0) { var lcName = files[0].ToLower(); if (lcName.EndsWith(".zip")) { inFormat = SetDefaultsBasedOnInputFormat(InputFormat.PTBUNDLE, projectInfo, lcName); break; } if (lcName.EndsWith(".xhtml")) { inFormat = SetDefaultsBasedOnInputFormat(InputFormat.XHTML, projectInfo, lcName, "Dictionary"); break; } if (lcName.EndsWith(".usx")) { inFormat = SetDefaultsBasedOnInputFormat(InputFormat.USX, projectInfo, lcName); break; } if (lcName.EndsWith(".usfm") || lcName.EndsWith(".sfm")) { inFormat = SetDefaultsBasedOnInputFormat(InputFormat.USFM, projectInfo, lcName); break; } } Usage(); throw new ArgumentException("Invalid Command Line Argument: " + args[i]); } return i; }
public void SetFormat(InputFormat format) { nvttSetInputOptionsFormat(options, format); }
private void cmb_StationPacket_KeyPress(object sender, KeyPressEventArgs e) { InputFormat ifobj = new InputFormat(); ifobj.HalfWidthFormat(e); }
private void txt_Description_Select_KeyPress(object sender, KeyPressEventArgs e) { InputFormat ifobj = new InputFormat(); ifobj.HalfWidthFormat(e); }
private void textBox_DutyRemark_KeyPress(object sender, KeyPressEventArgs e) { InputFormat ifobj = new InputFormat(); ifobj.HalfWidthFormat(e); }
private static void ProcessInputFormat(InputFormat inFormat, List<string> files, PublicationInformation projectInfo) { if (inFormat == InputFormat.PTBUNDLE) { var zf = new FastZip(); zf.ExtractZip(files[0], projectInfo.ProjectPath, ".*"); var metadata = Common.DeclareXMLDocument(false); metadata.Load(Common.PathCombine(projectInfo.ProjectPath, "metadata.xml")); var titleNode = metadata.SelectSingleNode("//bookList[@id='default']/name"); Param.UpdateTitleMetadataValue(Param.Title, titleNode.InnerText, false); var descriptionNode = metadata.SelectSingleNode("//bookList[@id='default']//range"); Param.UpdateMetadataValue(Param.Description, descriptionNode.InnerText); var rightsHolderNode = metadata.SelectSingleNode("//rightsHolder"); Param.UpdateMetadataValue(Param.Publisher, rightsHolderNode.InnerText); var copyrightNode = metadata.SelectSingleNode("//statement"); Param.UpdateMetadataValue(Param.CopyrightHolder, copyrightNode.InnerText); var bookNodes = metadata.SelectNodes("//bookList[@id='default']//book"); var usxFolder = Common.PathCombine(projectInfo.ProjectPath, "USX"); var xmlText = new StringBuilder(@"<usx version=""2.0"">"); var oneBook = Common.DeclareXMLDocument(false); foreach (XmlElement bookNode in bookNodes) { oneBook.Load(Common.PathCombine(usxFolder, bookNode.SelectSingleNode("@code").InnerText + ".usx")); xmlText.Append(oneBook.DocumentElement.InnerXml); oneBook.RemoveAll(); } xmlText.Append(@"</usx>" + "\r\n"); var fileName = Common.PathCombine(projectInfo.ProjectPath, titleNode.InnerText + ".xhtml"); var databaseName = metadata.SelectSingleNode("//identification/abbreviation"); var linkParam = new Dictionary<string, object>(); linkParam["dateTime"] = DateTime.Now.ToShortDateString(); linkParam["user"] = "******"; var wsNode = metadata.SelectSingleNode("//language/iso"); linkParam["ws"] = wsNode.InnerText; linkParam["userWs"] = wsNode.InnerText; var nameNode = metadata.SelectSingleNode("//identification/description"); linkParam["projName"] = nameNode.InnerText; var langNameNode = metadata.SelectSingleNode("//language/name"); linkParam["langInfo"] = string.Format("{0}:{1}", wsNode.InnerText, langNameNode.InnerText); ConvertUsx2Xhtml(databaseName, linkParam, xmlText, fileName); projectInfo.DefaultXhtmlFileWithPath = fileName; var ptxStyle2Css = new XslCompiledTransform(); ptxStyle2Css.Load(XmlReader.Create(Assembly.GetExecutingAssembly().GetManifestResourceStream( "PathwayB.ptx2css.xsl"))); projectInfo.DefaultCssFileWithPath = Common.PathCombine(projectInfo.ProjectPath, titleNode.InnerText + ".css"); var writer = new XmlTextWriter(projectInfo.DefaultCssFileWithPath, Encoding.UTF8); ptxStyle2Css.Transform(Common.PathCombine(projectInfo.ProjectPath, "styles.xml"), null, writer); writer.Close(); metadata.RemoveAll(); inFormat = InputFormat.XHTML; } if (inFormat == InputFormat.USFM) { // convert from USFM to xhtml UsfmToXhtml(projectInfo, files); } else if (inFormat == InputFormat.USX) { // convert from USX to xhtml UsxToXhtml(projectInfo, files); } else if (inFormat == InputFormat.XHTML) { SetFileName(projectInfo, files); } }
private HtmlValidatorResult FixBrokenHeaders(Stream output, OutputFormat outputFormat, string input, InputFormat inputFormat, IHtmlValidatorSettings settings, string validatorAddress) { Stream checkStream = mStreamFactory.GetMemoryStream(); if (outputFormat != OutputFormat.Soap12 || output.CanRead == false) { if (IsDefaultValidatorAddress(validatorAddress)) { System.Threading.Thread.Sleep(1000); } // Headers failed, so we will get the report again in Soap 1.2 format in // an in memory stream string checkData = GetFormData(input, inputFormat, OutputFormat.Soap12, settings); // This time, ignore headers. if (inputFormat == InputFormat.Fragment) { this.mHttpClient.Post(checkStream, validatorAddress, checkData); } else { this.mHttpClient.Get(checkStream, validatorAddress + "?" + checkData); } } else { output.Position = 0; output.CopyTo(checkStream); } checkStream.Position = 0; var response = mSoapResponseParser.ParseResponse(checkStream); var errors = response.Errors.Count(); var warnings = response.Warnings.Count(); var status = response.Validity ? "Valid" : "Invalid"; var recursion = 1; var result = new HtmlValidatorResult(status, errors, warnings, recursion); return result; }
private void txt_Directional_Add_KeyPress(object sender, KeyPressEventArgs e) { InputFormat ifobj = new InputFormat(); ifobj.HalfWidthFormat(e); }
private static extern void nvttSetInputOptionsFormat(IntPtr inputOptions, InputFormat format);
private string GetFormData(string input, InputFormat inputFormat, OutputFormat outputFormat, IHtmlValidatorSettings settings) { if (string.IsNullOrEmpty(input)) { throw new ArgumentNullException("input"); } string data = "uri=" + HttpUtility.UrlEncode(input); if (inputFormat == InputFormat.Fragment) { // fix fragment so it is possible to exclude the <html> and other envelope tags and only test specific tags. input = FixHtmlFragment(input); data = "fragment=" + HttpUtility.UrlEncode(input); } if (!string.IsNullOrEmpty(settings.CharSet)) data += "&charset=" + HttpUtility.UrlEncode(settings.CharSet); if (!string.IsNullOrEmpty(settings.DocType)) data += "&doctype=" + HttpUtility.UrlEncode(settings.DocType); if (outputFormat == OutputFormat.Soap12) data += "&output=soap12"; if (settings.Verbose) data += "&verbose=1"; if (settings.Debug) data += "&debug=1"; if (settings.ShowSource) data += "&ss=1"; if (settings.Outline) data += "&outline=1"; if (settings.GroupErrors) data += "&group=1"; if (settings.UseHtmlTidy) data += "&st=1"; return data; }
public virtual HtmlValidatorResult Validate(string filename, OutputFormat outputFormat, string input, InputFormat inputFormat, string validatorAddress) { return Validate(filename, outputFormat, input, inputFormat, new HtmlValidatorSettings(), validatorAddress); }
public virtual HtmlValidatorResult Validate(string input, InputFormat inputFormat) { return Validate((Stream)null, OutputFormat.Soap12, input, inputFormat, new HtmlValidatorSettings(), defaultValidatorAddress); }