public static InputFormat newPresegmentedTokenizer(InputFormat DISOWN)
{
global::System.IntPtr cPtr = udpipe_csharpPINVOKE.InputFormat_newPresegmentedTokenizer(InputFormat.getCPtrAndDisown(DISOWN));
InputFormat ret = (cPtr == global::System.IntPtr.Zero) ? null : new InputFormat(cPtr, true);
return(ret);
}
/// <summary>From each split sampled, take the first numSamples / numSplits records.</summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual K[] GetSample(InputFormat <K, V> inf, Job job)
{
// ArrayList::toArray doesn't preserve type
IList <InputSplit> splits = inf.GetSplits(job);
AList <K> samples = new AList <K>(numSamples);
int splitsToSample = Math.Min(maxSplitsSampled, splits.Count);
int samplesPerSplit = numSamples / splitsToSample;
long records = 0;
for (int i = 0; i < splitsToSample; ++i)
{
TaskAttemptContext samplingContext = new TaskAttemptContextImpl(job.GetConfiguration
(), new TaskAttemptID());
RecordReader <K, V> reader = inf.CreateRecordReader(splits[i], samplingContext);
reader.Initialize(splits[i], samplingContext);
while (reader.NextKeyValue())
{
samples.AddItem(ReflectionUtils.Copy(job.GetConfiguration(), reader.GetCurrentKey
(), null));
++records;
if ((i + 1) * samplesPerSplit <= records)
{
break;
}
}
reader.Close();
}
return((K[])Sharpen.Collections.ToArray(samples));
}
/// <summary>
/// Process the file and output to the target file.
/// </summary>
/// <param name="target">The target file to write to.</param>
public void Process(string target)
{
var csv = new ReadCSV(InputFilename.ToString(), ExpectInputHeaders, InputFormat);
TextWriter tw = new StreamWriter(target);
ResetStatus();
while (csv.Next())
{
var line = new StringBuilder();
UpdateStatus(false);
line.Append(GetColumnData(FileData.Date, csv));
line.Append(" ");
line.Append(GetColumnData(FileData.Time, csv));
line.Append(";");
line.Append(InputFormat.Format(double.Parse(GetColumnData(FileData.Open, csv)), Precision));
line.Append(";");
line.Append(InputFormat.Format(double.Parse(GetColumnData(FileData.High, csv)), Precision));
line.Append(";");
line.Append(InputFormat.Format(double.Parse(GetColumnData(FileData.Low, csv)), Precision));
line.Append(";");
line.Append(InputFormat.Format(double.Parse(GetColumnData(FileData.Close, csv)), Precision));
line.Append(";");
line.Append(InputFormat.Format(double.Parse(GetColumnData(FileData.Volume, csv)), Precision));
tw.WriteLine(line.ToString());
}
ReportDone(false);
csv.Close();
tw.Close();
}
private static void Process(InputFormat inputFormat, OutputFormat outputFormat, string fromCode,
string fromList,
VjudgeOverflowBehavior overflowBehavior, int penaltyTime)
{
if (!string.IsNullOrEmpty(fromCode) && !string.IsNullOrEmpty(fromList))
{
Console.WriteLine("'--fromCode' and '--fromList' can not be simultaneously set!");
return;
}
if (string.IsNullOrEmpty(fromCode) && string.IsNullOrEmpty(fromList))
{
Console.WriteLine("At least one of '--fromCode' and '--fromList' must be set!");
return;
}
if (!string.IsNullOrEmpty(fromCode))
{
Process(inputFormat, outputFormat, fromCode, overflowBehavior, penaltyTime);
}
else
{
var lines = File.ReadAllLines(fromList);
foreach (var line in lines)
{
Process(inputFormat, outputFormat, line, overflowBehavior, penaltyTime);
}
}
}
/// <summary>From each split sampled, take the first numSamples / numSplits records.</summary>
/// <exception cref="System.IO.IOException"/>
public virtual K[] GetSample(InputFormat <K, V> inf, JobConf job)
{
// ArrayList::toArray doesn't preserve type
InputSplit[] splits = inf.GetSplits(job, job.GetNumMapTasks());
AList <K> samples = new AList <K>(numSamples);
int splitsToSample = Math.Min(maxSplitsSampled, splits.Length);
int splitStep = splits.Length / splitsToSample;
int samplesPerSplit = numSamples / splitsToSample;
long records = 0;
for (int i = 0; i < splitsToSample; ++i)
{
RecordReader <K, V> reader = inf.GetRecordReader(splits[i * splitStep], job, Reporter
.Null);
K key = reader.CreateKey();
V value = reader.CreateValue();
while (reader.Next(key, value))
{
samples.AddItem(key);
key = reader.CreateKey();
++records;
if ((i + 1) * samplesPerSplit <= records)
{
break;
}
}
reader.Close();
}
return((K[])Sharpen.Collections.ToArray(samples));
}
public Dominomics(MultipleDirectorySelector mds, InputFormat inputF)
{
Progress = 0;
MyInputFormat = inputF;
//Read all SEPro Files to obtain a dictionary of ID (key) and fasta (value) and peptide (key) SeproFiles (values)
locusFastaDict = new Dictionary <string, string>();
myResultPackages = new List <FileInfoResultPackage>();
matrixClassDescriptionDictionary = new Dictionary <int, string>();
foreach (DirectoryClassDescription myDir in mds.MyDirectoryDescriptionDictionary)
{
if (inputF == InputFormat.SEPro)
{
FileInfo[] resultFilteredFiles = new DirectoryInfo(myDir.MyDirectoryFullName).GetFiles("*.sepr", SearchOption.AllDirectories);
FeedLocusFastaDictSEPro(myDir, resultFilteredFiles);
}
else if (inputF == InputFormat.MPex)
{
FileInfo[] resultFilteredFiles = new DirectoryInfo(myDir.MyDirectoryFullName).GetFiles("*.mpex", SearchOption.AllDirectories);
FeedLocusFastaDictPex(myDir, resultFilteredFiles);
}
matrixClassDescriptionDictionary.Add(myDir.ClassLabel, myDir.Description);
}
bdf = new HMMVerifier.BatchDomainFetcher();
}
/// <summary>
/// Prints keys and values from <see cref="RocksDbSharp.RocksDb"/> located at <paramref name="rocksdbPath"/>.
/// </summary>
/// <param name="prefix">The prefix of keys to filter.</param>
/// <param name="inputFormat">An format of <paramref name="prefix"/>.
/// For instance, if <paramref name="inputFormat"/> is <see cref="InputFormat.Base64"/>, <paramref name="prefix"/>
/// should be base64 encoded value. If <paramref name="inputFormat"/> is <see cref="InputFormat.String"/>,
/// it treats <paramref name="prefix"/> as <see cref="Encoding.UTF8"/> encoded string.</param>
/// <param name="outputFormat">A format of output (i.e. keys and values).</param>
/// <param name="rocksdbPath">The path of <see cref="RocksDbSharp.RocksDb"/> to load.</param>
public void List(
[Option] string prefix = "",
[Option] InputFormat inputFormat = InputFormat.String,
[Option] OutputFormat outputFormat = OutputFormat.Hex,
[Option] string?rocksdbPath = null)
{
rocksdbPath ??= Directory.GetCurrentDirectory();
try
{
using var db = _rocksDbService.Load(rocksdbPath);
byte[] prefixBytes = inputFormat.Decode(prefix);
_inputOutputErrorContainer.Error.WriteLine("Key\tValue");
using var iterator = db.NewIterator();
for (iterator.Seek(prefixBytes);
iterator.Valid() && iterator.Key().Take(prefixBytes.Length).SequenceEqual(prefixBytes);
iterator.Next())
{
_inputOutputErrorContainer.Out.Write(outputFormat.Encode(iterator.Key()));
_inputOutputErrorContainer.Out.Write('\t');
_inputOutputErrorContainer.Out.WriteLine(outputFormat.Encode(iterator.Value()));
}
}
catch (Exception e)
{
_inputOutputErrorContainer.Error.WriteLine(e.Message);
}
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test(expected = MediaTypeNotSupportedException.class) public void canNotGetInputFormatBasedOnWildcardMediaType() throws Exception
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
public virtual void CanNotGetInputFormatBasedOnWildcardMediaType()
{
InputFormat format = _repository.inputFormat(MediaType.WILDCARD_TYPE);
format.ReadValue("foo");
fail("Got InputFormat based on wild card type: " + format);
}
public static BomFormat InputFormatHelper(string inputFile, InputFormat inputFormat)
{
if (inputFormat == InputFormat.autodetect)
{
if (string.IsNullOrEmpty(inputFile))
{
Console.Error.WriteLine("Unable to auto-detect input stream format, please specify a value for --input-format");
}
var inputBomFormat = CLIUtils.DetectFileFormat(inputFile);
if (inputBomFormat == BomFormat.Unsupported)
{
Console.Error.WriteLine("Unable to auto-detect input format from input filename");
}
return(inputBomFormat);
}
else
{
if (inputFormat == InputFormat.json)
{
return(BomFormat.Json);
}
else if (inputFormat == InputFormat.xml)
{
return(BomFormat.Xml);
}
else if (inputFormat == InputFormat.csv)
{
return(BomFormat.Csv);
}
}
return(BomFormat.Unsupported);
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: public ExtensionService(@Context InputFormat input, @Context OutputFormat output, @Context PluginInvocator extensions, @Context Database database)
public ExtensionService(InputFormat input, OutputFormat output, PluginInvocator extensions, Database database)
{
this._input = input;
this._output = output;
this._extensions = extensions;
this._graphDb = database.Graph;
}
public static InputFormat newVerticalInputFormat()
{
global::System.IntPtr cPtr = udpipe_csharpPINVOKE.InputFormat_newVerticalInputFormat__SWIG_1();
InputFormat ret = (cPtr == global::System.IntPtr.Zero) ? null : new InputFormat(cPtr, true);
return(ret);
}
/// <exception cref="System.IO.IOException"/>
internal static long ReadBench(JobConf conf)
{
// InputFormat instantiation
InputFormat inf = conf.GetInputFormat();
string fn = conf.Get("test.filebench.name", string.Empty);
Path pin = new Path(FileInputFormat.GetInputPaths(conf)[0], fn);
FileStatus @in = pin.GetFileSystem(conf).GetFileStatus(pin);
RecordReader rr = inf.GetRecordReader(new FileSplit(pin, 0, @in.GetLen(), (string
[])null), conf, Reporter.Null);
try
{
object key = rr.CreateKey();
object val = rr.CreateValue();
DateTime start = new DateTime();
while (rr.Next(key, val))
{
}
DateTime end = new DateTime();
return(end.GetTime() - start.GetTime());
}
finally
{
rr.Close();
}
}
/// <summary>
/// Randomize the split order, then take the specified number of keys from
/// each split sampled, where each key is selected with the specified
/// probability and possibly replaced by a subsequently selected key when
/// the quota of keys from that split is satisfied.
/// </summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual K[] GetSample(InputFormat <K, V> inf, Job job)
{
// ArrayList::toArray doesn't preserve type
IList <InputSplit> splits = inf.GetSplits(job);
AList <K> samples = new AList <K>(numSamples);
int splitsToSample = Math.Min(maxSplitsSampled, splits.Count);
Random r = new Random();
long seed = r.NextLong();
r.SetSeed(seed);
Log.Debug("seed: " + seed);
// shuffle splits
for (int i = 0; i < splits.Count; ++i)
{
InputSplit tmp = splits[i];
int j = r.Next(splits.Count);
splits.Set(i, splits[j]);
splits.Set(j, tmp);
}
// our target rate is in terms of the maximum number of sample splits,
// but we accept the possibility of sampling additional splits to hit
// the target sample keyset
for (int i_1 = 0; i_1 < splitsToSample || (i_1 < splits.Count && samples.Count <
numSamples); ++i_1)
{
TaskAttemptContext samplingContext = new TaskAttemptContextImpl(job.GetConfiguration
(), new TaskAttemptID());
RecordReader <K, V> reader = inf.CreateRecordReader(splits[i_1], samplingContext);
reader.Initialize(splits[i_1], samplingContext);
while (reader.NextKeyValue())
{
if (r.NextDouble() <= freq)
{
if (samples.Count < numSamples)
{
samples.AddItem(ReflectionUtils.Copy(job.GetConfiguration(), reader.GetCurrentKey
(), null));
}
else
{
// When exceeding the maximum number of samples, replace a
// random element with this one, then adjust the frequency
// to reflect the possibility of existing elements being
// pushed out
int ind = r.Next(numSamples);
if (ind != numSamples)
{
samples.Set(ind, ReflectionUtils.Copy(job.GetConfiguration(), reader.GetCurrentKey
(), null));
}
freq *= (numSamples - 1) / (double)numSamples;
}
}
}
reader.Close();
}
return((K[])Sharpen.Collections.ToArray(samples));
}
/// <exception cref="System.IO.IOException"/>
public virtual InputSplit[] GetSplits(JobConf conf, int numSplits)
{
JobConf confCopy = new JobConf(conf);
IList <InputSplit> splits = new AList <InputSplit>();
IDictionary <Path, InputFormat> formatMap = MultipleInputs.GetInputFormatMap(conf);
IDictionary <Path, Type> mapperMap = MultipleInputs.GetMapperTypeMap(conf);
IDictionary <Type, IList <Path> > formatPaths = new Dictionary <Type, IList <Path> >();
// First, build a map of InputFormats to Paths
foreach (KeyValuePair <Path, InputFormat> entry in formatMap)
{
if (!formatPaths.Contains(entry.Value.GetType()))
{
formatPaths[entry.Value.GetType()] = new List <Path>();
}
formatPaths[entry.Value.GetType()].AddItem(entry.Key);
}
foreach (KeyValuePair <Type, IList <Path> > formatEntry in formatPaths)
{
Type formatClass = formatEntry.Key;
InputFormat format = (InputFormat)ReflectionUtils.NewInstance(formatClass, conf);
IList <Path> paths = formatEntry.Value;
IDictionary <Type, IList <Path> > mapperPaths = new Dictionary <Type, IList <Path> >();
// Now, for each set of paths that have a common InputFormat, build
// a map of Mappers to the paths they're used for
foreach (Path path in paths)
{
Type mapperClass = mapperMap[path];
if (!mapperPaths.Contains(mapperClass))
{
mapperPaths[mapperClass] = new List <Path>();
}
mapperPaths[mapperClass].AddItem(path);
}
// Now each set of paths that has a common InputFormat and Mapper can
// be added to the same job, and split together.
foreach (KeyValuePair <Type, IList <Path> > mapEntry in mapperPaths)
{
paths = mapEntry.Value;
Type mapperClass = mapEntry.Key;
if (mapperClass == null)
{
mapperClass = conf.GetMapperClass();
}
FileInputFormat.SetInputPaths(confCopy, Sharpen.Collections.ToArray(paths, new Path
[paths.Count]));
// Get splits for each input path and tag with InputFormat
// and Mapper types by wrapping in a TaggedInputSplit.
InputSplit[] pathSplits = format.GetSplits(confCopy, numSplits);
foreach (InputSplit pathSplit in pathSplits)
{
splits.AddItem(new TaggedInputSplit(pathSplit, conf, format.GetType(), mapperClass
));
}
}
}
return(Sharpen.Collections.ToArray(splits, new InputSplit[splits.Count]));
}
[InlineData(InputFormat.String, "")] // empty
public void Remove(InputFormat inputFormat, string key)
{
_command.Remove(
key: key,
inputFormat: inputFormat,
rocksdbPath: _temporaryDirectory);
using var db = _rocksDbService.Load(_temporaryDirectory);
Assert.Null(db.Get(inputFormat.Decode(key)));
}
/// <summary>
/// Initializes a new instance of the FormatInfo class.
/// </summary>
/// <param name="inputFormat">Possible values include:
/// 'org.apache.hadoop.mapred.SequenceFileInputFormat',
/// 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'</param>
/// <param name="outputFormat">Possible values include:
/// 'org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat',
/// 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'</param>
/// <param name="formatType">Possible values include: 'csv',
/// 'parquet'</param>
/// <param name="serializeLib">Possible values include:
/// 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe',
/// 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'</param>
/// <param name="properties">Property bag.</param>
/// <param name="serDeName">SerDe name.</param>
public FormatInfo(InputFormat inputFormat, OutputFormat outputFormat, FormatType formatType, SerializeLib serializeLib, System.Collections.Generic.IDictionary <string, JToken> properties = default(System.Collections.Generic.IDictionary <string, JToken>), string serDeName = default(string))
{
InputFormat = inputFormat;
OutputFormat = outputFormat;
FormatType = formatType;
Properties = properties;
SerDeName = serDeName;
SerializeLib = serializeLib;
}
private static InputFormat SetDefaultsBasedOnInputFormat(InputFormat inFormat, PublicationInformation projectInfo, string lcName, string inputType)
{
if (string.IsNullOrEmpty(projectInfo.ProjectPath))
{
projectInfo.ProjectPath = Path.GetDirectoryName(lcName);
}
projectInfo.ProjectInputType = inputType;
return(inFormat);
}
public void ConvertMethod_ThrowsException_InCaseOfInvalidInput()
{
// arrange
var logFactory = new NullLoggerFactory();
var invalidInput = new InputFormat[1] {
new InputFormat("test13;4803;1835;zzz")
}; // Note zzzs at the end - should be integers
var actual = Transformations.ConvertCSVToFormatA(invalidInput, logFactory.CreateLogger("mock"));
}
/// <summary>
/// Randomize the split order, then take the specified number of keys from
/// each split sampled, where each key is selected with the specified
/// probability and possibly replaced by a subsequently selected key when
/// the quota of keys from that split is satisfied.
/// </summary>
/// <exception cref="System.IO.IOException"/>
public virtual K[] GetSample(InputFormat <K, V> inf, JobConf job)
{
// ArrayList::toArray doesn't preserve type
InputSplit[] splits = inf.GetSplits(job, job.GetNumMapTasks());
AList <K> samples = new AList <K>(numSamples);
int splitsToSample = Math.Min(maxSplitsSampled, splits.Length);
Random r = new Random();
long seed = r.NextLong();
r.SetSeed(seed);
Log.Debug("seed: " + seed);
// shuffle splits
for (int i = 0; i < splits.Length; ++i)
{
InputSplit tmp = splits[i];
int j = r.Next(splits.Length);
splits[i] = splits[j];
splits[j] = tmp;
}
// our target rate is in terms of the maximum number of sample splits,
// but we accept the possibility of sampling additional splits to hit
// the target sample keyset
for (int i_1 = 0; i_1 < splitsToSample || (i_1 < splits.Length && samples.Count <
numSamples); ++i_1)
{
RecordReader <K, V> reader = inf.GetRecordReader(splits[i_1], job, Reporter.Null);
K key = reader.CreateKey();
V value = reader.CreateValue();
while (reader.Next(key, value))
{
if (r.NextDouble() <= freq)
{
if (samples.Count < numSamples)
{
samples.AddItem(key);
}
else
{
// When exceeding the maximum number of samples, replace a
// random element with this one, then adjust the frequency
// to reflect the possibility of existing elements being
// pushed out
int ind = r.Next(numSamples);
if (ind != numSamples)
{
samples.Set(ind, key);
}
freq *= (numSamples - 1) / (double)numSamples;
}
key = reader.CreateKey();
}
}
reader.Close();
}
return((K[])Sharpen.Collections.ToArray(samples));
}
public static async Task <int> Convert(string inputFile, string outputFile, InputFormat inputFormat, ConvertOutputFormat outputFormat)
{
var inputBomFormat = InputFormatHelper(inputFile, inputFormat);
if (inputBomFormat == BomFormat.Unsupported)
{
return((int)ExitCode.ParameterValidationError);
}
BomFormat outputBomFormat = BomFormat.Unsupported;
string inputBomString;
CycloneDX.Models.v1_2.Bom inputBom;
string outputBomString;
if (outputFormat == ConvertOutputFormat.autodetect)
{
if (string.IsNullOrEmpty(outputFile))
{
Console.Error.WriteLine("You must specify a value for --output-format when standard output is used");
return((int)ExitCode.ParameterValidationError);
}
outputBomFormat = Utils.DetectFileFormat(outputFile);
if (outputBomFormat == BomFormat.Unsupported)
{
Console.Error.WriteLine("Unable to auto-detect output format from output filename");
return((int)ExitCode.ParameterValidationError);
}
}
else
{
outputBomFormat = (BomFormat)outputFormat;
}
inputBomString = InputFileHelper(inputFile);
if (inputBomString == null)
{
return((int)ExitCode.ParameterValidationError);
}
inputBom = Utils.BomDeserializer(inputBomString, inputBomFormat);
outputBomString = Utils.BomSerializer(inputBom, outputBomFormat);
if (string.IsNullOrEmpty(outputFile))
{
Console.Write(outputBomString);
}
else
{
Console.WriteLine("Writing output file...");
await File.WriteAllTextAsync(outputFile, outputBomString);
}
return((int)ExitCode.Ok);
}
private string BuildInputFormat()
{
if (InputFormat != null)
{
return($"-f {InputFormat.ToString()} ");
}
else
{
return(string.Empty);
}
}
/// <exception cref="System.IO.IOException"/>
public virtual RecordReader GetRecordReader(InputSplit split, JobConf job, Reporter
reporter)
{
InputFormat indirIF = (InputFormat)ReflectionUtils.NewInstance(job.GetClass(GenericMRLoadGenerator
.IndirectInputFormat, typeof(SequenceFileInputFormat)), job);
GenericMRLoadGenerator.IndirectInputFormat.IndirectSplit @is = ((GenericMRLoadGenerator.IndirectInputFormat.IndirectSplit
)split);
return(indirIF.GetRecordReader(new FileSplit(@is.GetPath(), 0, @is.GetLength(), (
string[])null), job, reporter));
}
public static InputFormat newInputFormat(string name)
{
global::System.IntPtr cPtr = udpipe_csharpPINVOKE.InputFormat_newInputFormat(name);
InputFormat ret = (cPtr == global::System.IntPtr.Zero) ? null : new InputFormat(cPtr, true);
if (udpipe_csharpPINVOKE.SWIGPendingException.Pending)
{
throw udpipe_csharpPINVOKE.SWIGPendingException.Retrieve();
}
return(ret);
}
/// <summary>Constructs the DelegatingRecordReader.</summary>
/// <param name="split">TaggegInputSplit object</param>
/// <param name="context">TaskAttemptContext object</param>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public DelegatingRecordReader(InputSplit split, TaskAttemptContext context)
{
// Find the InputFormat and then the RecordReader from the
// TaggedInputSplit.
TaggedInputSplit taggedInputSplit = (TaggedInputSplit)split;
InputFormat <K, V> inputFormat = (InputFormat <K, V>)ReflectionUtils.NewInstance(taggedInputSplit
.GetInputFormatClass(), context.GetConfiguration());
originalRR = inputFormat.CreateRecordReader(taggedInputSplit.GetInputSplit(), context
);
}
public virtual InputFormat newTokenizer(string options)
{
global::System.IntPtr cPtr = udpipe_csharpPINVOKE.Model_newTokenizer(swigCPtr, options);
InputFormat ret = (cPtr == global::System.IntPtr.Zero) ? null : new InputFormat(cPtr, true);
if (udpipe_csharpPINVOKE.SWIGPendingException.Pending)
{
throw udpipe_csharpPINVOKE.SWIGPendingException.Retrieve();
}
return(ret);
}
private void CoreAction(
BackgroundWorker bw,
InputFormat format,
string inputDir,
string outputDir,
string tempDir)
{
Converter converter = new Converter();
switch (format)
{
case InputFormat.MusicXml:
case InputFormat.Xml:
case InputFormat.Mxl:
{
File.Copy(inputDir, outputDir, true);
return;
}
case InputFormat.Vsq3:
converter.ImportVsq3(new List <string> {
inputDir
});
break;
case InputFormat.Vsq4:
converter.ImportVsq4(new List <string> {
inputDir
});
break;
case InputFormat.Vpr:
converter.ImportVpr(new List <string> {
inputDir
}, tempDir);
break;
case InputFormat.Ust:
converter.ImportUst(new List <string> {
inputDir
});
break;
case InputFormat.Ccs:
converter.ImportCcs(new List <string> {
inputDir
});
break;
default:
throw new NeutrinoStudioFileConverterOperationException("Input format not supported.");
}
converter.ExportMusicXml(outputDir);
}
/// <exception cref="System.IO.IOException"/>
public virtual RecordReader <K, V> GetRecordReader(InputSplit split, JobConf conf,
Reporter reporter)
{
// Find the InputFormat and then the RecordReader from the
// TaggedInputSplit.
TaggedInputSplit taggedInputSplit = (TaggedInputSplit)split;
InputFormat <K, V> inputFormat = (InputFormat <K, V>)ReflectionUtils.NewInstance(taggedInputSplit
.GetInputFormatClass(), conf);
return(inputFormat.GetRecordReader(taggedInputSplit.GetInputSplit(), conf, reporter
));
}
/// <summary>
/// Process the file.
/// </summary>
///
/// <param name="outputFile">The output file.</param>
/// <param name="method">The method to use.</param>
public void Process(FileInfo outputFile, IMLRegression method)
{
var csv = new ReadCSV(InputFilename.ToString(),
ExpectInputHeaders, InputFormat);
if (method.InputCount > _inputCount)
{
throw new AnalystError("This machine learning method has "
+ method.InputCount
+ " inputs, however, the data has " + _inputCount
+ " inputs.");
}
IMLData input = new BasicMLData(method.InputCount);
StreamWriter tw = AnalystPrepareOutputFile(outputFile);
ResetStatus();
while (csv.Next())
{
UpdateStatus(false);
var row = new LoadedRow(csv, _idealCount);
int dataIndex = 0;
// load the input data
for (int i = 0; i < _inputCount; i++)
{
String str = row.Data[i];
double d = InputFormat.Parse(str);
input[i] = d;
dataIndex++;
}
// do we need to skip the ideal values?
dataIndex += _idealCount;
// compute the result
IMLData output = method.Compute(input);
// display the computed result
for (int i = 0; i < _outputCount; i++)
{
double d = output[i];
row.Data[dataIndex++] = InputFormat.Format(d, Precision);
}
WriteRow(tw, row);
}
ReportDone(false);
tw.Close();
csv.Close();
}
public static char[] Parse(InputFormat format)
{
if (format.IsCustom)
{
var delimiters = format
.GetDelimitersToParse()
.ToCharArray();
return(delimiters);
}
return(DEFAULT_DELIMITERS);
}
public Hex(string data, InputFormat inputFormat)
{
switch (inputFormat)
{
case InputFormat.String:
this.FromString(data);
break;
case InputFormat.Hex:
_data = data;
break;
}
}
private void txt_FirstStationHeadPlace_KeyPress(object sender, KeyPressEventArgs e)
{
InputFormat ifobj = new InputFormat();
ifobj.HalfWidthFormat(e);
}
private void txt_StaHeadAntennaB_KeyPress(object sender, KeyPressEventArgs e)
{
InputFormat ifobj = new InputFormat();
ifobj.HalfWidthFormat(e);
}
private void txt_EndStationHeadAddress_KeyPress(object sender, KeyPressEventArgs e)
{
InputFormat ifobj = new InputFormat();
ifobj.HalfWidthFormat(e);
}
public virtual HtmlValidatorResult Validate(string filename, OutputFormat outputFormat, string input, InputFormat inputFormat, IHtmlValidatorSettings settings)
{
return Validate(filename, outputFormat, input, inputFormat, settings, defaultValidatorAddress);
}
private static InputFormat SetDefaultsBasedOnInputFormat(InputFormat inFormat, PublicationInformation projectInfo, string lcName, string inputType)
{
if (string.IsNullOrEmpty(projectInfo.ProjectPath))
{
projectInfo.ProjectPath = Path.GetDirectoryName(lcName);
}
projectInfo.ProjectInputType = inputType;
return inFormat;
}
private void textBox_EmployeeCensusRegister_KeyPress(object sender, KeyPressEventArgs e)
{
InputFormat ifobj = new InputFormat();
ifobj.HalfWidthFormat(e);
}
public virtual HtmlValidatorResult Validate(Stream output, OutputFormat outputFormat, string input, InputFormat inputFormat, string validatorAddress)
{
return Validate(output, outputFormat, input, inputFormat, new HtmlValidatorSettings(), validatorAddress);
}
private static InputFormat SetDefaultsBasedOnInputFormat(InputFormat inFormat, PublicationInformation projectInfo, string lcName)
{
return SetDefaultsBasedOnInputFormat(inFormat, projectInfo, lcName, "Scripture");
}
public virtual HtmlValidatorResult Validate(Stream output, OutputFormat outputFormat, string input, InputFormat inputFormat, IHtmlValidatorSettings settings)
{
return Validate(output, outputFormat, input, inputFormat, settings, defaultValidatorAddress);
}
public virtual HtmlValidatorResult Validate(string filename, OutputFormat outputFormat, string input, InputFormat inputFormat, IHtmlValidatorSettings settings, string validatorAddress)
{
HtmlValidatorResult result = null;
// Create the directory automatically if it doesn't exist.
string directory = Path.GetDirectoryName(filename);
if (!Directory.Exists(directory))
{
Directory.CreateDirectory(directory);
}
if (outputFormat == OutputFormat.Html)
{
// Copy the resources to the output directory if they don't already exist.
// This is to ensure all dependencies of the HTML document are there.
mResourceCopier.CopyResources(directory);
}
using (FileStream output = new FileStream(filename, FileMode.Create))
{
result = Validate(output, outputFormat, input, inputFormat, settings, validatorAddress);
}
return result;
}
private void textBox_EmpStateOfHealth_KeyPress(object sender, KeyPressEventArgs e)
{
InputFormat ifobj = new InputFormat();
ifobj.HalfWidthFormat(e);
}
private void textBox_EmpWorkPlace_KeyPress(object sender, KeyPressEventArgs e)
{
InputFormat ifobj = new InputFormat();
ifobj.HalfWidthFormat(e);
}
private void txtName_KeyPress(object sender, KeyPressEventArgs e)
{
InputFormat ifobj = new InputFormat();
ifobj.HalfWidthFormat(e);
}
/// <summary>
/// Runs PathwayB on the data and applies the back end
/// </summary>
private string RunPathwayB(InputFormat inFormat, string files, string project, string layout, string inputType, string backend, string message)
{
const bool overwrite = true;
var arg = new StringBuilder();
var inPath = Common.PathCombine(_inputPath, project);
if (project.Length < 1 || layout.Length < 1 || inputType.Length < 1 || backend.Length < 1)
{
// missing some args -- call usage on PathwayB
arg.Append("-h ");
}
switch (inFormat)
{
case InputFormat.XHTML:
// "-d .../TestFiles/Output"
arg.Append("-d \"");
arg.Append(_outputPath);
// "-if xhtml"
arg.Append("\" -if xhtml");
// "-c .../TestFiles/Output/<layout>.css"
arg.Append(" -c \"");
arg.Append(_outputPath);
arg.Append(Path.DirectorySeparatorChar);
arg.Append(layout);
arg.Append(".css\"");
arg.Append(" ");
break;
case InputFormat.USFM:
// "-d .../TestFiles/Output"
arg.Append("-d \"");
arg.Append(_outputPath);
// "-if usfm"
arg.Append("\" -if usfm");
arg.Append(" ");
break;
case InputFormat.USX:
// "-u .../TestFiles/Output"
arg.Append("-d \"");
arg.Append(_outputPath);
// "-if usx"
arg.Append("\" -if usx");
arg.Append(" ");
break;
default:
break;
}
arg.Append("-f ");
arg.Append(files);
arg.Append(" -t \"");
arg.Append(backend);
arg.Append("\" -i \"");
arg.Append(inputType);
arg.Append("\" -n \"");
arg.Append(project);
arg.Append("\"");
Debug.WriteLine("Calling PathwayB.exe " + arg.ToString());
var sb = new StringBuilder();
string errorMessage = "", outputMessage = "";
const int timeout = 60;
try
{
var proc = new Process
{
StartInfo =
{
FileName = Common.PathCombine(PathwayPath.GetPathwayDir(), "PathwayB.exe"),
Arguments = arg.ToString(),
RedirectStandardError = true,
RedirectStandardOutput = true,
WindowStyle = ProcessWindowStyle.Hidden,
UseShellExecute = false
}
};
proc.Start();
proc.WaitForExit
(
timeout * 100 * 60
);
errorMessage = proc.StandardError.ReadToEnd();
proc.WaitForExit();
outputMessage = proc.StandardOutput.ReadToEnd();
proc.WaitForExit();
}
catch (Exception e)
{
sb.AppendLine("Exception encountered in RunPathwayB:");
sb.AppendLine();
sb.AppendLine(e.Message);
if (e.InnerException != null)
{
sb.AppendLine("Inner Exception:");
sb.AppendLine(e.InnerException.ToString());
}
errorMessage = sb.ToString();
}
if (errorMessage.Length > 0) Debug.WriteLine(errorMessage);
if (outputMessage.Length > 0) Debug.WriteLine(outputMessage);
// check the results
switch (backend)
{
case "E-Book (Epub2 and Epub3)":
epubCheck(layout, message);
break;
case "OpenOffice/LibreOffice":
OdtCheck(project, message);
break;
case "InDesign":
IdmlCheck(project, message);
break;
default:
break;
}
if (errorMessage.Length > 0) return errorMessage;
if (outputMessage.Length > 0) return outputMessage;
return null;
}
private void textBox_EmployeeArchives_KeyPress(object sender, KeyPressEventArgs e)
{
InputFormat ifobj = new InputFormat();
ifobj.HalfWidthFormat(e);
}
public virtual HtmlValidatorResult Validate(Stream output, OutputFormat outputFormat, string input, InputFormat inputFormat, IHtmlValidatorSettings settings, string validatorAddress)
{
if (string.IsNullOrEmpty(input))
{
throw new ArgumentNullException("input");
}
if (settings == null)
{
throw new ArgumentNullException("settings");
}
if (string.IsNullOrEmpty(validatorAddress))
validatorAddress = defaultValidatorAddress;
NameValueCollection headers;
string data = GetFormData(input, inputFormat, outputFormat, settings);
if (inputFormat == InputFormat.Fragment)
{
headers = this.mHttpClient.Post(output, validatorAddress, data);
}
else
{
headers = this.mHttpClient.Get(output, validatorAddress + "?" + data);
}
HtmlValidatorResult result = ParseResult(headers);
// W3C API BUG: for some reason it doesn't return headers every time it is accessed.
// Check if the W3C headers were returned
if (string.IsNullOrEmpty(result.Status))
{
result = FixBrokenHeaders(output, outputFormat, input, inputFormat, settings, validatorAddress);
}
return result;
}
public virtual HtmlValidatorResult Validate(string input, InputFormat inputFormat, IHtmlValidatorSettings settings, string validatorAddress)
{
return Validate((Stream)null, OutputFormat.Soap12, input, inputFormat, settings, validatorAddress);
}
private static int ProcessExportType(string[] args, int i, PublicationInformation projectInfo, List<string> files,
ref InputFormat inFormat, ref bool bShowDialog, ref string exportType,
ref bool bOutputSpecified)
{
switch (args[i++])
{
case "--directory":
case "-d":
projectInfo.ProjectPath = args[i++];
break;
case "--files":
case "-f":
i = CaptureFileList(args, i, files);
break;
case "--inputformat":
case "-if":
var format = args[i++];
switch (format)
{
case "xhtml":
inFormat = InputFormat.XHTML;
break;
case "usfm":
inFormat = InputFormat.USFM;
break;
case "usx":
inFormat = InputFormat.USX;
break;
case "ptb":
inFormat = InputFormat.PTBUNDLE;
break;
}
break;
case "--xhtml":
case "-x":
// retained for backwards compatibility (not documented)
inFormat = InputFormat.XHTML;
projectInfo.DefaultXhtmlFileWithPath = args[i++];
break;
case "--showdialog":
case "-s":
bShowDialog = true;
break;
case "--css":
case "-c":
projectInfo.DefaultCssFileWithPath = args[i++];
break;
case "--target":
case "-t":
//Note: If export type is more than one word, quotes must be used
exportType = args[i++];
bOutputSpecified = true;
break;
case "--input":
case "-i":
projectInfo.ProjectInputType = args[i++];
break;
case "--launch":
case "-l":
projectInfo.IsOpenOutput = true;
break;
case "--name":
case "-n":
projectInfo.ProjectName = args[i++];
break;
case "-?":
case "-h":
Usage();
Environment.Exit(0);
break;
default:
i = CaptureFileList(args, --i, files);
if (files.Count > 0)
{
var lcName = files[0].ToLower();
if (lcName.EndsWith(".zip"))
{
inFormat = SetDefaultsBasedOnInputFormat(InputFormat.PTBUNDLE, projectInfo, lcName);
break;
}
if (lcName.EndsWith(".xhtml"))
{
inFormat = SetDefaultsBasedOnInputFormat(InputFormat.XHTML, projectInfo, lcName, "Dictionary");
break;
}
if (lcName.EndsWith(".usx"))
{
inFormat = SetDefaultsBasedOnInputFormat(InputFormat.USX, projectInfo, lcName);
break;
}
if (lcName.EndsWith(".usfm") || lcName.EndsWith(".sfm"))
{
inFormat = SetDefaultsBasedOnInputFormat(InputFormat.USFM, projectInfo, lcName);
break;
}
}
Usage();
throw new ArgumentException("Invalid Command Line Argument: " + args[i]);
}
return i;
}
public void SetFormat(InputFormat format)
{
nvttSetInputOptionsFormat(options, format);
}
private void cmb_StationPacket_KeyPress(object sender, KeyPressEventArgs e)
{
InputFormat ifobj = new InputFormat();
ifobj.HalfWidthFormat(e);
}
private void txt_Description_Select_KeyPress(object sender, KeyPressEventArgs e)
{
InputFormat ifobj = new InputFormat();
ifobj.HalfWidthFormat(e);
}
private void textBox_DutyRemark_KeyPress(object sender, KeyPressEventArgs e)
{
InputFormat ifobj = new InputFormat();
ifobj.HalfWidthFormat(e);
}
private static void ProcessInputFormat(InputFormat inFormat, List<string> files, PublicationInformation projectInfo)
{
if (inFormat == InputFormat.PTBUNDLE)
{
var zf = new FastZip();
zf.ExtractZip(files[0], projectInfo.ProjectPath, ".*");
var metadata = Common.DeclareXMLDocument(false);
metadata.Load(Common.PathCombine(projectInfo.ProjectPath, "metadata.xml"));
var titleNode = metadata.SelectSingleNode("//bookList[@id='default']/name");
Param.UpdateTitleMetadataValue(Param.Title, titleNode.InnerText, false);
var descriptionNode = metadata.SelectSingleNode("//bookList[@id='default']//range");
Param.UpdateMetadataValue(Param.Description, descriptionNode.InnerText);
var rightsHolderNode = metadata.SelectSingleNode("//rightsHolder");
Param.UpdateMetadataValue(Param.Publisher, rightsHolderNode.InnerText);
var copyrightNode = metadata.SelectSingleNode("//statement");
Param.UpdateMetadataValue(Param.CopyrightHolder, copyrightNode.InnerText);
var bookNodes = metadata.SelectNodes("//bookList[@id='default']//book");
var usxFolder = Common.PathCombine(projectInfo.ProjectPath, "USX");
var xmlText = new StringBuilder(@"<usx version=""2.0"">");
var oneBook = Common.DeclareXMLDocument(false);
foreach (XmlElement bookNode in bookNodes)
{
oneBook.Load(Common.PathCombine(usxFolder, bookNode.SelectSingleNode("@code").InnerText + ".usx"));
xmlText.Append(oneBook.DocumentElement.InnerXml);
oneBook.RemoveAll();
}
xmlText.Append(@"</usx>" + "\r\n");
var fileName = Common.PathCombine(projectInfo.ProjectPath, titleNode.InnerText + ".xhtml");
var databaseName = metadata.SelectSingleNode("//identification/abbreviation");
var linkParam = new Dictionary<string, object>();
linkParam["dateTime"] = DateTime.Now.ToShortDateString();
linkParam["user"] = "******";
var wsNode = metadata.SelectSingleNode("//language/iso");
linkParam["ws"] = wsNode.InnerText;
linkParam["userWs"] = wsNode.InnerText;
var nameNode = metadata.SelectSingleNode("//identification/description");
linkParam["projName"] = nameNode.InnerText;
var langNameNode = metadata.SelectSingleNode("//language/name");
linkParam["langInfo"] = string.Format("{0}:{1}", wsNode.InnerText, langNameNode.InnerText);
ConvertUsx2Xhtml(databaseName, linkParam, xmlText, fileName);
projectInfo.DefaultXhtmlFileWithPath = fileName;
var ptxStyle2Css = new XslCompiledTransform();
ptxStyle2Css.Load(XmlReader.Create(Assembly.GetExecutingAssembly().GetManifestResourceStream(
"PathwayB.ptx2css.xsl")));
projectInfo.DefaultCssFileWithPath = Common.PathCombine(projectInfo.ProjectPath, titleNode.InnerText + ".css");
var writer = new XmlTextWriter(projectInfo.DefaultCssFileWithPath, Encoding.UTF8);
ptxStyle2Css.Transform(Common.PathCombine(projectInfo.ProjectPath, "styles.xml"), null, writer);
writer.Close();
metadata.RemoveAll();
inFormat = InputFormat.XHTML;
}
if (inFormat == InputFormat.USFM)
{
// convert from USFM to xhtml
UsfmToXhtml(projectInfo, files);
}
else if (inFormat == InputFormat.USX)
{
// convert from USX to xhtml
UsxToXhtml(projectInfo, files);
}
else if (inFormat == InputFormat.XHTML)
{
SetFileName(projectInfo, files);
}
}
private HtmlValidatorResult FixBrokenHeaders(Stream output, OutputFormat outputFormat, string input, InputFormat inputFormat, IHtmlValidatorSettings settings, string validatorAddress)
{
Stream checkStream = mStreamFactory.GetMemoryStream();
if (outputFormat != OutputFormat.Soap12 || output.CanRead == false)
{
if (IsDefaultValidatorAddress(validatorAddress))
{
System.Threading.Thread.Sleep(1000);
}
// Headers failed, so we will get the report again in Soap 1.2 format in
// an in memory stream
string checkData = GetFormData(input, inputFormat, OutputFormat.Soap12, settings);
// This time, ignore headers.
if (inputFormat == InputFormat.Fragment)
{
this.mHttpClient.Post(checkStream, validatorAddress, checkData);
}
else
{
this.mHttpClient.Get(checkStream, validatorAddress + "?" + checkData);
}
}
else
{
output.Position = 0;
output.CopyTo(checkStream);
}
checkStream.Position = 0;
var response = mSoapResponseParser.ParseResponse(checkStream);
var errors = response.Errors.Count();
var warnings = response.Warnings.Count();
var status = response.Validity ? "Valid" : "Invalid";
var recursion = 1;
var result = new HtmlValidatorResult(status, errors, warnings, recursion);
return result;
}
private void txt_Directional_Add_KeyPress(object sender, KeyPressEventArgs e)
{
InputFormat ifobj = new InputFormat();
ifobj.HalfWidthFormat(e);
}
private static extern void nvttSetInputOptionsFormat(IntPtr inputOptions, InputFormat format);
private string GetFormData(string input, InputFormat inputFormat, OutputFormat outputFormat, IHtmlValidatorSettings settings)
{
if (string.IsNullOrEmpty(input))
{
throw new ArgumentNullException("input");
}
string data = "uri=" + HttpUtility.UrlEncode(input);
if (inputFormat == InputFormat.Fragment)
{
// fix fragment so it is possible to exclude the <html> and other envelope tags and only test specific tags.
input = FixHtmlFragment(input);
data = "fragment=" + HttpUtility.UrlEncode(input);
}
if (!string.IsNullOrEmpty(settings.CharSet))
data += "&charset=" + HttpUtility.UrlEncode(settings.CharSet);
if (!string.IsNullOrEmpty(settings.DocType))
data += "&doctype=" + HttpUtility.UrlEncode(settings.DocType);
if (outputFormat == OutputFormat.Soap12)
data += "&output=soap12";
if (settings.Verbose)
data += "&verbose=1";
if (settings.Debug)
data += "&debug=1";
if (settings.ShowSource)
data += "&ss=1";
if (settings.Outline)
data += "&outline=1";
if (settings.GroupErrors)
data += "&group=1";
if (settings.UseHtmlTidy)
data += "&st=1";
return data;
}
public virtual HtmlValidatorResult Validate(string filename, OutputFormat outputFormat, string input, InputFormat inputFormat, string validatorAddress)
{
return Validate(filename, outputFormat, input, inputFormat, new HtmlValidatorSettings(), validatorAddress);
}
public virtual HtmlValidatorResult Validate(string input, InputFormat inputFormat)
{
return Validate((Stream)null, OutputFormat.Soap12, input, inputFormat, new HtmlValidatorSettings(), defaultValidatorAddress);
}
