/**
* Returns true if the specified file store path can be opened as an XML document, and calling {@link
* #accept(org.w3c.dom.Document)} returns true.
*
* @param fileStore the file store containing the named file path.
* @param fileName the named file path in question.
*
* @return true if the file name should be accepted; false otherwise.
*
* @throws ArgumentException if either the file store or the file name are null.
*/
public bool accept(FileStore fileStore, String fileName)
{
if (fileStore == null)
{
String msg = Logging.getMessage("nullValue.FileStoreIsNull");
Logging.logger().severe(msg);
throw new ArgumentException(msg);
}
if (fileName == null)
{
String message = Logging.getMessage("nullValue.FilePathIsNull");
Logging.logger().severe(message);
throw new ArgumentException(message);
}
// Attempt to locate the named path in the FileStore, optionally checking the class path. If a file with that
// name cannot be located, then return false.
java.net.URL url = fileStore.findFile(fileName, true);
if (url == null)
{
return(false);
}
// Attempt to convert the URL to a local file path. If that succeeds, then continue treating the URL as if
// it were a File.
java.io.File file = WWIO.convertURLToFile(url);
if (file != null)
{
return(this.accept(file));
}
return(this.accept(url));
}
public Spriter(com.discobeard.spriter.dom.SpriterData spriterData, com.brashmonkey.spriter.file.FileLoader loader, java.io.File scmlFile)
{
this.scmlFile = scmlFile;
this.spriterData = spriterData;
this.loader = loader;
this.loadResources();
}
/**
* Returns true if the specified file can be opened as an XML document, and calling {@link
* #accept(org.w3c.dom.Document)} returns true.
*
* @param file the file in question.
*
* @return true if the file should be accepted; false otherwise.
*
* @throws ArgumentException if the file is null.
*/
public bool accept(java.io.File file)
{
if (file == null)
{
String msg = Logging.getMessage("nullValue.FileIsNull");
Logging.logger().severe(msg);
throw new ArgumentException(msg);
}
// First check the file path, optionally returning false if the path cannot be accepted for any reason.
if (!this.acceptFilePath(file.getPath()))
{
return(false);
}
Document doc = null;
try
{
doc = WWXML.openDocumentFile(file.getPath(), this.GetType());
}
catch (Exception e)
{
// Not interested in logging the exception. We just want to return false, indicating that the File cannot
// be opened as an XML document.
}
return((doc != null) && (doc.getDocumentElement() != null) && this.accept(doc));
}
public static int getBooleanAttributes0(object @this, java.io.File file)
{
int rv = 0;
var path = GetPath(file);
if (File.Exists(path) || Directory.Exists(path))
{
rv |= 1;
}
if (File.Exists(path))
{
rv |= 2;
}
if (Directory.Exists(path))
{
rv |= 4;
}
if (rv != 0)
{
var attrs = File.GetAttributes(path);
if ((attrs & FileAttributes.Hidden) != 0)
{
rv |= 8;
}
}
return(rv);
}
public static bool checkAccess(object @this, java.io.File file, int access)
{
var path = GetPath(file);
if ((access & 4) != 0)
{
try
{
File.OpenRead(path).Dispose();
}
catch (IOException)
{
return(false);
}
}
if ((access & 2) != 0)
{
try
{
File.Open(path, FileMode.Open, FileAccess.Write).Dispose();
}
catch (IOException)
{
return(false);
}
}
return(true);
}
static public void query(Sentence[] sentences, string query, string indexPath = "luceneIndex")
{
Dictionary <string, Sentence> map = new Dictionary <string, Sentence>();
foreach (Sentence s in sentences)
{
if (!map.ContainsKey(s.sentnece))
{
map.Add(s.sentnece, s);
}
}
java.io.File indexDir = new java.io.File(indexPath);
FSDirectory indexFSDir = new SimpleFSDirectory(indexDir);
IndexSearcher searcher = new IndexSearcher(IndexReader.open(indexFSDir));
EnglishAnalyzer luceneAnalyzer = new EnglishAnalyzer(org.apache.lucene.util.Version.LUCENE_35);
QueryParser qp = new QueryParser(org.apache.lucene.util.Version.LUCENE_35, "text", luceneAnalyzer);
Query q = qp.parse(query);
TopDocs tdocs = searcher.search(q, 99999999);
ScoreDoc[] sdocs = tdocs.scoreDocs;
for (int i = 0; i < sdocs.Length; i++)
{
ScoreDoc sd = sdocs[i];
Document res = searcher.doc(sd.doc);
string docText = res.get("text");
float score = sd.score;
map[docText].lucene = score;
}
searcher.close();
}
public static string InternalGetHintPath()
{
var ff = "";
try
{
var ct = typeof(CPtr);
var c = ct.ToClass();
var url = c.getResource("");
var path = url.ToString();
// jar:file:/W:x/bin/Debug/staging/web/bin/x.__interface.__delegate.clr.dll!/EstEIDPerso/ChipXpressPlugin/
if (path.StartsWith("jar:file:"))
{
path = path.Substring("jar:file:".Length);
if (path.StartsWith("/"))
{
path = path.Substring(1);
}
path = path.Substring(0, path.IndexOf("!"));
}
ff = new java.io.File(path).getCanonicalPath();
}
catch (csharp.ThrowableException ex)
{
throw new System.InvalidOperationException("InternalGetHintPath");
}
return(Path.GetDirectoryName(ff));
}
// X:\jsc.svn\examples\java\android\AndroidFileExplorerActivity\AndroidFileExplorerActivity\ApplicationActivity.cs
public static string[] GetFiles(string p)
{
var f = new java.io.File(p);
var files = f.listFiles();
return(files.Where(x => x.isFile()).Select(x => x.getAbsolutePath()).ToArray());
}
public void SetMigrationStatus(Org.Neo4j.Io.fs.FileSystemAbstraction fs, java.io.File stateFile, string info)
{
if (fs.FileExists(stateFile))
{
try
{
fs.Truncate(stateFile, 0);
}
catch (IOException e)
{
throw new Exception(e);
}
}
try
{
using (Writer writer = fs.OpenAsWriter(stateFile, StandardCharsets.UTF_8, false))
{
writer.write(name());
writer.write('\n');
writer.write(info);
writer.flush();
}
}
catch (IOException e)
{
throw new Exception(e);
}
}
public Spriter(com.discobeard.spriter.dom.SpriterData spriterData, com.brashmonkey.spriter.file.FileLoader loader, java.io.File scmlFile)
{
this.scmlFile = scmlFile;
this.spriterData = spriterData;
this.loader = loader;
this.loadResources();
}
/// <summary>
/// Create information about a new .apk
/// NOTE: This constructor is called with ActivityThread's lock held,
/// so MUST NOT call back out to the activity manager.
/// </summary>
/// <remarks>
/// Create information about a new .apk
/// NOTE: This constructor is called with ActivityThread's lock held,
/// so MUST NOT call back out to the activity manager.
/// </remarks>
public LoadedApk(android.app.ActivityThread activityThread, android.content.pm.ApplicationInfo
aInfo, android.content.res.CompatibilityInfo compatInfo, android.app.ActivityThread
mainThread, java.lang.ClassLoader baseLoader, bool securityViolation, bool includeCode
)
{
mActivityThread = activityThread;
mApplicationInfo = aInfo;
mPackageName = aInfo.packageName;
mAppDir = aInfo.sourceDir;
mResDir = aInfo.uid == android.os.Process.myUid() ? aInfo.sourceDir : aInfo.publicSourceDir;
mSharedLibraries = aInfo.sharedLibraryFiles;
mDataDir = aInfo.dataDir;
mDataDirFile = mDataDir != null ? new java.io.File(mDataDir) : null;
mLibDir = aInfo.nativeLibraryDir;
mBaseClassLoader = baseLoader;
mSecurityViolation = securityViolation;
mIncludeCode = includeCode;
mCompatibilityInfo.set(compatInfo);
if (mAppDir == null)
{
if (android.app.ActivityThread.mSystemContext == null)
{
android.app.ActivityThread.mSystemContext = android.app.ContextImpl.createSystemContext
(mainThread);
android.app.ActivityThread.mSystemContext.getResources().updateConfiguration(mainThread
.getConfiguration(), mainThread.getDisplayMetricsLocked(compatInfo, false), compatInfo
);
}
//Slog.i(TAG, "Created system resources "
// + mSystemContext.getResources() + ": "
// + mSystemContext.getResources().getConfiguration());
mClassLoader = android.app.ActivityThread.mSystemContext.getClassLoader();
mResources = android.app.ActivityThread.mSystemContext.getResources();
}
}
// X:\jsc.svn\examples\java\android\AndroidFileExplorerActivity\AndroidFileExplorerActivity\ApplicationActivity.cs
public static string[] GetFiles(string p)
{
var f = new java.io.File(p);
var files = f.listFiles();
return files.Where(x => x.isFile()).Select(x => x.getAbsolutePath()).ToArray();
}
private java.io.FileInputStream searchDefaultCacerts()
{
try
{
string javaHome = java.lang.System.getProperty("java.home");
if (javaHome == null)
{
return(null);
}
string keyStorePath = javaHome + "/lib/security/cacerts";
//Console.WriteLine("keyStorePath = {0}", keyStorePath);
java.io.File f = new java.io.File(keyStorePath);
if (!f.exists())
{
return(null);
}
return(new java.io.FileInputStream(f));
}
catch (Exception e)
{
#if DEBUG
//todo log it
Console.WriteLine(e.GetType() + ":" + e.Message + "\n" + e.StackTrace);
#endif
return(null);
}
}
public Spriter(string scmlPath, com.brashmonkey.spriter.file.FileLoader loader
)
{
this.scmlFile = new java.io.File(scmlPath);
this.spriterData = com.brashmonkey.spriter.xml.SCMLReader.load(scmlPath);
this.loader = loader;
loadResources();
}
public java.lang.Class parseClass(java.io.File file)
{
java.lang.Class[] args = new java.lang.Class[] { java.lang.Class.forName("java.io.File") };
java.lang.reflect.Method parseClass = this._groovyClassLoaderClass.getMethod("parseClass", args);
java.lang.Class clazz = (java.lang.Class)parseClass.invoke(this._classLoader, file);
return(clazz);
}
public Spriter(string scmlPath, com.brashmonkey.spriter.file.FileLoader loader
)
{
this.scmlFile = new java.io.File(scmlPath);
this.spriterData = com.brashmonkey.spriter.xml.SCMLReader.load(scmlPath);
this.loader = loader;
loadResources();
}
public static readonly FileOperation private static java.io.File fromFile(Org.Neo4j.Io.fs.FileSystemAbstraction fs, java.io.File directory, String name, boolean skipNonExistent)
{
java.io.File fromFile = new java.io.File(directory, name); if (skipNonExistent && !fs.fileExists(fromFile))
{
return(null);
}
return(fromFile);
}
public static void SaveInstances(weka.core.Instances instances, string fileName)
{
var file = new java.io.File(fileName);
//m_arffSaver.setDestination(file);
m_arffSaver.setInstances(instances);
m_arffSaver.setFile(file);
m_arffSaver.writeBatch();
}
public string MaybeReadInfo(Org.Neo4j.Io.fs.FileSystemAbstraction fs, java.io.File stateFile, string currentInfo)
{
if (!string.ReferenceEquals(currentInfo, null))
{
return(currentInfo);
}
Pair <string, string> data = ReadFromFile(fs, stateFile, this);
return(data == null ? null : data.Other());
}
private static string GenerateDependencyXml(string inputFolderPath)
{
string dependencyFilePath = Path.GetTempFileName();
com.kirkk.analyzer.textui.XMLUISummary analyzer = new com.kirkk.analyzer.textui.XMLUISummary();
java.io.File inputFolder = new java.io.File(inputFolderPath);
java.io.File outputFile = new java.io.File(dependencyFilePath);
analyzer.createSummary(inputFolder, outputFile);
return(dependencyFilePath);
}
public static bool Exists(string path)
{
bool exists = false;
if (path != null && path.Length != 0)
{
var file = new java.io.File(path);
exists = file.exists() && file.isDirectory();
}
return(exists);
}
/**
* Returns true if the specified file is a Feature Table.
*
* @param file the file in question.
*
* @return true if the file should be accepted; false otherwise.
*
* @throws ArgumentException if the file is null.
*/
public bool accept(java.io.File file)
{
if (file == null)
{
String msg = Logging.getMessage("nullValue.FileIsNull");
Logging.logger().severe(msg);
throw new ArgumentException(msg);
}
return(VPFUtils.getFeatureTypeName(file.getName()) != null);
}
private static string GenerateDependencyXml(string inputFolderPath)
{
string dependencyFilePath = Path.GetTempFileName();
com.kirkk.analyzer.textui.XMLUISummary analyzer = new com.kirkk.analyzer.textui.XMLUISummary();
java.io.File inputFolder = new java.io.File(inputFolderPath);
java.io.File outputFile = new java.io.File(dependencyFilePath);
analyzer.createSummary(inputFolder, outputFile);
return dependencyFilePath;
}
private void loadGroovyTasksFromClass(java.lang.Class clazz, java.io.File file)
{
java.lang.reflect.Method[] methods = GroovyUtil.getMethodsAnnotatedWith(clazz, "com.agilex.ci.cifactory.nant.task");
foreach (java.lang.reflect.Method method in methods)
{
string[] args = GroovyUtil.extractTaskArgumentNames(method.getName(), new StreamReader(file.getAbsolutePath()).ReadToEnd());
GroovyTaskBuilder builder = new GroovyTaskBuilder(method.getName(), clazz, method, args);
TypeFactory.TaskBuilders.Add(builder);
}
}
private static string GetPath(java.io.File file)
{
string path = file.getPath();
if (path.StartsWith("file:/") || path.StartsWith("file:\\"))
{
path = path.Substring("file:/".Length);
}
return(path);
}
bool java.io.FilenameFilter.accept(java.io.File arg0, java.lang.String arg1)
{
global::MonoJavaBridge.JNIEnv @__env = global::MonoJavaBridge.JNIEnv.ThreadEnv;
if (!IsClrObject)
{
return(@__env.CallBooleanMethod(this.JvmHandle, global::java.io.FilenameFilter_._accept12580, global::MonoJavaBridge.JavaBridge.ConvertToValue(arg0), global::MonoJavaBridge.JavaBridge.ConvertToValue(arg1)));
}
else
{
return(@__env.CallNonVirtualBooleanMethod(this.JvmHandle, global::java.io.FilenameFilter_.staticClass, global::java.io.FilenameFilter_._accept12580, global::MonoJavaBridge.JavaBridge.ConvertToValue(arg0), global::MonoJavaBridge.JavaBridge.ConvertToValue(arg1)));
}
}
public virtual void addFile(java.lang.String arg0, java.io.File arg1, int arg2)
{
global::MonoJavaBridge.JNIEnv @__env = global::MonoJavaBridge.JNIEnv.ThreadEnv;
if (!IsClrObject)
{
@__env.CallVoidMethod(this.JvmHandle, global::android.os.DropBoxManager._addFile6394, global::MonoJavaBridge.JavaBridge.ConvertToValue(arg0), global::MonoJavaBridge.JavaBridge.ConvertToValue(arg1), global::MonoJavaBridge.JavaBridge.ConvertToValue(arg2));
}
else
{
@__env.CallNonVirtualVoidMethod(this.JvmHandle, global::android.os.DropBoxManager.staticClass, global::android.os.DropBoxManager._addFile6394, global::MonoJavaBridge.JavaBridge.ConvertToValue(arg0), global::MonoJavaBridge.JavaBridge.ConvertToValue(arg1), global::MonoJavaBridge.JavaBridge.ConvertToValue(arg2));
}
}
public virtual int compareTo(java.io.File arg0)
{
global::MonoJavaBridge.JNIEnv @__env = global::MonoJavaBridge.JNIEnv.ThreadEnv;
if (!IsClrObject)
{
return(@__env.CallIntMethod(this.JvmHandle, global::java.io.File._compareTo12498, global::MonoJavaBridge.JavaBridge.ConvertToValue(arg0)));
}
else
{
return(@__env.CallNonVirtualIntMethod(this.JvmHandle, global::java.io.File.staticClass, global::java.io.File._compareTo12498, global::MonoJavaBridge.JavaBridge.ConvertToValue(arg0)));
}
}
public virtual bool renameTo(java.io.File arg0)
{
global::MonoJavaBridge.JNIEnv @__env = global::MonoJavaBridge.JNIEnv.ThreadEnv;
if (!IsClrObject)
{
return(@__env.CallBooleanMethod(this.JvmHandle, global::java.io.File._renameTo12530, global::MonoJavaBridge.JavaBridge.ConvertToValue(arg0)));
}
else
{
return(@__env.CallNonVirtualBooleanMethod(this.JvmHandle, global::java.io.File.staticClass, global::java.io.File._renameTo12530, global::MonoJavaBridge.JavaBridge.ConvertToValue(arg0)));
}
}
public static long getLastModifiedTime(object @this, java.io.File file)
{
var path = GetPath(file);
try
{
return(File.GetLastWriteTime(path).ToFileTime());
}
catch (IOException)
{
return(0);
}
}
public static DirectoryInfo CreateDirectory(string path)
{
ThrowHelper.ThrowIfNull(path);
path = path.Trim();
if (path.Length == 0)
{
throw new System.ArgumentException();
}
var file = new java.io.File(path);
file.mkdirs();
return(new DirectoryInfo(file));
}
/// <summary>
/// Constructs a new
/// <code>PrintStream</code>
/// with
/// <code>file</code>
/// as its target. The
/// character set named
/// <code>csn</code>
/// is used for character encoding.
/// </summary>
/// <param name="file">
/// the target file. If the file already exists, its contents are
/// removed, otherwise a new file is created.
/// </param>
/// <param name="csn">the name of the character set used for character encoding.</param>
/// <exception cref="FileNotFoundException">if an error occurs while opening or creating the target file.
/// </exception>
/// <exception cref="System.ArgumentNullException">
/// if
/// <code>csn</code>
/// is
/// <code>null</code>
/// .
/// </exception>
/// <exception cref="UnsupportedEncodingException">
/// if the encoding specified by
/// <code>csn</code>
/// is not supported.
/// </exception>
/// <exception cref="java.io.FileNotFoundException"></exception>
/// <exception cref="java.io.UnsupportedEncodingException"></exception>
public PrintStream(java.io.File file, string csn) : base(new java.io.FileOutputStream
(file))
{
if (csn == null)
{
throw new System.ArgumentNullException();
}
if (!java.nio.charset.Charset.isSupported(csn))
{
throw new java.io.UnsupportedEncodingException(csn);
}
encoding = csn;
}
private void loadGroovyTargetFile(java.io.File file)
{
java.lang.Class clazz;
try
{
clazz = _groovyClassManager.parseClass(file);
}
catch (java.lang.reflect.InvocationTargetException e)
{
throw new BuildException("File " + file.getAbsolutePath() + " did not compile correctly.", e);
}
loadGroovyTargetsFromClass(clazz);
loadGroovyTasksFromClass(clazz, file);
}
public static long getLength(object @this, java.io.File file)
{
var path = GetPath(file);
try
{
using (var stream = File.Open(path, FileMode.Open, FileAccess.Read))
return(stream.Length);
}
catch (IOException)
{
return(0);
}
}
public void CanLoadTypeChecker()
{
var log = LogManager.Create("repo");
var repos = new java.util.ArrayList();
var mgr = CeylonUtils.makeRepositoryManager(repos, OutRepoLocation, log);
var tcb = new TypeCheckerBuilder();
tcb.verbose(true);
tcb.setRepositoryManager(mgr);
var srcFile=new java.io.File("hello.ceylon");
tcb.addSrcDirectory(srcFile);
var typeChecker= tcb.getTypeChecker();
typeChecker.process();
}
public JsonResult<Dictionary<string, string>> Categorize(string jsonContent)
{
Dictionary<string, string> docValues = JsonConvert.DeserializeObject<Dictionary<string, string>>(jsonContent);
string text = docValues.ContainsKey("Text") ? docValues["Text"] : docValues["text"];
//MODEL CONFIGURATION
string modulesConfig, inputRawFile, classLabelsOutputFileName, featuresDictOutputFile, modelOutputFileName, libSvmOutputFileName;
SolverType liblinearSolver;
double liblinearC, liblinearEps;
int minFeaturesFrequency;
bool normalize;
ScaleRange scaleRange;
ModelConfiguration.SetParams(out modulesConfig, out inputRawFile, out classLabelsOutputFileName, out featuresDictOutputFile, out modelOutputFileName, out libSvmOutputFileName, out liblinearSolver, out liblinearC, out liblinearEps, out minFeaturesFrequency, out normalize, out scaleRange);
//LOAD PIPELINE
List<FeatureExtractionModule> modules = PipelineConfiguration.GetExtractionModules();
FeatureExtractionPipeline pipeline = PipelineConfiguration.BuildPipeline(modulesConfig, modules);
//load features
FeatureStatisticsDictionaryBuilder featureStatisticsDictBuilder = new FeatureStatisticsDictionaryBuilder();
featureStatisticsDictBuilder.LoadFromFile(featuresDictOutputFile);
//load labels
var classLabels = LexiconReaderHelper.LoadDictionaryFromFile(classLabelsOutputFileName);
//load model
var modelFileLoad = new java.io.File(modelOutputFileName);
var modelLoaded = Model.load(modelFileLoad);
SparseItemInt sparseItem = ProcessingHelpers.ProcessTextAndGetSparseItem(pipeline, featureStatisticsDictBuilder, minFeaturesFrequency, normalize, scaleRange, text, 0);
var itemFeatureNodes = ProcessingHelpers.ConvertToSortedFeatureNodeArray(sparseItem.Features);
//predict
double prediction = Linear.predict(modelLoaded, itemFeatureNodes);
int label = (int)prediction;
string labelName = "unknown";
if (classLabels.ContainsValue(label))
{
labelName = classLabels.ToList().Where(k => k.Value == label).Select(k => k.Key).FirstOrDefault();
}
docValues["LabelName"] = labelName;
docValues["Label"] = (prediction).ToString();
docValues["Confidence"] = (0.00).ToString();
return Json(docValues);
}
// Don't allow others to instantiate this class
/// <summary>Scan a file for OBB information.</summary>
/// <remarks>Scan a file for OBB information.</remarks>
/// <param name="filePath">path to the OBB file to be scanned.</param>
/// <returns>ObbInfo object information corresponding to the file path</returns>
/// <exception cref="System.ArgumentException">if the OBB file couldn't be found</exception>
/// <exception cref="System.IO.IOException">if the OBB file couldn't be read</exception>
public static android.content.res.ObbInfo getObbInfo(string filePath)
{
if (filePath == null)
{
throw new System.ArgumentException("file path cannot be null");
}
java.io.File obbFile = new java.io.File(filePath);
if (!obbFile.exists())
{
throw new System.ArgumentException("OBB file does not exist: " + filePath);
}
string canonicalFilePath = obbFile.getCanonicalPath();
android.content.res.ObbInfo obbInfo = new android.content.res.ObbInfo();
obbInfo.filename = canonicalFilePath;
getObbInfo_native(canonicalFilePath, obbInfo);
return obbInfo;
}
// called by?
public static string __GetFullPath(string e)
{
// http://www.devx.com/tips/Tip/13804
var f = new java.io.File(e);
var c = default(string);
try
{
c = f.getCanonicalPath();
}
catch
{
throw;
}
return c;
}
public static Type ToType(this string jar, string TypeFullName)
{
Type t = null;
try
{
var url = new java.io.File(jar).toURL();
var clazzLoader = new InternalURLClassLoader(new URL[] { url }, null);
var c = clazzLoader.loadClass(TypeFullName);
t = c.ToType();
}
catch
{
}
return t;
}
public void CanInheritVisitor()
{
var log = LogManager.Create("repo");
var repos = new java.util.ArrayList();
var mgr = CeylonUtils.makeRepositoryManager(repos, OutRepoLocation, log);
var tcb = new TypeCheckerBuilder();
tcb.verbose(true);
tcb.setRepositoryManager(mgr);
var srcFile = new java.io.File("hello.ceylon");
tcb.addSrcDirectory(srcFile);
var tc = tcb.getTypeChecker();
tc.process();
var v = new TestVisitor(log);
foreach(var unit in tc.getPhasedUnits().getPhasedUnits().Enumerate<PhasedUnit>())
{
unit.getCompilationUnit().visit(v);
}
}
private SSLSocketFactory getSSLSocketFactory()
{
SSLSocketFactory factory = null;
try
{
//reading the keyStore path and password from the environment properties
string keyStorePath = java.lang.System.getProperty("javax.net.ssl.keyStore");
java.io.FileInputStream keyStoreStream = null;
if (keyStorePath != null)
{
java.io.File file = new java.io.File(keyStorePath);
if(file.exists())
keyStoreStream = new java.io.FileInputStream(file);
else
keyStoreStream = searchDefaultCacerts();
}
else
keyStoreStream = searchDefaultCacerts();
string keyStorePassWord = java.lang.System.getProperty("javax.net.ssl.keyStorePassword");
if (keyStorePassWord == null)
keyStorePassWord = "******";
char[] passphrase = keyStorePassWord.ToCharArray();
//initiating SSLContext
SSLContext ctx = SSLContext.getInstance("TLS");
KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
KeyStore ks = KeyStore.getInstance("JKS");
if (keyStoreStream != null)
ks.load(keyStoreStream,passphrase);
else
ks.load(null,null);
kmf.init(ks, passphrase);
tmf.init(ks);
ctx.init(kmf.getKeyManagers(), tmf.getTrustManagers(), null);
factory = ctx.getSocketFactory();
}
catch (Exception e)
{
factory = null;
#if DEBUG
Console.WriteLine("Can't get SSL Socket Factory, the exception is {0}, {1}", e.GetType(), e.Message);
#endif
}
return factory;
}
public static string InternalGetHintPath()
{
var ff = "";
try
{
var ct = typeof(CPtr);
var c = ct.ToClass();
var url = c.getResource("");
var path = url.ToString();
// jar:file:/W:x/bin/Debug/staging/web/bin/x.__interface.__delegate.clr.dll!/EstEIDPerso/ChipXpressPlugin/
if (path.StartsWith("jar:file:"))
{
path = path.Substring("jar:file:".Length);
if (path.StartsWith("/"))
path = path.Substring(1);
path = path.Substring(0, path.IndexOf("!"));
}
ff = new java.io.File(path).getCanonicalPath();
}
catch (csharp.ThrowableException ex)
{
throw new System.InvalidOperationException("InternalGetHintPath");
}
return Path.GetDirectoryName(ff);
}
public void WebMethod2(string e, Action<string, string> y)
{
// https://github.com/drewnoakes/metadata-extractor.git
// https://sites.google.com/a/jsc-solutions.net/work/knowledge-base/15-dualvr/20150722
// jsc, on android how do I get the exif thumbnail of the pictures?
// firs let make sure we have .android referenced
var DIRECTORY_DCIM = global::android.os.Environment.DIRECTORY_DCIM;
var path = global::android.os.Environment.getExternalStoragePublicDirectory(DIRECTORY_DCIM).getAbsolutePath();
path += "/Camera";
var s = new Stopwatch();
s.Start();
var files0 =
from fname in Directory.GetFiles(path)
let f = new FileInfo(fname)
orderby f.LastWriteTime descending
select f;
var dir = new java.io.File(path);
Console.WriteLine("before listFiles " + new { s.Elapsed });
var files = files0.Take(350).ToArray();
Console.WriteLine("after listFiles " + new { s.Elapsed });
var sw = Stopwatch.StartNew();
files.WithEach(
ff =>
{
//y(f.getName());
//I/System.Console(20875): 518b:ab3a { Name = /storage/emulated/0/DCIM/Camera/20150430_183822.jpg }
//I/System.Console(20875): 518b:ab3a { Name = /storage/emulated/0/DCIM/Camera/20150430_183825.mp4 }
var f = new java.io.File(ff.FullName);
Console.WriteLine(new { sw.ElapsedMilliseconds, ff.Name, ff.Length });
if (ff.Extension != ".jpg")
//if (!ff.Name.EndsWith(".jpg"))
if (!ff.Name.EndsWith(".png"))
return;
// I/System.Console(31633): 7b91:b252 { Name = /storage/emulated/0/DCIM/Camera/20150721_143000.jpg, dir = JPEG, tag = Image Height }
//I/System.Console(31633): 7b91:b252 { Name = /storage/emulated/0/DCIM/Camera/20150721_143000.jpg, dir = JPEG, tag = Image Width }
try
{
// http://stackoverflow.com/questions/8578441/can-the-android-sdk-work-with-jdk-1-7
// http://stackoverflow.com/questions/15848332/does-adt-support-java-7-api
// https://code.google.com/p/android/issues/detail?id=22970
// [dx] trouble processing:
//[dx] bad class file magic (cafebabe) or version (0033.0000)
//[dx] ...while parsing com/drew/imaging/bmp/BmpMetadataReader.class
//[dx] ...while processing com/drew/imaging/bmp/BmpMetadataReader.class
//{ tag = [Exif Thumbnail] Thumbnail Compression - JPEG (old-style) }
//{ tag = [Exif Thumbnail] Orientation - Top, left side (Horizontal / normal) }
//{ tag = [Exif Thumbnail] X Resolution - 72 dots per inch }
//{ tag = [Exif Thumbnail] Y Resolution - 72 dots per inch }
//{ tag = [Exif Thumbnail] Resolution Unit - Inch }
//{ tag = [Exif Thumbnail] Thumbnail Offset - 1292 bytes }
//{ tag = [Exif Thumbnail] Thumbnail Length - 9092 bytes }
// http://drewnoakes.com/code/exif/
var m = ImageMetadataReader.readMetadata(f);
// http://stackoverflow.com/questions/10166373/metadata-extraction-java
var src = default(string);
var ButtonTitle = ff.Name;
int w = 0;
int h = 0;
// I/System.Console( 814): 032e:b35d { Name = /storage/emulated/0/DCIM/Camera/20150708_105032.jpg, dir = Xmp, tag = XMP Value Count }
var cJpegDirectory = typeof(com.drew.metadata.jpeg.JpegDirectory).ToClass();
if (m.containsDirectoryOfType(cJpegDirectory))
{
var x = (com.drew.metadata.jpeg.JpegDirectory)m.getFirstDirectoryOfType(cJpegDirectory);
w = x.getImageWidth();
h = x.getImageHeight();
ButtonTitle += " " + w + ":" + h;
//if (w < 6000)
// return;
// http://exsight360.com/blog/how-to-upload-non-android-360-panoramas-to-google-maps/
var cXmpDirectory = typeof(com.drew.metadata.xmp.XmpDirectory).ToClass();
if (m.containsDirectoryOfType(cXmpDirectory))
{
var xmp = (com.drew.metadata.xmp.XmpDirectory)m.getFirstDirectoryOfType(cXmpDirectory);
var p = xmp.getXmpProperties();
var equirectangular = ((string)p.get("GPano:ProjectionType")) == "equirectangular";
if (equirectangular)
{
// link it into vr..
// http://androidwarzone.blogspot.com/2012/03/creating-symbolic-links-on-android-from.html
// or just copy?
//ff.CopyTo("sdcard/oculus/360Photos/" + ff.Name, true);
//java.nio.file.Fi
//java.io.File.cop
// http://stackoverflow.com/questions/106770/standard-concise-way-to-copy-a-file-in-java
// http://docs.oracle.com/javase/7/docs/api/java/nio/file/Files.html
var copysw = Stopwatch.StartNew();
// Caused by: java.lang.RuntimeException: /sdcard/oculus/360Photos/storage/emulated/0/DCIM/Camera/20150710_202301.jpg: open failed: ENOENT (No such file or directory)
//I/System.Console(17759): 455f:bd8c InternalReadAllBytes { path = /storage/emulated/0/DCIM/Camera/20150624_120258.jpg }
//I/System.Console(17759): 455f:bd8c { copysw = 00:00:00.19.0, target = /sdcard/oculus/360Photos//storage/emulated/0/DCIM/Camera/20150624_120258.jpg }
// I/System.Console(20294): 4f46:bf53 GetFileName /storage/emulated/0/DCIM/Camera/20150624_120258.jpg
//Console.WriteLine("GetFileName " + Path.GetFileName(ff.FullName));
var target = "/sdcard/oculus/360Photos/" + ff.Name;
// 20ms
var bytes = File.ReadAllBytes(ff.FullName);
File.WriteAllBytes(target, bytes);
Console.WriteLine(new { copysw, target });
//f.co.CopyTo("sdcard/oculus/360Photos/" + ff.Name, true);
}
ButtonTitle += new { equirectangular };
Console.WriteLine(new { ButtonTitle });
// http://search.cpan.org/dist/Image-ExifTool-9.76/lib/Image/ExifTool/TagNames.pod#XMP_GPano_Tags
// http://www.panotwins.de/technical/how-to-add-mandatory-photo-sphere-meta-data-to-an-equirectangular-image/
// public virtual XMPMeta getXMPMeta();
//public virtual Map getXmpProperties();
//var xmpmeta = xmp.getXMPMeta();
//xmpmeta.dumpObject();
var pk = p.keySet();
var pka = pk.toArray();
for (int i = 0; i < pka.Length; i++)
{
var key = pka[i];
Console.WriteLine(key + ": " + p.get(key));
}
//I/System.Console(10680): 29b8:b8f0 { ButtonTitle = /storage/emulated/0/DCIM/Camera/20150708_110444.jpg 6582:3290, i = 0, key = GPano:ProjectionType, value = equirectangular }
//I/System.Console(10680): 29b8:b8f0 { ButtonTitle = /storage/emulated/0/DCIM/Camera/20150708_110444.jpg 6582:3290, i = 1, key = GPano:LargestValidInteriorRectTop, value = 0 }
//I/System.Console(10680): 29b8:b8f0 { ButtonTitle = /storage/emulated/0/DCIM/Camera/20150708_110444.jpg 6582:3290, i = 2, key = GPano:CroppedAreaLeftPixels, value = 0 }
//I/System.Console(10680): 29b8:b8f0 { ButtonTitle = /storage/emulated/0/DCIM/Camera/20150708_110444.jpg 6582:3290, i = 3, key = GPano:CroppedAreaTopPixels, value = 0 }
//I/System.Console(10680): 29b8:b8f0 { ButtonTitle = /storage/emulated/0/DCIM/Camera/20150708_110444.jpg 6582:3290, i = 4, key = GPano:FullPanoHeightPixels, value = 3290 }
//I/System.Console(10680): 29b8:b8f0 { ButtonTitle = /storage/emulated/0/DCIM/Camera/20150708_110444.jpg 6582:3290, i = 5, key = GPano:CroppedAreaImageHeightPixels, value = 3290 }
//I/System.Console(10680): 29b8:b8f0 { ButtonTitle = /storage/emulated/0/DCIM/Camera/20150708_110444.jpg 6582:3290, i = 6, key = GPano:LargestValidInteriorRectLeft, value = 0 }
//I/System.Console(10680): 29b8:b8f0 { ButtonTitle = /storage/emulated/0/DCIM/Camera/20150708_110444.jpg 6582:3290, i = 7, key = GPano:FullPanoWidthPixels, value = 6582 }
//I/System.Console(10680): 29b8:b8f0 { ButtonTitle = /storage/emulated/0/DCIM/Camera/20150708_110444.jpg 6582:3290, i = 8, key = GPano:LargestValidInteriorRectHeight, value = 3290 }
//I/System.Console(10680): 29b8:b8f0 { ButtonTitle = /storage/emulated/0/DCIM/Camera/20150708_110444.jpg 6582:3290, i = 9, key = GPano:FirstPhotoDate, value = 2015-07-08T08:04:44.035Z }
//I/System.Console(10680): 29b8:b8f0 { ButtonTitle = /storage/emulated/0/DCIM/Camera/20150708_110444.jpg 6582:3290, i = 10, key = GPano:CroppedAreaImageWidthPixels, value = 6582 }
//I/System.Console(10680): 29b8:b8f0 { ButtonTitle = /storage/emulated/0/DCIM/Camera/20150708_110444.jpg 6582:3290, i = 11, key = GPano:UsePanoramaViewer, value = True }
//I/System.Console(10680): 29b8:b8f0 { ButtonTitle = /storage/emulated/0/DCIM/Camera/20150708_110444.jpg 6582:3290, i = 12, key = GPano:LargestValidInteriorRectWidth, value = 6582 }
//I/System.Console(10680): 29b8:b8f0 { ButtonTitle = /storage/emulated/0/DCIM/Camera/20150708_110444.jpg 6582:3290, i = 13, key = GPano:LastPhotoDate, value = 2015-07-08T08:06:40.751Z }
//I/System.Console(10680): 29b8:b8f0 { ButtonTitle = /storage/emulated/0/DCIM/Camera/20150708_110444.jpg 6582:3290, i = 14, key = GPano:PoseHeadingDegrees, value = 300.4 }
//I/System.Console(10680): 29b8:b8f0 { ButtonTitle = /storage/emulated/0/DCIM/Camera/20150708_110444.jpg 6582:3290, i = 15, key = GPano:SourcePhotosCount, value = 64 }
//I/System.Console(10680): 29b8:b8f0 { ElapsedMilliseconds = 2442, Name = /storage/emulated/0/DCIM/Camera/20150708_110137.jpg, Length = 5759881 }
}
// NDK implement JDK implement CLR?
#region ExifThumbnailDirectory
var cExifThumbnailDirectory = typeof(com.drew.metadata.exif.ExifThumbnailDirectory).ToClass();
var containsDirectoryOfType = m.containsDirectoryOfType(cExifThumbnailDirectory);
if (containsDirectoryOfType)
{
var xExifThumbnailDirectory = (com.drew.metadata.exif.ExifThumbnailDirectory)m.getFirstDirectoryOfType(cExifThumbnailDirectory);
// in 3sec load thumbnails...
if (sw.ElapsedMilliseconds < 3000)
{
//Console.WriteLine(
// f.getName()
//);
var data = (byte[])(object)xExifThumbnailDirectory.getThumbnailData();
//Console.WriteLine(new { data });
if (data != null)
{
src = "data:image/jpg;base64," +
Convert.ToBase64String(
data
);
}
}
}
#endregion
y(
ButtonTitle,
src
);
}
// we have static memory. on android..
__inspect[ButtonTitle] = delegate
{
//var m = ImageMetadataReader.readMetadata(f);
// http://stackoverflow.com/questions/10166373/metadata-extraction-java
//var t = typeof(com.drew.metadata.exif.ExifThumbnailDirectory).ToClass();
//com.drew.metadata.exif.ExifThumbnailDirectory.
var i = m.getDirectories().iterator();
while (i.hasNext())
{
var directory = (com.drew.metadata.Directory)i.next();
var tags = directory.getTags().toArray();
foreach (com.drew.metadata.Tag tag in tags)
{
// https://developers.google.com/photo-sphere/metadata/
// GPano:ProjectionType
// -xmp:UsePanoramaViewer=True
Console.WriteLine(new { ff.Name, dir = directory.getName(), tag = tag.getTagName() });
// I/System.Console( 7296): 1c80:b6bc { Name = /storage/emulated/0/DCIM/Camera/20150708_133445.jpg, dir = Xmp, tag = XMP Value Count }
if (directory.getName() == "Xmp")
{
//directory as Xmp
}
//tag.
//y(new { tag }.ToString());
}
}
};
}
catch
{
// Caused by: java.lang.RuntimeException: File format is not supported
throw;
}
}
);
s.Stop();
Console.WriteLine(
new { s.Elapsed }
);
}
private java.io.File getPreferencesDir()
{
lock (mSync)
{
if (mPreferencesDir == null)
{
mPreferencesDir = new java.io.File(getDataDirFile(), "shared_prefs");
}
return mPreferencesDir;
}
}
public static void LoadAndRegisterAssemblyFrom(File assemblyFile, ClassLoader classLoader)
{
string assemblyPath = new Uri(assemblyFile.getCanonicalFile().toURI().toString()).LocalPath;
Assembly assembly;
if (System.IO.File.Exists(assemblyPath))
{
assembly = Assembly.LoadFrom(assemblyPath);
}
else
{
string current = Path.Combine(homeDir, assemblyPath);
if (System.IO.File.Exists(current))
{
assembly = Assembly.LoadFrom(current);
}
else
{
throw new FileNotFoundException(assemblyPath);
}
}
if (knownAssemblies.ContainsKey(assembly))
{
if (Setup.Verbose)
{
Console.WriteLine("skipped " + assembly + " from " + assembly.Location);
}
return;
}
if (Setup.Verbose)
{
Console.WriteLine("loading " + assembly + " from " + assembly.Location);
}
knownAssemblies.Add(assembly, assembly);
Registry.RegisterAssembly(assembly, true, classLoader);
if (Setup.Verbose)
{
Console.WriteLine("loaded " + assembly + " from " + assembly.Location);
}
}
private java.io.FileInputStream searchDefaultCacerts()
{
try
{
string javaHome = java.lang.System.getProperty("java.home");
if(javaHome == null)
return null;
string keyStorePath = javaHome + "/lib/security/cacerts";
//Console.WriteLine("keyStorePath = {0}", keyStorePath);
java.io.File f = new java.io.File(keyStorePath);
if(!f.exists())
return null;
return new java.io.FileInputStream(f);
}
catch(Exception e)
{
#if DEBUG
//todo log it
Console.WriteLine(e.GetType() + ":" + e.Message + "\n" + e.StackTrace);
#endif
return null;
}
}
static void Main(string[] args)
{
string modulesConfig = "annotate_words,plain_bow,nsuff_3,chngram_3,word2gram,doc_end";
if (args.Length > 1)
{
modulesConfig = args[1];
}
//Process input file
string inputFile = "data\troll-comments.txt";
if (args.Length > 0)
{
inputFile = args[0];
}
//Define feature extraction modules
#region 1 - Define all possible feature extraction modules
List<FeatureExtractionModule> modules = new List<FeatureExtractionModule>();
//Extract words from text and add them as annotations for later use
modules.Add(new ActionFeatureExtractionModule("annotate_words", (text, features, annotations) =>
{
FeatureExtractionNlpHelpers.TokenizeAndAppendAnnotations(text, annotations);
var wordAnnotations = annotations.Where(a => a.Type == "Word").ToList();
foreach (var wordAnn in wordAnnotations)
{
var loweredWordAnnotation = new Annotation() { Type = "Word_Lowered", Text = wordAnn.Text.ToLower(), FromIndex = wordAnn.FromIndex, ToIndex = wordAnn.ToIndex };
annotations.Add(loweredWordAnnotation);
}
}));
modules.Add(new ActionFeatureExtractionModule("annotate_words_troll_sentence", (text, features, annotations) =>
{
List<Annotation> annotationsAll = new List<Annotation>();
string txt = text.Replace(",", " , ").Replace(";", " ; ").Replace("?", " ? ");
FeatureExtractionNlpHelpers.TokenizeAndAppendAnnotationsRegEx(txt, annotationsAll, @"([#\w,;?-]+)");
var annotationsTroll = annotationsAll.Where(a => a.Text.ToLower().StartsWith("трол") || a.Text.ToLower().StartsWith("мурзи")).ToList();
var annotationsTrollWindow = new List<Annotation>();
int wordScope = 2;
foreach (var ann in annotationsTroll)
{
int trollIndex = annotationsAll.IndexOf(ann);
for (int i = Math.Max((trollIndex - wordScope), 0); i < Math.Min((trollIndex + wordScope), annotationsAll.Count); i++)
{
var annToAdd = annotationsAll[i];
if (!annotationsTrollWindow.Contains(annToAdd))
{
annotationsTrollWindow.Add(annToAdd);
}
}
}
foreach (var ann in annotationsTrollWindow)
{
annotations.Add(ann);
}
Console.WriteLine(string.Join(" ", annotations.Select(a => a.Text)));
var wordAnnotations = annotations.Where(a => a.Type == "Word").ToList();
foreach (var wordAnn in wordAnnotations)
{
var loweredWordAnnotation = new Annotation() { Type = "Word_Lowered", Text = wordAnn.Text.ToLower(), FromIndex = wordAnn.FromIndex, ToIndex = wordAnn.ToIndex };
annotations.Add(loweredWordAnnotation);
}
}));
modules.Add(new ActionFeatureExtractionModule("annotate_words_troll_words", (text, features, annotations) =>
{
List<Annotation> annotationsAll = new List<Annotation>();
string txt = text.Replace(",", " , ").Replace(";", " ; ").Replace("?", " ? ");
FeatureExtractionNlpHelpers.TokenizeAndAppendAnnotationsRegEx(txt, annotationsAll, @"([#\w,;?-]+)");
List<string> allowedWords = new List<string>()
{
"аз","ти","той","тя","то","ние","вие","те",
"съм","си","е","сте","са","сме",
"ми","ни","ви","им","му",
"нас","вас","тях",
"ги", "го", "я"
//"коя","кое","кои","кой",
};
annotationsAll = annotationsAll.Where(a => a.Text.ToLower().StartsWith("трол") || a.Text.ToLower().StartsWith("мурзи") || allowedWords.Contains(a.Text.ToLower())).ToList();
foreach (var ann in annotationsAll)
{
annotations.Add(ann);
}
Debug.WriteLine(string.Join(" ", annotations.Select(a => a.Text)));
var wordAnnotations = annotations.Where(a => a.Type == "Word").ToList();
foreach (var wordAnn in wordAnnotations)
{
var loweredWordAnnotation = new Annotation() { Type = "Word_Lowered", Text = wordAnn.Text.ToLower(), FromIndex = wordAnn.FromIndex, ToIndex = wordAnn.ToIndex };
annotations.Add(loweredWordAnnotation);
}
}));
//Generate bag of words features
modules.Add(new ActionFeatureExtractionModule("plain_bow", (text, features, annotations) =>
{
var wordAnnotations = annotations.Where(a => a.Type == "Word_Lowered").ToList();
foreach (var wordAnn in wordAnnotations)
{
FeaturesDictionaryHelpers.IncreaseFeatureFrequency(features, "plain_bow_" + wordAnn.Text.ToLower(), 1.0);
}
}));
//Generate word prefixes from all lowered word tokens
int[] charPrefixLengths = new int[] { 2, 3, 4 };
foreach (var charPrefLength in charPrefixLengths)
{
modules.Add(new ActionFeatureExtractionModule("npref_" + charPrefLength, (text, features, annotations) =>
{
var wordAnnotations = annotations.Where(a => a.Type == "Word_Lowered").ToList();
foreach (var wordAnn in wordAnnotations)
{
FeatureExtractionNlpHelpers.ExtractPrefixFeatureFromSingleTokenAndUpdateItemFeatures(features, wordAnn.Text.ToLower(), charPrefLength);
}
}));
}
//Generate word suffixes from all lowered word tokens
int[] charSuffixLengths = new int[] { 2, 3, 4 };
foreach (var charSuffLength in charSuffixLengths)
{
modules.Add(new ActionFeatureExtractionModule("nsuff_" + charSuffLength, (text, features, annotations) =>
{
var wordAnnotations = annotations.Where(a => a.Type == "Word_Lowered").ToList();
foreach (var wordAnn in wordAnnotations)
{
FeatureExtractionNlpHelpers.ExtractSuffixFeatureFromSingleTokenAndUpdateItemFeatures(features, wordAnn.Text.ToLower(), charSuffLength);
}
}));
}
//Generate word character ngrams
int[] charNGramLengths = new int[] { 2, 3, 4 };
foreach (var charNGramLength in charNGramLengths)
{
modules.Add(new ActionFeatureExtractionModule("chngram_" + charNGramLength, (text, features, annotations) =>
{
var wordAnnotations = annotations.Where(a => a.Type == "Word_Lowered").ToList();
foreach (var wordAnn in wordAnnotations)
{
FeatureExtractionNlpHelpers.ExtractCharNgramFeaturesFromSingleTokenAndUpdateItemFeatures(features, wordAnn.Text.ToLower(), charNGramLength);
}
}));
}
//Generate Stemmed word features, using Bulstem(P.Nakov)
bool useStemmer = true;
Stemmer stemmer = useStemmer ? new Stemmer(StemmingLevel.Medium) : null;
modules.Add(new ActionFeatureExtractionModule("plain_word_stems", (text, features, annotations) =>
{
var wordAnnotations = annotations.Where(a => a.Type == "Word_Lowered").ToList();
foreach (var wordAnn in wordAnnotations)
{
FeatureExtractionNlpHelpers.ExtractStemFeatureFromSingleTokenAndUpdateItemFeatures(stemmer, features, wordAnn.Text.ToLower());
}
}));
//Generate Word Ngram features
int[] wordNgramLengths = new int[] { 2, 3, 4 };
foreach (var ngramLength in wordNgramLengths)
{
modules.Add(new ActionFeatureExtractionModule(string.Format("word{0}gram", ngramLength), (text, features, annotations) =>
{
var wordTokens = annotations.Where(a => a.Type == "Word_Lowered").Select(a => a.Text).ToList();
FeatureExtractionNlpHelpers.ExtractWordNGramFeaturesFromTextTokensAndUpdateItemFeatures(features, wordTokens, ngramLength);
}));
}
modules.Add(new ActionFeatureExtractionModule("count_punct", (text, features, annotations) =>
{
FeatureExtractionNlpHelpers.ExtractTextPunctuationFeaturesAndUpdateItemFeatures(text, features);
}));
modules.Add(new ActionFeatureExtractionModule("emoticons_dnevnikbg", (text, features, annotations) =>
{
FeatureExtractionNlpHelpers.ExtractDnevnikEmoticonsFeaturesAndUpdateItemFeatures(text, features);
}));
//Doc starts with
modules.Add(new ActionFeatureExtractionModule("doc_start", (text, features, annotations) =>
{
if (text.Length < 3)
{
return;
}
FeaturesDictionaryHelpers.IncreaseFeatureFrequency(features, "doc_starts_3_" + text.Substring(0, 3), 1.0);
if (text.Length < 4)
{
return;
}
FeaturesDictionaryHelpers.IncreaseFeatureFrequency(features, "doc_starts_4_" + text.Substring(0, 4), 1.0);
if (text.Length < 5)
{
return;
}
FeaturesDictionaryHelpers.IncreaseFeatureFrequency(features, "doc_starts_5_" + text.Substring(0, 5), 1.0);
}));
//Doc ends with
modules.Add(new ActionFeatureExtractionModule("doc_end", (text, features, annotations) =>
{
if (text.Length < 3)
{
return;
}
FeaturesDictionaryHelpers.IncreaseFeatureFrequency(features, "doc_ends_3_" + text.Substring(text.Length - 3, 3), 1.0);
if (text.Length < 4)
{
return;
}
FeaturesDictionaryHelpers.IncreaseFeatureFrequency(features, "doc_ends_4_" + text.Substring(text.Length - 4, 4), 1.0);
if (text.Length < 5)
{
return;
}
FeaturesDictionaryHelpers.IncreaseFeatureFrequency(features, "doc_ends_5_" + text.Substring(text.Length - 5, 5), 1.0);
}));
//custom module
//modules.Add(new ActionFeatureExtractionModule("", (text, features, annotations) => {
//}));
//Print possible module options
foreach (var module in modules)
{
Debug.Write(module.Name + ",");
}
#endregion
//configure which modules to use
#region 2 - Configure which module configurations
//string modulesConfig = "annotate_words,plain_bow,npref_2,npref_3,npref_4,nsuff_2,nsuff_3,nsuff_4,chngram_2,chngram_3,chngram_4,plain_word_stems,word2gram,word3gram, word4gram,count_punct,emoticons_dnevnikbg,doc_start,doc_end";
//string settingsConfig = "annotate_words,plain_word_stems, npref_4, nsuff_3";
Console.WriteLine("Module configurations:");
var moduleNamesToConfigure = modulesConfig.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
foreach (var moduleName in moduleNamesToConfigure)
{
Console.WriteLine(moduleName);
}
Console.WriteLine();
var modulesToRegister = modules.Where(m => moduleNamesToConfigure.Contains(m.Name)).ToList();
FeatureExtractionPipeline pipeline = new FeatureExtractionPipeline();
foreach (var module in modulesToRegister)
{
pipeline.RegisterModule(module);
}
#endregion
string classLabelsFileName = inputFile + ".classlabels";
string featuresDictFile = inputFile + ".features";
string libSvmFileName = inputFile + ".libsvm";
Console.WriteLine("Input file:{0}", inputFile);
//char fieldSeparator = '\t';
#region 3 - Build features dictionary - process documents and extract all possible features
//build features dictionary
var featureStatisticsDictBuilder = new FeatureStatisticsDictionaryBuilder();
Console.WriteLine("Building a features dictionary...");
var timeStart = DateTime.Now;
int itemsCnt = 0;
Dictionary<string, int> classLabels = new Dictionary<string, int>();
int maxClassLabelIndex = 0;
using (var filereader = new LabeledTextDocumentFileReader(inputFile))
{
while (!filereader.EndOfSource())
{
var doc = filereader.ReadDocument();
//class label and doc contents
string classLabel = doc.ClassLabel;
string docContent = doc.DocContent;
//build class labels dictionary
if (!classLabels.ContainsKey(classLabel))
{
classLabels[classLabel] = maxClassLabelIndex;
maxClassLabelIndex++;
}
Dictionary<string, double> docFeatures = new Dictionary<string, double>();
pipeline.ProcessDocument(docContent, docFeatures);
featureStatisticsDictBuilder.UpdateInfoStatisticsFromSingleDoc(docFeatures);
itemsCnt++;
if (itemsCnt % 500 == 0)
{
Console.WriteLine("{0} processed so far", itemsCnt);
}
}
}
//order classes by name - until now they are ordered by first occurence in dataset
var ordered = classLabels.OrderBy(cv => cv.Key);
var orderedClassLabels = new Dictionary<string, int>();
int classIndexCounter = 0;
foreach (var item in ordered)
{
orderedClassLabels.Add(item.Key, classIndexCounter);
classIndexCounter++;
}
classLabels = orderedClassLabels;
LexiconReaderHelper.SaveDictionaryToFile(classLabels, classLabelsFileName);
Console.WriteLine("Class labels saved to file {0}", classLabelsFileName);
Console.WriteLine("Extracted {0} features from {1} documents", featureStatisticsDictBuilder.FeatureInfoStatistics.Count, itemsCnt);
Console.WriteLine("Done - {0}", (DateTime.Now - timeStart));
int minFeaturesFrequency = 5;
RecomputeFeatureIndexes(featureStatisticsDictBuilder, minFeaturesFrequency);
Console.WriteLine("Selected {0} features with min freq {1} ", featureStatisticsDictBuilder.FeatureInfoStatistics.Count, minFeaturesFrequency);
//save fetures for later use
if (System.IO.File.Exists(featuresDictFile))
{
System.IO.File.Delete(featuresDictFile);
}
featureStatisticsDictBuilder.SaveToFile(featuresDictFile);
Console.WriteLine("Features saved to file {0}", featuresDictFile);
#endregion
//4- Load features from file
featureStatisticsDictBuilder.LoadFromFile(featuresDictFile);
classLabels = LexiconReaderHelper.LoadDictionaryFromFile(classLabelsFileName);
#region 5 - Build items with features from text documents and features dictionary
//Build libsvm file from text insput file and features dictionary
var sparseItemsWithIndexFeatures = new List<SparseItemInt>();
timeStart = DateTime.Now;
Console.WriteLine("Exporting to libsvm file format...");
bool normalize = false;
var scaleRange = ScaleRange.ZeroToOne;
using (System.IO.TextWriter textWriter = new System.IO.StreamWriter(libSvmFileName, false))
{
LibSvmFileBuilder libSvmFileBuilder = new LibSvmFileBuilder(textWriter);
using (var filereader = new LabeledTextDocumentFileReader(inputFile))
{
while (!filereader.EndOfSource())
{
var doc = filereader.ReadDocument();
//class label and doc contents
string classLabel = doc.ClassLabel;
string docContent = doc.DocContent;
Dictionary<string, double> docFeatures = new Dictionary<string, double>();
pipeline.ProcessDocument(docContent, docFeatures);
int classLabelIndex = classLabels[classLabel];
//Append extracted features
//A - Extracted indexed features
var itemIndexedFeatures = LibSvmFileBuilder.GetIndexedFeaturesFromStringFeatures(docFeatures, featureStatisticsDictBuilder.FeatureInfoStatistics, minFeaturesFrequency, normalize, scaleRange);
libSvmFileBuilder.AppendItem(classLabelIndex, itemIndexedFeatures);
sparseItemsWithIndexFeatures.Add(new SparseItemInt() { Label = classLabelIndex, Features = itemIndexedFeatures });
//B - Or extract indexed features and append
//libSvmFileBuilder.PreprocessStringFeaturesAndAppendItem(classLabelIndex, docFeatures, featureStatisticsDictBuilder.FeatureInfoStatistics, minFeaturesFrequency);
}
}
}
Console.WriteLine("Done - {0}", (DateTime.Now - timeStart));
Console.WriteLine("Libsvm file saved to {0}", libSvmFileName);
Console.WriteLine();
#endregion
#region 6 - Train and test classifier
//LIBLINEAR - TRAIN AND EVAL CLASSIFIER
//Build problem X and Y
var solver = SolverType.L1R_LR;//L2R_LR
var c = 1.0;
var eps = 0.01;
//Split data on train and test dataset
int trainRate = 4;
int testRate = 1;
FeatureNode[][] problemXTrain = null;
double[] problemYTrain = null;
int allItemsCnt = sparseItemsWithIndexFeatures.Count;
int trainCnt = (int)((double)allItemsCnt * trainRate / (double)(trainRate + testRate));
int testCnt = allItemsCnt - trainCnt;
var trainItems = sparseItemsWithIndexFeatures.Take(trainCnt).ToList();
var testItems = sparseItemsWithIndexFeatures.Skip(trainCnt).Take(testCnt).ToList();
SetLibLinearProblemXandYFromSparseItems(trainItems, out problemXTrain, out problemYTrain);
//Create liblinear problem
var problem = new Problem();
problem.l = problemXTrain.Length;
problem.n = featureStatisticsDictBuilder.FeatureInfoStatistics.Count;
//problem.x = new FeatureNode[][]{
// new FeatureNode[]{new FeatureNode(1,0.0), new FeatureNode(2,1.0)},
// new FeatureNode[]{new FeatureNode(1,2.0), new FeatureNode(2,0.0)}
//};
//problem.y = new double[] { 1, 2 };
problem.x = problemXTrain;
problem.y = problemYTrain;
Console.WriteLine("Training a classifier with {0} items...", sparseItemsWithIndexFeatures.Count);
timeStart = DateTime.Now;
var parameter = new Parameter(solver, c, eps);
var model = Linear.train(problem, parameter);
Console.WriteLine("Done - {0}", (DateTime.Now - timeStart));
string modelFileName = inputFile + ".train.model";
var modelFile = new java.io.File(modelFileName);
model.save(modelFile);
Console.WriteLine("Model saved to {0}", modelFileName);
var modelLoaded = Model.load(modelFile);
//evaluation
List<FeatureNode[]> problemXEvalList = new List<FeatureNode[]>();
List<double> problemYEvalList = new List<double>();
//SetLibLinearProblemXandYFromSparseItems(testItems, out problemXEval, out problemYEval);
List<double> predictedY = new List<double>();
foreach (var item in testItems)
{
var itemFeatureNodes = ConvertToSortedFeatureNodeArray(item.Features);
//fill eval list
problemXEvalList.Add(itemFeatureNodes);
problemYEvalList.Add((double)item.Label);
//predict
double prediction = Linear.predict(model, itemFeatureNodes);
predictedY.Add(prediction);
}
int[][] matrix = ResultCalculationMetricsHelpers.BuildConfusionMatrix(problemYEvalList.ToArray(), predictedY, classLabels.Count);
Console.WriteLine("Class labels:");
foreach (var label in classLabels)
{
Console.WriteLine(string.Format("{1} - {0}", label.Key, label.Value));
}
Console.WriteLine();
ResultCalculationMetricsHelpers.PrintMatrix(matrix, true);
for (int i = 0; i < matrix.Length; i++)
{
int truePositivesCnt = matrix[i][i];
int falsePositievesCnt = matrix[i].Sum() - matrix[i][i];
int falseNegativesCnt = matrix.Select(m => m[i]).Sum() - matrix[i][i];
double precision;
double recall;
double fScore;
ResultCalculationMetricsHelpers.CalculatePRF(truePositivesCnt, falsePositievesCnt, falseNegativesCnt, out precision, out recall, out fScore);
Console.WriteLine(string.Format("[{0} - {4}] P={1:0.0000}, R={2:0.0000}, F={3:0.0000} ", i, precision, recall, fScore, orderedClassLabels.ToList().ToDictionary(kv => kv.Value, kv => kv.Key)[i]));
}
int truePositivesCntOverall = 0;
int testedCnt = 0;
for (int i = 0; i < matrix.Length; i++)
{
truePositivesCntOverall += matrix[i][i];
testedCnt += matrix[i].Sum();
}
double accuracyOverall = (double)truePositivesCntOverall / (double)testedCnt;
Console.WriteLine(string.Format("[{0}] Accuracy={1:0.0000}", "Overall", accuracyOverall));
//CROSSVALIDATION
//int crossValidationFold = 5;
//Console.WriteLine("Training with {0} items and 5 fold crossvalidation...", sparseItemsWithIndexFeatures.Count);
//timeStart = DateTime.Now;
//double[] target = new double[problem.l];
//Linear.crossValidation(problem, parameter, crossValidationFold, target);
//Console.WriteLine("Done - {0}", (DateTime.Now - timeStart));
//WriteResult(target);
#endregion
//Console.ReadKey();
//var instancesToTest = new Feature[] { new FeatureNode(1, 0.0), new FeatureNode(2, 1.0) };
//var prediction = Linear.predict(model, instancesToTest);
//string docToProcess = "Оставете правителството да си работи на спокойствие!";
//SparseItemString docItem = new SparseItemString()
//{
// Label = 1,
//};
//pipeline.ProcessDocument(docToProcess, docItem.Features);
//foreach (var feature in docItem.Features)
//{
// Debug.WriteLine(string.Format("{0} - {1}", feature.Key, feature.Value));
//}
}
private static void TrainLibLinearProblemAndSveModel(SolverType liblinearSolver, double liblinearC, double liblinearEps, FeatureStatisticsDictionaryBuilder featureStatisticsDictBuilder, string trainDataModelFileName, FeatureNode[][] problemXTrain, double[] problemYTrain)
{
//Create liblinear problem
var problem = new Problem();
problem.l = problemXTrain.Length;
problem.n = featureStatisticsDictBuilder.FeatureInfoStatistics.Count;
problem.x = problemXTrain;
problem.y = problemYTrain;
Console.WriteLine("Training a classifier with {0} items...", problemXTrain.Length);
DateTime timeStart = DateTime.Now;
var parameter = new Parameter(liblinearSolver, liblinearC, liblinearEps);
Model model = Linear.train(problem, parameter);
Console.WriteLine("Done - {0}", (DateTime.Now - timeStart));
var modelFile = new java.io.File(trainDataModelFileName);
model.save(modelFile);
Console.WriteLine("Train data model saved to {0}", trainDataModelFileName);
}
protected override Local create(File path)
{
return new LocalImpl(path);
}
public static void LoadAndRegisterAssemblyFromClassLoader(File assemblyFile, ClassLoader classLoader)
{
string assemblyPath = new Uri(assemblyFile.getCanonicalFile().toURI().toString()).LocalPath;
Assembly assembly;
if (System.IO.File.Exists(assemblyPath))
{
assembly = Assembly.LoadFrom(assemblyPath);
}
else
{
string current = Path.Combine(homeDir, assemblyPath);
if (System.IO.File.Exists(current))
{
assembly = Assembly.LoadFrom(current);
}
else
{
throw new FileNotFoundException(assemblyPath);
}
}
RegisterAssembly(assembly, classLoader);
}
public static void LoadAndRegisterAssemblyFrom(File assemblyFile)
{
LoadAndRegisterAssemblyFromClassLoader(assemblyFile, null);
}
public JsonResult<Dictionary<string, string>> CategorizeTextDoc(Dictionary<string, string> docValues)
{
//Load model configuration
string modulesConfig, inputRawFile, classLabelsOutputFileName, featuresDictOutputFile, modelOutputFileName, libSvmOutputFileName;
SolverType liblinearSolver;
double liblinearC, liblinearEps;
int minFeaturesFrequency;
bool normalize;
ScaleRange scaleRange;
ModelConfiguration.SetParams(out modulesConfig, out inputRawFile, out classLabelsOutputFileName, out featuresDictOutputFile, out modelOutputFileName, out libSvmOutputFileName, out liblinearSolver, out liblinearC, out liblinearEps, out minFeaturesFrequency, out normalize, out scaleRange);
//Load pipeline
List<FeatureExtractionModule> modules = PipelineConfiguration.GetExtractionModules();
FeatureExtractionPipeline pipeline = PipelineConfiguration.BuildPipeline(modulesConfig, modules);
//Load features
FeatureStatisticsDictionaryBuilder featureStatisticsDictBuilder = new FeatureStatisticsDictionaryBuilder();
featureStatisticsDictBuilder.LoadFromFile(featuresDictOutputFile);
//Load labels
var classLabels = LexiconReaderHelper.LoadDictionaryFromFile(classLabelsOutputFileName);
//Load model
var modelFileLoad = new java.io.File(modelOutputFileName);
var modelLoaded = Model.load(modelFileLoad);
//Categorize single item
string text = docValues.ContainsKey("Text") ? docValues["Text"] : docValues["text"];
double prediction, confidence;
int label;
string labelName;
CategorizeText(text, minFeaturesFrequency, normalize, scaleRange, pipeline, featureStatisticsDictBuilder, classLabels, modelLoaded, out prediction, out label, out labelName, out confidence);
docValues["LabelName"] = labelName;
docValues["Label"] = (prediction).ToString();//label might also be used for categorization
docValues["Confidence"] = (confidence).ToString();
return Json(docValues);
}
/// <summary>
/// Retrieve overall information about an application package defined
/// in a package archive file
/// </summary>
/// <param name="archiveFilePath">The path to the archive file</param>
/// <param name="flags">
/// Additional option flags. Use any combination of
/// <see cref="GET_ACTIVITIES">GET_ACTIVITIES</see>
/// ,
/// <see cref="GET_GIDS">GET_GIDS</see>
/// ,
/// <see cref="GET_CONFIGURATIONS">GET_CONFIGURATIONS</see>
/// ,
/// <see cref="GET_INSTRUMENTATION">GET_INSTRUMENTATION</see>
/// ,
/// <see cref="GET_PERMISSIONS">GET_PERMISSIONS</see>
/// ,
/// <see cref="GET_PROVIDERS">GET_PROVIDERS</see>
/// ,
/// <see cref="GET_RECEIVERS">GET_RECEIVERS</see>
/// ,
/// <see cref="GET_SERVICES">GET_SERVICES</see>
/// ,
/// <see cref="GET_SIGNATURES">GET_SIGNATURES</see>
/// , to modify the data returned.
/// </param>
/// <returns>
/// Returns the information about the package. Returns
/// null if the package could not be successfully parsed.
/// </returns>
/// <seealso cref="GET_ACTIVITIES">GET_ACTIVITIES</seealso>
/// <seealso cref="GET_GIDS">GET_GIDS</seealso>
/// <seealso cref="GET_CONFIGURATIONS">GET_CONFIGURATIONS</seealso>
/// <seealso cref="GET_INSTRUMENTATION">GET_INSTRUMENTATION</seealso>
/// <seealso cref="GET_PERMISSIONS">GET_PERMISSIONS</seealso>
/// <seealso cref="GET_PROVIDERS">GET_PROVIDERS</seealso>
/// <seealso cref="GET_RECEIVERS">GET_RECEIVERS</seealso>
/// <seealso cref="GET_SERVICES">GET_SERVICES</seealso>
/// <seealso cref="GET_SIGNATURES">GET_SIGNATURES</seealso>
public virtual android.content.pm.PackageInfo getPackageArchiveInfo(string archiveFilePath
, int flags)
{
android.content.pm.PackageParser packageParser = new android.content.pm.PackageParser
(archiveFilePath);
android.util.DisplayMetrics metrics = new android.util.DisplayMetrics();
metrics.setToDefaults();
java.io.File sourceFile = new java.io.File(archiveFilePath);
android.content.pm.PackageParser.Package pkg = packageParser.parsePackage(sourceFile
, archiveFilePath, metrics, 0);
if (pkg == null)
{
return null;
}
if ((flags & GET_SIGNATURES) != 0)
{
packageParser.collectCertificates(pkg, 0);
}
return android.content.pm.PackageParser.generatePackageInfo(pkg, null, flags, 0,
0);
}
public override java.io.File getExternalFilesDir(string type)
{
lock (mSync)
{
if (mExternalFilesDir == null)
{
mExternalFilesDir = android.os.Environment.getExternalStorageAppFilesDirectory(getPackageName
());
}
if (!mExternalFilesDir.exists())
{
try
{
(new java.io.File(android.os.Environment.getExternalStorageAndroidDataDir(), ".nomedia"
)).createNewFile();
}
catch (System.IO.IOException)
{
}
if (!mExternalFilesDir.mkdirs())
{
android.util.Log.w(TAG, "Unable to create external files directory");
return null;
}
}
if (type == null)
{
return mExternalFilesDir;
}
java.io.File dir = new java.io.File(mExternalFilesDir, type);
if (!dir.exists())
{
if (!dir.mkdirs())
{
android.util.Log.w(TAG, "Unable to create external media directory " + dir);
return null;
}
}
return dir;
}
}
public static void Train(string modulesConfig, string inputRawFile, SolverType liblinearSolver, double liblinearC, double liblinearEps,
string classLabelsOutputFileName, string featuresDictOutputFile, string modelOutputFileName, string libSvmOutputFileName,
int minFeaturesFrequency,bool normalize, ScaleRange scaleRange)
{
//Define feature extraction modules
#region 1 - Define all possible feature extraction modules
List<FeatureExtractionModule> modules = PipelineConfiguration.GetExtractionModules();
//Print possible module options
foreach (var module in modules)
{
Debug.Write(module.Name + ",");
}
#endregion
//configure which modules to use
#region 2 - Configure which module configurations
//string modulesConfig = "annotate_words,plain_bow,npref_2,npref_3,npref_4,nsuff_2,nsuff_3,nsuff_4,chngram_2,chngram_3,chngram_4,plain_word_stems,word2gram,word3gram, word4gram,count_punct,emoticons_dnevnikbg,doc_start,doc_end";
//string settingsConfig = "annotate_words,plain_word_stems, npref_4, nsuff_3";
FeatureExtractionPipeline pipeline = PipelineConfiguration.BuildPipeline(modulesConfig, modules);
#endregion
Console.WriteLine("Input file:{0}", inputRawFile);
//char fieldSeparator = '\t';
#region 3 - Build features dictionary - process documents and extract all possible features
//build features dictionary
var featureStatisticsDictBuilder = new FeatureStatisticsDictionaryBuilder();
Console.WriteLine("Building a features dictionary...");
var timeStart = DateTime.Now;
int itemsCnt = 0;
Dictionary<string, int> classLabels = new Dictionary<string, int>();
int maxClassLabelIndex = 0;
using (var filereader = new LabeledTextDocumentFileReader(inputRawFile))
{
while (!filereader.EndOfSource())
{
var doc = filereader.ReadDocument();
//class label and doc contents
string classLabel = doc.ClassLabel;
string docContent = doc.DocContent;
//build class labels dictionary
if (!classLabels.ContainsKey(classLabel))
{
classLabels[classLabel] = maxClassLabelIndex;
maxClassLabelIndex++;
}
Dictionary<string, double> docFeatures = new Dictionary<string, double>();
pipeline.ProcessDocument(docContent, docFeatures);
featureStatisticsDictBuilder.UpdateInfoStatisticsFromSingleDoc(docFeatures);
itemsCnt++;
if (itemsCnt % 500 == 0)
{
Console.WriteLine("{0} processed so far", itemsCnt);
}
}
}
//order classes by name - until now they are ordered by first occurence in dataset
var ordered = classLabels.OrderBy(cv => cv.Key);
var orderedClassLabels = new Dictionary<string, int>();
int classIndexCounter = 0;
foreach (var item in ordered)
{
orderedClassLabels.Add(item.Key, classIndexCounter);
classIndexCounter++;
}
classLabels = orderedClassLabels;
LexiconReaderHelper.SaveDictionaryToFile(classLabels, classLabelsOutputFileName);
Console.WriteLine("Class labels saved to file {0}", classLabelsOutputFileName);
Console.WriteLine("Extracted {0} features from {1} documents", featureStatisticsDictBuilder.FeatureInfoStatistics.Count, itemsCnt);
Console.WriteLine("Done - {0}", (DateTime.Now - timeStart));
RecomputeFeatureIndexes(featureStatisticsDictBuilder, minFeaturesFrequency);
Console.WriteLine("Selected {0} features with min freq {1} ", featureStatisticsDictBuilder.FeatureInfoStatistics.Count, minFeaturesFrequency);
//save fetures for later use
if (System.IO.File.Exists(featuresDictOutputFile))
{
System.IO.File.Delete(featuresDictOutputFile);
}
featureStatisticsDictBuilder.SaveToFile(featuresDictOutputFile);
Console.WriteLine("Features saved to file {0}", featuresDictOutputFile);
#endregion
//4- Load features from file
featureStatisticsDictBuilder.LoadFromFile(featuresDictOutputFile);
classLabels = LexiconReaderHelper.LoadDictionaryFromFile(classLabelsOutputFileName);
#region 5 - Build items with features from text documents and features dictionary
//Build libsvm file from text insput file and features dictionary
var sparseItemsWithIndexFeatures = new List<SparseItemInt>();
timeStart = DateTime.Now;
Console.WriteLine("Exporting to libsvm file format...");
using (System.IO.TextWriter textWriter = new System.IO.StreamWriter(libSvmOutputFileName, false))
{
LibSvmFileBuilder libSvmFileBuilder = new LibSvmFileBuilder(textWriter);
using (var filereader = new LabeledTextDocumentFileReader(inputRawFile))
{
while (!filereader.EndOfSource())
{
var doc = filereader.ReadDocument();
//class label and doc contents
string classLabel = doc.ClassLabel;
string docContent = doc.DocContent;
int classLabelIndex = classLabels[classLabel];
SparseItemInt sparseItem = ProcessingHelpers.ProcessTextAndGetSparseItem(pipeline, featureStatisticsDictBuilder, minFeaturesFrequency, normalize, scaleRange, docContent, classLabelIndex);
libSvmFileBuilder.AppendItem(sparseItem.Label, sparseItem.Features);
sparseItemsWithIndexFeatures.Add(sparseItem);
//B - Or extract indexed features and append
//libSvmFileBuilder.PreprocessStringFeaturesAndAppendItem(classLabelIndex, docFeatures, featureStatisticsDictBuilder.FeatureInfoStatistics, minFeaturesFrequency);
}
}
}
Console.WriteLine("Done - {0}", (DateTime.Now - timeStart));
Console.WriteLine("Libsvm file saved to {0}", libSvmOutputFileName);
Console.WriteLine();
#endregion
#region 6 - Train and test classifier
//LIBLINEAR - TRAIN AND EVAL CLASSIFIER
//Build problem X and Y
//Split data on train and test dataset
int trainRate = 4;
int testRate = 1;
int allItemsCnt = sparseItemsWithIndexFeatures.Count;
int trainCnt = (int)((double)allItemsCnt * trainRate / (double)(trainRate + testRate));
int testCnt = allItemsCnt - trainCnt;
var trainItems = sparseItemsWithIndexFeatures.Take(trainCnt).ToList();
var testItems = sparseItemsWithIndexFeatures.Skip(trainCnt).Take(testCnt).ToList();
string trainDataModelFileName = inputRawFile + ".train.model";
FeatureNode[][] problemXTrain = null;
double[] problemYTrain = null;
SetLibLinearProblemXandYFromSparseItems(trainItems, out problemXTrain, out problemYTrain);
TrainLibLinearProblemAndSveModel(liblinearSolver, liblinearC, liblinearEps, featureStatisticsDictBuilder, trainDataModelFileName, problemXTrain, problemYTrain);
var modelFileLoad = new java.io.File(trainDataModelFileName);
var modelLoaded = Model.load(modelFileLoad);
//evaluation
List<FeatureNode[]> problemXEvalList = new List<FeatureNode[]>();
List<double> problemYEvalList = new List<double>();
//SetLibLinearProblemXandYFromSparseItems(testItems, out problemXEval, out problemYEval);
//EVALUATE
List<double> predictedY = new List<double>();
foreach (var item in testItems)
{
var itemFeatureNodes = ProcessingHelpers.ConvertToSortedFeatureNodeArray(item.Features);
//fill eval list
problemXEvalList.Add(itemFeatureNodes);
problemYEvalList.Add((double)item.Label);
//predict
double prediction = Linear.predict(modelLoaded, itemFeatureNodes);
predictedY.Add(prediction);
}
int[][] matrix = ResultCalculationMetricsHelpers.BuildConfusionMatrix(problemYEvalList.ToArray(), predictedY, classLabels.Count);
Console.WriteLine("Class labels:");
foreach (var label in classLabels)
{
Console.WriteLine(string.Format("{1} - {0}", label.Key, label.Value));
}
Console.WriteLine();
ResultCalculationMetricsHelpers.PrintMatrix(matrix, true);
for (int i = 0; i < matrix.Length; i++)
{
int truePositivesCnt = matrix[i][i];
int falsePositievesCnt = matrix[i].Sum() - matrix[i][i];
int falseNegativesCnt = matrix.Select(m => m[i]).Sum() - matrix[i][i];
double precision;
double recall;
double fScore;
ResultCalculationMetricsHelpers.CalculatePRF(truePositivesCnt, falsePositievesCnt, falseNegativesCnt, out precision, out recall, out fScore);
Console.WriteLine(string.Format("[{0} - {4}] P={1:0.0000}, R={2:0.0000}, F={3:0.0000} ", i, precision, recall, fScore, orderedClassLabels.ToList().ToDictionary(kv => kv.Value, kv => kv.Key)[i]));
}
int truePositivesCntOverall = 0;
int testedCnt = 0;
for (int i = 0; i < matrix.Length; i++)
{
truePositivesCntOverall += matrix[i][i];
testedCnt += matrix[i].Sum();
}
double accuracyOverall = (double)truePositivesCntOverall / (double)testedCnt;
Console.WriteLine(string.Format("[{0}] Accuracy={1:0.0000}", "Overall", accuracyOverall));
//----TRAIN MODEL IWTH ALL DATA
Console.WriteLine("------------------------------------------------");
Console.WriteLine("Train model on all data");
FeatureNode[][] problemXAll = null;
double[] problemYAll = null;
var allItems = sparseItemsWithIndexFeatures;
SetLibLinearProblemXandYFromSparseItems(allItems, out problemXAll, out problemYAll);
TrainLibLinearProblemAndSveModel(liblinearSolver, liblinearC, liblinearEps, featureStatisticsDictBuilder, modelOutputFileName, problemXAll, problemYAll);
//CROSSVALIDATION
//int crossValidationFold = 5;
//Console.WriteLine("Training with {0} items and 5 fold crossvalidation...", sparseItemsWithIndexFeatures.Count);
//timeStart = DateTime.Now;
//double[] target = new double[problem.l];
//Linear.crossValidation(problem, parameter, crossValidationFold, target);
//Console.WriteLine("Done - {0}", (DateTime.Now - timeStart));
//WriteResult(target);
#endregion
//Console.ReadKey();
//var instancesToTest = new Feature[] { new FeatureNode(1, 0.0), new FeatureNode(2, 1.0) };
//var prediction = Linear.predict(model, instancesToTest);
}
public LoadedApk(android.app.ActivityThread activityThread, string name, android.content.Context
systemContext, android.content.pm.ApplicationInfo info, android.content.res.CompatibilityInfo
compatInfo)
{
mActivityThread = activityThread;
mApplicationInfo = info != null ? info : new android.content.pm.ApplicationInfo();
mApplicationInfo.packageName = name;
mPackageName = name;
mAppDir = null;
mResDir = null;
mSharedLibraries = null;
mDataDir = null;
mDataDirFile = null;
mLibDir = null;
mBaseClassLoader = null;
mSecurityViolation = false;
mIncludeCode = true;
mClassLoader = systemContext.getClassLoader();
mResources = systemContext.getResources();
mCompatibilityInfo.set(compatInfo);
}
public LocalImpl(File path)
: base(path)
{
;
}
/// <summary>
/// Create information about a new .apk
/// NOTE: This constructor is called with ActivityThread's lock held,
/// so MUST NOT call back out to the activity manager.
/// </summary>
/// <remarks>
/// Create information about a new .apk
/// NOTE: This constructor is called with ActivityThread's lock held,
/// so MUST NOT call back out to the activity manager.
/// </remarks>
public LoadedApk(android.app.ActivityThread activityThread, android.content.pm.ApplicationInfo
aInfo, android.content.res.CompatibilityInfo compatInfo, android.app.ActivityThread
mainThread, java.lang.ClassLoader baseLoader, bool securityViolation, bool includeCode
)
{
mActivityThread = activityThread;
mApplicationInfo = aInfo;
mPackageName = aInfo.packageName;
mAppDir = aInfo.sourceDir;
mResDir = aInfo.uid == android.os.Process.myUid() ? aInfo.sourceDir : aInfo.publicSourceDir;
mSharedLibraries = aInfo.sharedLibraryFiles;
mDataDir = aInfo.dataDir;
mDataDirFile = mDataDir != null ? new java.io.File(mDataDir) : null;
mLibDir = aInfo.nativeLibraryDir;
mBaseClassLoader = baseLoader;
mSecurityViolation = securityViolation;
mIncludeCode = includeCode;
mCompatibilityInfo.set(compatInfo);
if (mAppDir == null)
{
if (android.app.ActivityThread.mSystemContext == null)
{
android.app.ActivityThread.mSystemContext = android.app.ContextImpl.createSystemContext
(mainThread);
android.app.ActivityThread.mSystemContext.getResources().updateConfiguration(mainThread
.getConfiguration(), mainThread.getDisplayMetricsLocked(compatInfo, false), compatInfo
);
}
//Slog.i(TAG, "Created system resources "
// + mSystemContext.getResources() + ": "
// + mSystemContext.getResources().getConfiguration());
mClassLoader = android.app.ActivityThread.mSystemContext.getClassLoader();
mResources = android.app.ActivityThread.mSystemContext.getResources();
}
}
protected override ch.cyberduck.core.local.Local create(File path)
{
return new LocalImpl(path);
}
public override java.io.File getFilesDir()
{
lock (mSync)
{
if (mFilesDir == null)
{
mFilesDir = new java.io.File(getDataDirFile(), "files");
}
if (!mFilesDir.exists())
{
if (!mFilesDir.mkdirs())
{
android.util.Log.w(TAG, "Unable to create files directory " + mFilesDir.getPath()
);
return null;
}
android.os.FileUtils.setPermissions(mFilesDir.getPath(), android.os.FileUtils.S_IRWXU
| android.os.FileUtils.S_IRWXG | android.os.FileUtils.S_IXOTH, -1, -1);
}
return mFilesDir;
}
}
